diff --git a/.travis.yml b/.travis.yml index 3594ca5f168a142d8c6f6e8b24f6eb496927f453..8572565b80b612618f7dd8f4c6eecf9bcf1a1b2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,7 +95,6 @@ script: - initdb --version - createdb --version - psql --version - - gpmigrator --version - gpssh --version - gpmapreduce --version - gpfdist --version diff --git a/gpAux/Makefile b/gpAux/Makefile index 486ad4294dd2fe87aebbbcbbb436fd01bb568ade..ac9f1e0e538905ca00cb88e52218a18853fe73ac 100644 --- a/gpAux/Makefile +++ b/gpAux/Makefile @@ -889,8 +889,6 @@ SET_VERSION_SCRIPTS = \ bin/gpinitsystem \ bin/gpload.py \ bin/gplogfilter \ - bin/gpmigrator \ - bin/gpmigrator_mirror \ bin/gpmovemirrors \ bin/gprecoverseg \ bin/gpreload \ diff --git a/gpMgmt/Makefile b/gpMgmt/Makefile index 93b755d25e404191e515fcfb40de38266f076b2c..2a7423325b0f5af4513ea6df5457c9d068788200 100644 --- a/gpMgmt/Makefile +++ b/gpMgmt/Makefile @@ -20,8 +20,6 @@ SET_VERSION_SCRIPTS = \ bin/gpinitsystem \ bin/gpload.py \ bin/gplogfilter \ - bin/gpmigrator \ - bin/gpmigrator_mirror \ bin/gpmovemirrors \ bin/gprecoverseg \ bin/gpreload \ diff --git a/gpMgmt/bin/README b/gpMgmt/bin/README index e7f8dc871bb22b531b3de64512e65647cffa9aa2..43c48266fa17b5e36af5a037e7e8b83f0cd69d38 100644 --- a/gpMgmt/bin/README +++ b/gpMgmt/bin/README @@ -43,7 +43,6 @@ bin/gpload - Sets env variables and calls gpload.py List of Management Scripts Written in Python (no libraries) ----------------------------------------------------------- bin/gpload.py - Loads data into a Greenplum Database -bin/gpmigrator - Upgrades from previous versions bin/gpsys1 - Print system information on a host (???) diff --git a/gpMgmt/bin/gpmigrator b/gpMgmt/bin/gpmigrator deleted file mode 100755 index e9f07bd73c16f9258884032dc4586766605a4eef..0000000000000000000000000000000000000000 --- a/gpMgmt/bin/gpmigrator +++ /dev/null @@ -1,3156 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -''' -gpmigrator [options] old-gphome new-gphome - -Options: - -h, --help show this help message and exit - -v, --version show the program's version number and exit - -q quiet mode - -d DIRECTORY the master host data directory - -l DIRECTORY log file directory - -R revert a previous gpmigrator run - --debug debugging information -''' - -#============================================================ -import sys, os - -# Python version 2.6.2 is expected, must be between 2.5-3.0 -if sys.version_info < (2, 5, 0) or sys.version_info >= (3, 0, 0): - sys.stderr.write("Error: %s is supported on Python versions 2.5 or greater\n" - "Please upgrade python installed on this machine." - % os.path.split(__file__)[-1]) - sys.exit(1) - -try: - from gppylib.operations.gpMigratorUtil import * -except ImportError, e: - sys.exit('Error: unable to import module: ' + str(e)) - -libdir = os.path.join(sys.path[0], 'lib/') - -logger = get_default_logger() -EXECNAME = os.path.split(__file__)[-1] -MIGRATIONUSER = 'gpmigrator' -LOCKEXT = '.gpmigrator_orig' -WORKDIR = 'gpmigrator' -BACKUPDIR = 'backup' -UPGRADEDIR = 'upgrade' -PARALLELISM = 16 - - - - -#============================================================ -__version__ = '$Revision$' - - -#============================================================ -def makeCommand(oldHome, newHome, oldVersion, newVersion, - command, pid, dataDirectories, filespaces, option1, - option2, method, isUpgrade, isRevert, mirrors): - - # space separated list of directories - datadirs = base64.urlsafe_b64encode(pickle.dumps(dataDirectories)) - fse = base64.urlsafe_b64encode(pickle.dumps(filespaces)) - - hasMirrors = len(mirrors) > 0 - - cmd = [ - EXECNAME, - oldHome, - newHome, - '--internal-oldversion=' + urllib.quote(str(oldVersion)), - '--internal-newversion=' + urllib.quote(str(newVersion)), - '--internal-command=' + str(command), - '--internal-pid=' + str(pid), - '--internal-dirs=' + urllib.quote(datadirs), - '--internal-filespaces=' + urllib.quote(fse), - '--internal-option=' + urllib.quote(str(option1)), - '--internal-option2=' + urllib.quote(str(option2)), - '--debug', - '--quiet', - ] - if method: - cmd.append('--internal-method='+ urllib.quote(str(method))) - if isUpgrade: - cmd.append('--internal-isupgrade') - if isRevert: - cmd.append('-R') - if hasMirrors: - cmd.append('--internal-mirrors') - - return cmd - -#============================================================ -class GpControlData(base.Command): - """ - Get the control data for a specified directory. - """ - def __init__(self, name, directory, ctxt=base.LOCAL, remoteHost=None): - self.controldata = None - - cmdStr = "$GPHOME/bin/pg_controldata %s" % directory - base.Command.__init__(self, name, cmdStr, ctxt, remoteHost) - - def get_controldata(self): - if not self.controldata: - self.controldata = {} - for line in self.results.stdout.split('\n'): - try: - (key, value) = line.split(':', 1) - self.controldata[key] = value.strip() - except: - pass - return self.controldata - - @staticmethod - def local(name,directory): - cmd=GpControlData(name,directory) - cmd.run(validateAfter=True) - return cmd.get_controldata() - - -#============================================================ -class GPUpgrade(GPUpgradeBase): - ''' - Greenplum Database Upgrade Utility - ''' - - # INTERNAL command: - # MASTER - default, run on master, [not included in list] - # MKDIR - Create directories, run on every host - # RMDIR - Remove directories, run on every host - # CHKDIR - Check directories, run on every host - # RESETXLOG - Syncronize xlogs - # DEEPLINK - Recursive hardlink - # EXTRACTCATFILES - build a list of catalog files to be copied - # BUILDSKEL - build a skeleton copy of the old system - # TRANSFORMCAT - apply catalog transformations - # SETCATVERSION - set catalog version - # EXTRACTAOSEGS - get ao seg info - # GETHIGHESTOID - find the highest OID in the cluster - commands = ['SETSTATE', 'MKDIR', 'RMDIR', 'CHKDIR', 'RESETXLOG', - 'DEEPLINK', 'CHKDOWN', 'LOCKDOWN', 'UNLOCK', - 'EXTRACTCATFILES', 'BUILDSKEL', 'TRANSFORMCAT', - 'SETCATVERSION', 'EXTRACTAOSEGS', 'GETHIGHESTOID', - 'TOUCHUPCONFIG', 'FIXCONFIG'] - - #------------------------------------------------------------ - def __init__(self): - ''' - The most basic of initialization - ''' - super(GPUpgrade, self).__init__() - - self.resume = False - self.mirrormode = None - - self.pid = os.getpid() # Process ID of the master - - # Non-master variables - self.option2 = None # Argument passed to cmd - - # Environment and system info - self.datadirs = None # Set of all data and mirror directories - self.filespaces= None # Set of filespace directory (dboid as key) - self.host = None # master hostname - self.upgrade = None # True for upgrade, False for downgrade - self.method = None # transform or dumprestore ? - - # Upgrade information and state - self.quiet = False - self.revert = False - self.checkschema = True - self.warnings = False - self.gpperfmon = None - self.oldversion = None - self.newversion = None - self.newmaster = None # directory: workdir/upgrade/gp-1 - self.state = None # file: backup/state - self.config = {} # gp_init_config for new db - self.segments = [] # list of segments - - #------------------------------------------------------------ - def Setup(self): - ''' - Basic initialization, separate from __init__ for exception - handling purposes. - ''' - - # GPHOME, PYTHONPATH must be setup properly - # GPHOME must match the location of this file - # The first PYTHONPATH must be based on the proper gphome - gphome_bin = os.path.realpath(os.path.split(__file__)[0]) - gphome = os.path.split(gphome_bin)[0] - env_GPHOME = os.path.realpath(os.environ.get('GPHOME')) - if (env_GPHOME != gphome): - logger.fatal(" $GPHOME is set to %s which is not newhome" % env_GPHOME) - logger.fatal(' source the greenplum.sh from the newhome to setup env ') - raise UpgradeError('Initialization failed') - - pythonpath = os.path.join(gphome, "lib", "python") - env_PYTHONPATH = os.path.realpath(os.environ.get('PYTHONPATH').split(':')[0]) - if (env_PYTHONPATH != pythonpath): - logger.fatal(' $PYTHONPATH is incorrect ') - logger.fatal(' source the greenplum.sh from the newhome to setup env ') - raise UpgradeError('Initialization failed') - - # This is the same path used by gpinitsystem - self.path = '/usr/kerberos/bin:/usr/sfw/bin:/opt/sfw/bin' - self.path += ':/usr/local/bin:/bin:/usr/bin:/sbin:/usr/sbin:/usr/ucb' - self.path += ':/sw/bin' - - # Set defaults - self.user = os.environ.get('USER') or os.environ.get('LOGNAME') - self.host = self.RunCmd('hostname') - self.masterdir = os.environ.get('MASTER_DATA_DIRECTORY') - - self.ParseInput() - - # Setup worker pool - self.pool = base.WorkerPool(numWorkers=PARALLELISM); - - # Extract path from input master directory - if self.cmd == 'MASTER': - logger.info('Beginning upgrade') - logger.info('Checking configuration') - - if not self.masterdir or len(self.masterdir) == 0: - raise UpgradeError('MASTER_DATA_DIRECTORY is not defined') - self.masterdir = self.masterdir.rstrip('/') - - # The configuration file - conf = os.path.join(self.masterdir, 'postgresql.conf') - - # Simple function to look for settings in the conf file - def getconf(x): - conf_re = re.compile('^\s*%s\s*=\s*(\w+)' % x) - try: - conf_str = self.RunCmd('grep %s %s' % (x, conf)) - except CmdError: - conf_str = "" # grep returns errorcode on no match - value = None - for line in conf_str.split('\n'): - match = conf_re.search(line) - if match: - value = match.group(1) - return value - - # Find the port for this segment: - self.masterport = getconf('port') - if self.masterport == None: - raise UpgradeError('Could not determine master port from ' + conf) - self.masterport = int(self.masterport) - - # Determine if perfmon is enabled - self.gpperfmon = getconf('gp_enable_gpperfmon') - - self.sock_dir = getconf('unix_socket_directory') - if not self.sock_dir: - self.sock_dir = '/tmp/' - - # Verify that (max_connections == superuser_reserved_connections) - # max_conn = getconf('max_connections') - # reserved = getconf('superuser_reserved_connections') - - masterpath, masterdir = os.path.split(self.masterdir) - self.workdir = os.path.join(masterpath, WORKDIR) - self.newmaster = os.path.join(self.workdir, UPGRADEDIR, masterdir) - self.oldenv = self.SetupEnv(self.oldhome, self.masterdir) - self.newenv = self.SetupEnv(self.newhome, self.newmaster) - - else: - self.oldenv = self.SetupEnv(self.oldhome, None) - self.newenv = self.SetupEnv(self.newhome, None) - - #------------------------------------------------------------ - def ParseInput(self): - ''' - Parses and validates input to the script - ''' - - try: - parser = optparse.OptionParser(usage=(cli_help(EXECNAME) or __doc__), add_help_option=False) - parser.add_option('-v', '--version', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('-h', '--help', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('-d', dest='directory', - help=optparse.SUPPRESS_HELP) - parser.add_option('-l', dest='logdir', - help=optparse.SUPPRESS_HELP) - parser.add_option('-q', '--quiet', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('-R', dest='revert', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('-c', '--check-only', dest='checkonly', action='store_true', - help=optparse.SUPPRESS_HELP) - - parser.add_option('--debug', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-disable-checkschema', - dest='nocheckschema', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-command', dest='command', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-pid', dest='pid', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-dirs', dest='datadirs', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-filespaces', dest='fse', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-option', dest='option', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-option2', dest='option2', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-method', dest='method', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-oldversion', dest='oldversion', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-newversion', dest='newversion', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-isupgrade', dest='upgrade', - action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-mirrors', dest='mirrors', - action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('--skip-checkcat', dest='skipcheckcat', - action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('--fault-injection', dest='faultinjection', - type='int', help=optparse.SUPPRESS_HELP) - (options, args) = parser.parse_args() - - if options.version: - print EXECNAME + ' ' + __version__ - sys.exit(0) - - if options.help: - usage(EXECNAME) - sys.exit(0) - - if len(args) != 2: - usage(EXECNAME) - msg = "incorrect number of arguments" - if len(args) > 0: - msg += ": %s" % str(args) - parser.error(msg) - - except Exception, e: - usage(EXECNAME) - raise UpgradeError('Error parsing input: ' + str(e)) - - if options.revert: - self.revert = True - - if options.checkonly: - self.checkonly = True - - if options.directory: - self.masterdir = options.directory - if options.command: - if options.command not in self.commands: - parser.error('INVALID INTERNAL COMMAND: ' + options.command) - self.cmd = options.command - if options.pid: - if self.cmd == 'MASTER': - parser.error('INVALID INTERNAL COMMAND: ' + self.cmd) - self.pid = int(options.pid) - self.oldhome = args[0].rstrip('/') - self.newhome = args[1].rstrip('/') - if options.datadirs: - self.datadirs = pickle.loads(base64.urlsafe_b64decode((urllib.unquote(options.datadirs)))) - if options.fse: - self.filespaces = pickle.loads(base64.urlsafe_b64decode((urllib.unquote(options.fse)))) - if options.option: - self.option = urllib.unquote(options.option) - if options.option2: - self.option2 = urllib.unquote(options.option2) - if options.debug: - self.debug = True - enable_verbose_logging() - if options.quiet: - self.quiet = True - quiet_stdout_logging() - if options.logdir: - self.logdir = options.logdir - else: - self.logdir = os.path.join(os.environ['HOME'], 'gpAdminLogs') - - # --internal_mirrors just indicates that the cluster has mirrors, - # for simplicity we have self.mirrors still point to a list, but - # its contents are not directly useful. - if options.mirrors: - self.mirrors = [True] - - try: - setup_tool_logging(EXECNAME, - unix.getLocalHostname(), - unix.getUserName(), - self.logdir) - except OSError, e: - logger.fatal('cannot log to %s: %s' % (self.logdir, str(e))) - exit(1) - - if options.nocheckschema: - self.checkschema = False - - if self.cmd != 'MASTER': - self.oldversion = GpVersion(urllib.unquote(options.oldversion)) - self.newversion = GpVersion(urllib.unquote(options.newversion)) - self.method = options.method - if options.upgrade: - self.upgrade = True - else: - self.upgrade = False - self.skipcheckcat = options.skipcheckcat - self.faultinjection = options.faultinjection - - if self.cmd == 'MASTER': - self.method = "transform" # Currently the only supported method - - today = date.today().strftime('%Y%m%d') - if not os.path.isdir(self.logdir): - os.makedirs(self.logdir, 0700) - logname = os.path.join(self.logdir, '%s_%s.log' % (EXECNAME, today)) - self.logfile = open(logname, 'a') - - self.pid = os.getpid() - if self.masterdir == None: - logger.fatal('MASTER_DATA_DIRECTORY parameter not set') - raise UpgradeError('Initialization failed') - if not os.path.exists(self.masterdir): - logger.fatal('MASTER_DATA_DIRECTORY: %s not found' - % self.masterdir) - raise UpgradeError('Initialization failed') - if not os.path.isabs(self.masterdir): - self.masterdir = os.path.abspath(self.masterdir) - if not os.path.exists(self.oldhome): - logger.fatal(' directory not found: ' + self.oldhome) - raise UpgradeError('Initialization failed') - if not os.path.exists(self.newhome): - logger.fatal(' directory not found: ' + self.newhome) - raise UpgradeError('Initialization failed') - if not os.path.isabs(self.oldhome): - self.oldhome = os.path.abspath(self.oldhome) - if not os.path.isabs(self.newhome): - self.newhome = os.path.abspath(self.newhome) - - - #------------------------------------------------------------ - def Cleanup(self): - ''' - Cleanup open connections. - Separate from __del__ because that caused weird issues with exit() - behavior. - ''' - if self.cmd == 'MASTER': - logger.fatal('Fatal error occurred - Recovering') - try: - self.Shutdown() - except Exception, e: - logger.fatal('Cleanup failure: ' + str(e)) - - try: - self.ReleaseLockdown(self.oldenv) - except Exception, e: - logger.fatal('Cleanup failure: ' + str(e)) - - if self.pool: - del self.pool - - - - #------------------------------------------------------------ - def SetState(self, newstate): - ''' - Sets the state in the upgrader state file - ''' - - # Set state across all hosts, if all hosts succeed then set the - # master state - if self.cmd == 'MASTER': - self.CallSlaves('SETSTATE', newstate) - try: - self.state.write(str(newstate) + "\n") - self.state.flush() - except Exception, e: - raise UpgradeError('Error writing to statefile: (%s)' % str(e)) - else: - - # Build up a unique set of workdirectories on this host - locations = set() - for d in self.datadirs: - (location, _) = os.path.split(d) - locations.add(os.path.join(location, WORKDIR)) - - for d in locations: - statefile = os.path.join(d, 'state') - file = open(statefile, 'a') - file.write(str(newstate) + "\n") - file.close() - - #------------------------------------------------------------ - def CallSlaves(self, cmd, option='', option2='', includeMirrors=False): - ''' - Calls every host to execute the given command with the given option - value - ''' - - logger.debug("Remote call: %s" % cmd) - - # Check for things that should never happen - if self.cmd != 'MASTER': - raise UpgradeError("Recursive communication error") - if not self.array: - raise UpgradeError("Failure initializing array") - if not self.hostcache: - raise UpgradeError("Failure initializing host cache") - if not self.pool: - raise UpgradeError("Failure initializing worker pool") - - if self.upgrade or self.revert: - gphome = self.newhome - else: - gphome = self.oldhome - - # Construct the commands to pass to the worker pool - hosts = self.hostcache.get_hosts() - for host in hosts: - hostname = host.hostname - - # Skip any hosts in the cache that contain no segments for this - # configuration. - if len(host.dbs) == 0: - continue - - # Get the data directories for this host: - datadirs = [] - for seg in host.dbs: - if includeMirrors or seg.isSegmentPrimary(): - datadirs.append(seg.getSegmentDataDirectory()) - - fse = [] - for seg in host.dbs: - if includeMirrors or seg.isSegmentPrimary(): - sfs = seg.getSegmentFilespaces(); - for (oid, dir) in sfs.iteritems(): - fse.append(dir) - - # Skip any hosts that have no applicable data directories - if len(datadirs) == 0: - continue - - cmdList = makeCommand(oldHome=self.oldhome, - newHome=self.newhome, - oldVersion=self.oldversion, - newVersion=self.newversion, - command=cmd, - pid=self.pid, - dataDirectories=datadirs, - filespaces=fse, - option1=option, - option2=option2, - method=self.method, - isUpgrade=self.upgrade, - isRevert=self.revert, - mirrors=self.mirrors) - - c = GpUpgradeCmd("gpmigrator_mirror remote call", - cmdList, - ctxt=base.REMOTE, - remoteHost=hostname) - - self.pool.addCommand(c) - - # Wait for the segments to finish - try: - self.pool.join() - except: - self.pool.haltWork() - self.pool.joinWorkers() - - failure = False - results = [] - for cmd in self.pool.getCompletedItems(): - r = cmd.get_results() - - # Going through the gppylib Command interface all stderr from the - # remote calls gets redirected to stdout, which is unfortunate - # because we'd like to be able to differentiate between the two. - # - # We keep the stdout chatter to a minimum by passing --quiet to - # the remote calls which performs quiet stdout logging. - - # sys.stderr.write(r.stderr) - msg = r.stdout.strip() - results.append(msg) - - if not cmd.was_successful(): - log_literal(logger, logging.ERROR, msg) - failure=True - - if failure: - raise UpgradeError("Fatal Segment Error") - - # Warning this output contains everything written to stdout, - # which unfortunately includes some of the logging information - return "\n".join(results) - - - #------------------------------------------------------------ - def CheckDirectories(self): - if self.cmd == 'MASTER': - fsdirs = self.masterfilespace - else: - fsdirs = self.filespaces - - err = False - - # Build up a unique set of workdirectories on this host - locations = set() - for d in fsdirs: - (location, _) = os.path.split(d) - locations.add(os.path.join(location, WORKDIR)) - - # Check that none of them exist - for d in locations: - if os.path.exists(d): - logger.warn('%s : File exists' % d) - err = True - - if err: - raise UpgradeError('Directories exist') - - - - #------------------------------------------------------------ - def Revert(self): - ''' - Revert to backup - ''' - - # Check for indication that something is running: - if not self.dbup: - env = None - try: - env = self.CheckUp() - except UpgradeError: - pass - - # We use the catalog versions we cached during versionCheck to - # determine if the MASTER_DATA_DIRECTORY is actually an old - # master data directory, or a partially upgraded one. - # - # Note: it would be better to adjust gpstop so that it knows how - # to stop old versions and always shutdown using a new environment. - # - # Note: It would also be good to avoid starting and stopping more - # than necessary here. - if env == self.oldenv and self.datacat == self.newcat: - self.newenv['MASTER_DATA_DIRECTORY'] = \ - self.oldenv['MASTER_DATA_DIRECTORY'] - env = self.newenv - - if env: - self.dbup = [env, False] - - # Now that we've determined what, if anything, is running, we must - # stop it. - try: - if self.faultinjection == -1: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - self.Shutdown() - self.CheckDown(self.revert) - except BaseException, e: - logger.fatal('***************************************') - logger.fatal(str(e)) - logger.fatal('***************************************') - raise UpgradeError("Unable to REVERT\nShutdown the running " + - "database and rerun gpmigrator with '-R'") - - logger.info('Reverting to Backup') - self.revert = True - - statefile = os.path.join(self.workdir, 'state') - if not os.path.exists(statefile): - logger.warn(" '%s' not found" % statefile) - logger.info('Revert aborted') - sys.exit(0) - state = self.RunCmd('cat ' + statefile) - - needrevert = False - for line in state.split('\n'): - if line == 'BACKUP_VALID': - needrevert = True - - hba = os.path.join(self.masterdir, 'pg_hba.conf'+LOCKEXT) - ident = os.path.join(self.masterdir, 'pg_ident.conf'+LOCKEXT) - if os.path.exists(hba) or os.path.exists(ident): - needunlock = True - else: - needunlock = False - - if (needrevert or needunlock): - try: - self.ReadInfo() - except UpgradeError, e: - logger.fatal(str(e)) - logger.info('Revert aborted') - sys.exit(1) - - # Attempt revert - if needrevert: - logger.info('Restoring directories') - self.DeepLink('REVERT') - - # Truncate the statefile - # + Must occur AFTER actual Revert is processed - open(statefile, 'w').truncate() - - if needunlock: - self.ReleaseLockdown(self.oldenv) - - self.Startup(self.oldenv) - logger.info('Removing upgrade user') - self.Update('DROP USER IF EXISTS ' + MIGRATIONUSER) - self.Shutdown() - - # Cleanup anything leftover from gpupgrademirror - # + Should occur when the database is STOPPED - # + This forces a full remirroring of the system again, we - # should avoid this whenever possible. - logger.info('Performing Mirror rollback') - cmd = '%s/sbin/gpupgrademirror.py --rollback' % self.newhome - self.RunCmd(cmd, env=self.newenv) - - # done - logger.info('Revert Successful') - self.revert = False - - - #------------------------------------------------------------ - def CheckVersions(self): - ''' - Validates that the upgrade from old->new is okay - ''' - - # Log the OS type info - os_type = os.uname()[0] - os_ver = os.uname()[2] - logger.info('Operating System: %s %s' % (os_type, os_ver)) - - # Log version checking - logger.info('Checking version compatibility') - - # If we got a version, but it isn't recognized by the GpVersion class - # then it is likely not a Greenplum database. - oldversion = self.getversion(self.oldhome, self.oldenv) - newversion = self.getversion(self.newhome, self.newenv) - try: - oldversion = GpVersion(oldversion) - except: - raise UpgradeError('Source not a Greenplum Database: ' + oldversion) - try: - newversion = GpVersion(newversion) - except: - raise UpgradeError('Target not a Greenplum Database: ' + newversion) - - logger.info('Source Version: (Greenplum Database) %s' % str(oldversion)) - logger.info('Target Version: (Greenplum Database) %s' % str(newversion)) - - self.oldversion = oldversion - self.newversion = newversion - self.upgrade = (newversion >= oldversion) - - is_supported_version(oldversion, self.upgrade) - is_supported_version(newversion, self.upgrade) - - if newversion == oldversion: - raise UpgradeError("Greenplum Database is already version '%s'" - % str(newversion)) - elif newversion.isVersionRelease(oldversion): - raise UpgradeError("Upgrade not needed to go from version '%s' to version '%s'" - % (str(oldversion), str(newversion))) - - # We don't support downgrade - if not self.upgrade: - raise UpgradeError("Downgrade to Greenplum Database %s not supported from gpmigrator" - % str(newversion)) - - if self.upgrade and not newversion.isVersionCurrentRelease(): - main = GpVersion('main') - raise UpgradeError( - "Upgrade from '%s' to '%s' not supported. Target version should be Greenplum Database %s" - % (str(oldversion), str(newversion), main.getVersionRelease())) - - logger.debug('Using %s method for migration' % self.method) - - # Compare old version catalog number with the catalog in - # MASTER_DATA_DIRECTORY - catalog_re = re.compile('Catalog version number: *(.*)') - oldcat = self.RunCmd('postgres --catalog-version', env=self.oldenv) - logger.info("Source %s" % oldcat) - newcat = self.RunCmd('postgres --catalog-version', env=self.newenv) - logger.info("Target %s" % newcat) - - # May need this info later - self.oldcat = oldcat - self.newcat = newcat - - control = self.RunCmd('pg_controldata ' + self.masterdir, env=self.oldenv) - for line in control.split('\n'): - m = catalog_re.match(line) - if m: - self.datacat = line - logger.info("Data %s " % line) - if ((not self.revert and line != oldcat) or - (self.revert and line != oldcat and line != newcat)): - logger.debug('catalog mismatch: expected %s, found %s' - % (oldcat, line)) - msg = 'Catalog in %s does not match source binary' % self.masterdir - raise UpgradeError(msg) - break - - # For the moment everything goes: - # - Additional checks will be made once the old database is up - logger.info('Versions are compatible') - - - #------------------------------------------------------------ - def RemoveDirectories(self, force=False): - ''' - Removes upgrade/ and backup/ directories on all segments - ''' - - # If force is set then we remove the directories regardless of - # whether we created them or they were created by another process. - # If force is not set then we will throw an exception when we are - # done if there were directories that we failed to remove. - doerror = False - if self.cmd == 'MASTER': - logger.info('Removing temporary directories') - self.CallSlaves('RMDIR', force, includeMirrors=True) - fsdirs = self.masterfilespace - else: - fsdirs = self.filespaces - - # Build up a unique set of workdirectories on this host - locations = set() - for d in fsdirs: - (location, _) = os.path.split(d) - locations.add(os.path.join(location, WORKDIR)) - - for d in filter(os.path.exists, locations): - flag = os.path.join(d, 'flag') - if (not force) and os.path.exists(flag): - try: - dpid = self.RunCmd('cat ' + flag) - dpid = int(dpid) - except Exception, e: - logger.warn('Read error on file: ' + str(e)) - doerror = True - continue - - if dpid != self.pid: - logger.warn('Directory %s owned by pid %s' - % (d, dpid)) - doerror = True - continue - - # Regardless of the setting of force, if there is an active - # process running, we won't remove the directory. - running = self.RunCmd('find %s -name postmaster.pid -print' % d) - if (running): - logger.info(running) - logger.fatal('Active processes running') - doerror = True - else: - self.RunCmd('rm -rf ' + d) - - # We hold off throwing the error until the end, because we want to make - # sure we removed all of our own directories. - if doerror: - raise UpgradeError('Failed to remove old upgrade directories') - - #------------------------------------------------------------ - def CreateDirectories(self): - ''' - Creates the upgrade/ and backup/ directories on all segments - ''' - - if self.cmd == 'MASTER': - - dirs_exist = False - try: - self.CheckDirectories() - self.CallSlaves('CHKDIR', includeMirrors=True) - except UpgradeError, e: - dirs_exist = True - if dirs_exist: - self.RemoveDirectories(force=True) - - - logger.info('Creating temporary directories') - - # Have the slaves create their directories - self.CallSlaves('MKDIR', includeMirrors=True) - - fsdirs = self.masterfilespace - - else: - fsdirs = self.filespaces - if self.cmd != 'MKDIR': - raise Exception(self.cmd + ' != MKDIR]') - - # Given all the actual segment datadirs we need to look one - # directory up for the datadir locations and create the gpmigrator - # subdirectory there. - locations = set() - for d in fsdirs: - if not os.path.exists(d): - raise UpgradeError('Missing Data Directory: %s' % d) - (location, _) = os.path.split(d) - workpath = os.path.join(location, WORKDIR) - locations.add(workpath) - - # Actually create the directory, none of the should exist yet! - for d in locations: - # Creating directories is an atomic operation, if mkdir - # succeeds then it did not exist previously making it ours - # This prevents mulitiple upgrades running simultaneously. - os.mkdir(d) - self.RunCmd('chmod 700 %s' % d) - - # Claim it in the name of our parent process. We make use of - # this flag during tear down to confirm that we created it. - flag = open(os.path.join(d, 'flag'), 'w') - flag.write('%s\n' % str(self.pid)) - flag.close() - - # Create subdirectories for /upgrade and /backup - upgradepath = os.path.join(d, UPGRADEDIR) - os.mkdir(upgradepath) - self.RunCmd('chmod 700 %s' % upgradepath) - - backuppath = os.path.join(d, BACKUPDIR) - os.mkdir(backuppath) - self.RunCmd('chmod 700 %s' % backuppath) - - if (self.cmd == 'MASTER'): - # Finally now that we have those directories we'll make a state file - self.state = open('%s/state' % self.workdir, 'w') - - #------------------------------------------------------------ - def GenerateConfigFiles(self): - ''' - Creates 'gp_databases' 'gp_array_config' and 'gp_config' files - ''' - - logger.info('Generating config files') - - # Write a dump of the databases hash - db_file = os.path.join(self.workdir, 'gp_databases') - o = open(db_file, 'w') - for (oid, name) in self.dbs.iteritems(): - o.write('%s:%s\n' % (oid, name)) - o.close() - - # Write a dump of the gparray object - array_file = os.path.join(self.workdir, 'gp_array_config') - self.array.dumpToFile(array_file) - - # Write our new gp_config file, this contains the configuration - # information not captured in gp_array_config. - config_file = os.path.join(self.workdir, 'gp_config') - o = open(config_file, 'w') - for item, value in self.config.iteritems(): - if type(value) == int: - o.write('%s=%d\n' % (str(item), value)) - else: - o.write('%s=%s\n' % (str(item), str(value))) - o.close() - logger.info('Configuration files generated') - - - #------------------------------------------------------------ - def ReadInfo(self): - ''' - When restarting an upgrade this method reads the configuration - information from the cached status files. - - It corresponds directly with ExtractInfo() which gathers the same - information from an active database. - - It is also the inverse of GenerateConfigFiles() which creates the files - that this function reads. - - In normal processing usually ExtractInfo() is called rather than - ReadInfo(). This function is used /instead/ when we are doing Revert() - processing, or when resuming an upgrade. E.g. cases when we do not - want to start the database to get the information we need because the - database may not be in a stable state. - ''' - - # self.array - configuration information - array_config = os.path.join(self.workdir, 'gp_array_config') - if not os.path.exists(array_config): - raise UpgradeError(" '%s' not found" % array_config) - self.array = GpArray.initFromFile(array_config) - - # self.dbs - database information - db_file = os.path.join(self.workdir, 'gp_databases') - if not os.path.exists(db_file): - raise UpgradeError(" '%s' not found" % db_file) - f = open(db_file, 'r') - for line in f: - (oid, name) = line.split(':', 1) - self.dbs[oid] = name.strip() - f.close() - - # self.hostcache - cache of hostnames - self.hostcache = GpHostCache(self.array, self.pool) - failed_pings = self.hostcache.ping_hosts(self.pool) - if len(failed_pings) > 0: - raise UpgradeError( - "Cannot upgrade while there are unreachable hosts") - - # Setup the master filespace dir - self.masterfilespace = [] - for (id, loc) in self.array.master.getSegmentFilespaces().iteritems(): - self.masterfilespace.append(loc) - - # self.primaries and self.mirrors - dbs = self.array.getSegDbList() - self.segments = filter(GpDB.isSegmentPrimary, dbs) - self.mirrors = filter(GpDB.isSegmentMirror, dbs) - - # We don't support mirror any more in this script. - if (self.mirrors): - raise UpgradeError( - "Please use gpmigrator_mirror to upgrade a cluster with mirroring") - - # self.config - config_file = os.path.join(self.workdir, 'gp_config') - if not os.path.exists(config_file): - raise UpgradeError(" '%s' not found" % config_file) - f = open(config_file, 'r') - for line in f: - (key, value) = line.split('=', 1) - self.config[key] = value - f.close() - - - #------------------------------------------------------------ - def ExtractInfo(self): - ''' - Get all the information we need from the old instance - ''' - - # Can only extract with a running database - env = self.CheckUp() - - # -- Get the array information - logger.info('Extracting configuration information') - port = self.masterport - user = env['USER'] - url = dbconn.DbURL(port=port, dbname='template1', username=user) - array = GpArray.initFromCatalog(url, utility=True) - self.array = array - - # -- Determine if there are any invalid segments - logger.info('Checking that all segments are valid') - invalid = array.get_invalid_segdbs() - if len(invalid) > 0: - for db in invalid: - logger.fatal('INVALID SEGMENT: %s:/%s' % - (db.getSegmentHostName(), - db.getSegmentDataDirectory())) - raise UpgradeError('Cannot upgrade database with invalid segments') - - - # -- Get a list of available databases, note this includes "template0"! - logger.info('Creating list of databases') - databases = self.Select("SELECT oid, datname from pg_database") - for row in databases: - self.dbs[str(row['oid'])] = row['datname'] - - # -- Look for an existing user named MIGRATIONUSER - logger.info('Checking gpmigrator user') - upgradeuser = self.Select( - "SELECT count(*)::numeric FROM pg_user " + - "WHERE usename = '%s'" % MIGRATIONUSER) - if upgradeuser[0] > 0: - logger.warn("Database user '%s' already exists" % MIGRATIONUSER) - self.Update('DROP USER ' + MIGRATIONUSER) - - if (not self.upgrade): - raise UpgradeError("Downgrade not supported") - - # -- Older releases didn't have hostname as part of configuration - # make sure we have an acurate list of address->host lookups. - logger.info('Validating hosts') - self.hostcache = GpHostCache(self.array, self.pool) - for host in self.hostcache.get_hosts(): - for seg in host.dbs: - seg.setSegmentHostName(host.hostname) - failed_pings = self.hostcache.ping_hosts(self.pool) - if len(failed_pings) > 0: - raise UpgradeError("Cannot upgrade while there are unreachable " - "hosts") - - # Setup the master filespace dirs - self.masterfilespace = [] - for (id, loc) in self.array.master.getSegmentFilespaces().iteritems(): - self.masterfilespace.append(loc) - - master = array.master - standbymaster = array.standbyMaster - - # Oddly gparray doesn't have methods for fetching primary segments - # specifically - dbs = array.getSegDbList() - self.segments = filter(GpDB.isSegmentPrimary, dbs) - self.mirrors = filter(GpDB.isSegmentMirror, dbs) - - # We don't support mirror any more in this script. - if (self.mirrors): - raise UpgradeError( - "Please use gpmigrator_mirror to upgrade a cluster with mirroring") - - # Internal sanity checking - if len(self.segments) == 0: - raise UpgradeError('No segments found') - if standbymaster > 0: - raise UpgradeError('Cannot upgrade while standbymaster is running') - - - # To upgrade mirrors from 3.3 format mirrors to 4.0 format mirrors we - # must have been provided with the replication ports. - if len(self.mirrors) > 0: - if (not self.config['REPLICATION_PORT_BASE'] or - not self.config['MIRROR_REPLICATION_PORT_BASE'] or - not self.mirrormode): - logger.fatal("System configured with mirrors:") - if not self.config['REPLICATION_PORT_BASE']: - logger.fatal(" missing option: " - "--replication_port_base=") - if not self.config['MIRROR_REPLICATION_PORT_BASE']: - logger.fatal(" missing option: " - "--mirror_replication_port_base=") - if not self.mirrormode: - logger.fatal(" missing option: --mirror-mode") - raise UpgradeError("Missing required upgrade options") - - if self.mirrormode not in ('redundant', 'single'): - logger.fatal("Unrecognised mirror-mode: %s" % self.mirrormode) - logger.fatal("Valid modes are: redundant, single") - raise UpgradeError("Invalid mirror-mode") - - # Setup the replication ports for all segments in the array by host - for host in self.hostcache.get_hosts(): - - # The port ranges we are assigning into - p_port = self.config['REPLICATION_PORT_BASE'] + 1 - m_port = self.config['MIRROR_REPLICATION_PORT_BASE'] + 1 - - # Used to determine overlapping port ranges - pri_port = [65535, 0] # [min, max] - mir_port = [65535, 0] - pri_rep_port = [65535, 0] - mir_rep_port = [65535, 0] - - for seg in host.dbs: - port = seg.getSegmentPort() - if seg.isSegmentPrimary(): - seg.setSegmentReplicationPort(p_port) - pri_port[0] = min(pri_port[0], port) - pri_port[1] = max(pri_port[1], port) - pri_rep_port[0] = min(pri_rep_port[0], p_port) - pri_rep_port[1] = max(pri_rep_port[1], p_port) - p_port += 1 - else: - seg.setSegmentReplicationPort(m_port) - mir_port[0] = min(mir_port[0], port) - mir_port[1] = max(mir_port[1], port) - mir_rep_port[0] = min(mir_rep_port[0], m_port) - mir_rep_port[1] = max(mir_rep_port[1], m_port) - m_port += 1 - - # Check for overlapping port ranges: - port_mat = [pri_port, mir_port, pri_rep_port, mir_rep_port] - label_mat = ["Primary", "Mirror", "Replication", - "Mirror-Replication"] - - for x in range(0, 4): - for y in range(x+1, 4): - if ((port_mat[x][0] >= port_mat[y][0] and - port_mat[x][0] <= port_mat[y][1]) or - (port_mat[x][1] >= port_mat[y][0] and - port_mat[x][1] <= port_mat[y][1])): - logger.error("Port collision on host %s" % - host.hostname) - logger.error("... %s port range: [%d-%d]" % - (label_mat[x], - port_mat[x][0], - port_mat[x][1])) - logger.error("... %s port range: [%d-%d]" % - (label_mat[y], - port_mat[y][0], - port_mat[y][1])) - raise UpgradeError("Port collision on host %s" % - host.hostname) - - # Check for unsupported index types. - found = False - logger.info("Validating indexes") - oids = sorted(self.dbs.keys()) - for dboid in oids: - db = self.dbs[dboid] - - if db == "template0": - continue - indexes = self.Select(''' - SELECT quote_ident(n.nspname) || '.' || - quote_ident(c.relname) as index, - m.amname as kind - FROM pg_class c - join pg_namespace n on (c.relnamespace = n.oid) - join pg_am m on (c.relam = m.oid) - WHERE c.relkind = 'i' and m.amname in ('gin', 'hash') - ''', db=db) - if len(indexes) > 0: - if found == False: - logger.fatal("Unable to upgrade the following indexes") - found = True - logger.fatal(" Database: %s" % db) - for row in indexes: - logger.fatal(" %s [%s]" % (row['index'], row['kind'])) - if found: - raise UpgradeError("Deprecated index types must be removed prior " - "to upgrade") - - # Check for SQL_ASCII database encoding - logger.info("Validating database encoding") - encoding = self.Select("SELECT datname FROM pg_database WHERE encoding=0") - if len(encoding) > 0: - logger.error("Deprecated database encodings found:") - for datname in encoding: - logger.error(" %s [SQL_ASCII]" % datname) - raise UpgradeError("Deprecated database encodings found - contact " - "Greenplum Support") - - # ARRAY_NAME - arrayname = self.Select('SELECT gpname FROM gp_id') - self.config['ARRAY_NAME'] = arrayname[0] - - # Determine PORT_BASE, (min of segments - 1) - port_base = array.get_min_primary_port() - 1 - self.config['PORT_BASE'] = port_base - - # Determine MIRROR_PORT_BASE, - if len(self.mirrors) > 0: - mirror_port_base = array.get_min_mirror_port() - 1 - self.config['MIRROR_PORT_BASE'] = mirror_port_base - - # MASTER_HOSTNAME, super easy - self.config['MASTER_HOSTNAME'] = master.getSegmentHostName() - - # MASTER_DIRECTORY, also easy - we already set it up - self.config['MASTER_DIRECTORY'] = self.newmaster - - # MASTER_PORT, super easy - self.config['MASTER_PORT'] = master.getSegmentPort() - - # TRUSTED_SHELL, set manually since there's only one supported value - self.config['TRUSTED_SHELL'] = 'ssh' - - # CHECK_POINT_SEGMENTS - self.config['CHECKPOINT_SEGMENTS'] = \ - self.Select('show checkpoint_segments')[0] - - # ENCODING - self.config['CLIENT_ENCODING'] = \ - self.Select('show client_encoding')[0] - - # LOCALE - self.config['LOCALE'] = \ - self.Select('show lc_ctype')[0] - - # MAX_CONNECTIONS - self.config['MAX_CONNECTIONS'] = \ - self.Select('show max_connections')[0] - - - logger.info('Configuration acquired') - - - def copy_rewrite_hba(self, frm, to, upgrdfmt): - ''' - Between 3.2 and 3.3 we have a change in pg_hba.conf. The differences - are: - - "ident sameuser" becomes "ident" - - "ident " becomes "ident map=" - - "ldap "ldap url" becomes "ldap ldapserver=..." - - "pam servicename" becomes "pam pamservice="servicename"" - ''' - f = open(frm, 'r') - t = open(to, 'w') - logger.debug('writing %s to %s for upgrade = %s' - % (frm, to, str(upgrdfmt))) - for line in f: - # translate to new pg_hba.conf format - if upgrdfmt: - - # translate from 3.2 format => 3.3 format: - if self.oldversion < '3.3.0.0' and self.newversion >= '3.3.0.0': - # ident - line = re.compile('ident sameuser$').sub('ident', line) - line = re.compile('ident (?!map=)(.*)$').sub('ident map=\\1', line) - # pam - line = re.compile('pam (?!pamservice=)(.*)$').sub('pam pamservice="\\1"', line) - # ldap is a little more complex - # our objective is to parse the following: - # ldap[s]://[:]/[;prefix[;suffix]] - logger.debug("upgrading ldap entries") - r1 = re.compile('ldap "?(ldap[s]?)://([^:/]+)(:\d+)?/([^"]*)"?') - match = r1.search(line) - if match: - s = "ldapserver=" + match.group(2).strip() - if match.group(3): - s += " ldapport=" + str(match.group(3)[1:]).strip() - if match.group(4): - fixes = match.group(4).strip().split(';') - suffix = "" - prefix = "" - basedn = "" - if len(fixes) > 3: - raise UpgradeError('cannot rebuild ldap auth ' + - 'entry: ' + line) - if len(fixes) == 3: - suffix = ' ldapsuffix="' + fixes[2] + '"' - if len(fixes) >= 2: - prefix = ' ldapprefix="' + fixes[1] + '"' - if len(fixes) >= 1: - basedn = "" # not used - s += prefix + suffix - if match.group(1) == "ldaps": - s += " ldaptls=1" - line = re.compile('ldap (.*)$').sub('ldap ' + s, line) - - else: - # downgrade - if self.newversion < '3.3.0.0' and self.oldversion >= '3.3.0.0': - line = re.compile('ident$').sub('ident sameuser', line) - line = re.compile('ident map=(.*)$').sub('ident \\1', line) - - # logger.debug('writing to %s line %s' % (to, line)) - t.write(line) - f.close() - t.close() - - def move_rewrite_hba(self, frm, to, upgrade): - self.copy_rewrite_hba(frm, to, upgrade) - os.unlink(frm) - - #------------------------------------------------------------ - def SetupLockdown(self): - ''' - Change pg_hba.conf to disallow access to everyone except the - upgrade user. - ''' - - os_type = os.uname()[0].lower() - - if self.cmd == 'MASTER': - logger.info('Locking database') - self.CheckUp() - [env, utility] = self.dbup; - if utility: - raise UpgradeError("Cannot lock database in utility mode") - - logger.info('... Creating upgrade user') - self.Update('DROP USER IF EXISTS ' + MIGRATIONUSER) - self.Update('CREATE USER %s with superuser' % MIGRATIONUSER) - - logger.info('... Creating pg_hba.conf ') - - # To write out a new pg_hba.conf we need to know all the ip - # addresses that the master identifies with. - # - # If we re-enable segment locking then this will need to be - # passed to the segments as well. - if os_type == 'sunos': - cmd = 'ifconfig -a inet' - else: - cmd = 'ifconfig' - ifconf = self.RunCmd(cmd) - ipv4_re = re.compile('inet (?:addr:)?(\d+\.\d+\.\d+\.\d+)') - ipv6_re = re.compile('inet6 (?:addr: )?([a-zA-Z0-9:\.]+[a-zA-Z0-9:])') - ip = [] - for line in ifconf.split('\n'): - m = ipv4_re.search(line) - if m: - ip.append(m.group(1)) - m = ipv6_re.search(line) - if m: - ip.append(m.group(1)) - - # Currently we only lock the MASTER node - # - # locking down the segments requires having gpstart - # pass the PGUSER to the segments otherwise the segments - # don't have the permissions to startup. - # - # This leaves a "hole" in the lockdown that still allows - # utility connections to the segments. - hba = os.path.join(self.masterdir, 'pg_hba.conf') - ident = os.path.join(self.masterdir, 'pg_ident.conf') - - if env == self.oldenv or not os.path.exists(hba + LOCKEXT): - # pre-upgrade database makes a backup of files and then - # writes a new one - - # Solaris doesn't support ident authentication - if os_type != 'sunos': - self.RunCmd('mv -f %s %s' % (ident, ident+LOCKEXT)) - file = open(ident, 'w') - file.write('%s %s %s\n' % - (MIGRATIONUSER, self.user, MIGRATIONUSER)) - file.write('%s %s %s\n' % - (self.user, self.user, self.user)) - file.close() - - self.RunCmd('mv -f %s %s' % (hba, hba+LOCKEXT)) - file = open(hba, 'w') - if os_type == 'sunos': - file.write('local all %s trust\n' % MIGRATIONUSER) - file.write('local all %s trust\n' % self.user) - else: - file.write('local all %s ident map=%s\n' - % (MIGRATIONUSER, MIGRATIONUSER)) - file.write('local all %s ident map=%s\n' - % (self.user, self.user)) - - for addr in ip: - cidr_suffix = '/128' if ':' in addr else '/32' # MPP-15889 - file.write('host all %s %s%s trust\n' - % (MIGRATIONUSER, addr, cidr_suffix)) - file.write('host all %s %s%s trust\n' - % (self.user, addr, cidr_suffix)) - - file.close() - - if env == self.newenv: - # upgrade database just copies all locking information - # from the pre-upgrade database which should already be - # locked - dir = self.newmaster - - self.copy_rewrite_hba(hba, os.path.join(dir, 'pg_hba.conf'), - self.upgrade) - self.RunCmd('cp -f %s %s' % (ident, dir)) - - # Grab the original versions to (if any) - if os.path.exists(hba + LOCKEXT): - self.copy_rewrite_hba(hba+LOCKEXT, - os.path.join(dir, - 'pg_hba.conf'+LOCKEXT), - self.upgrade) - if os.path.exists(ident + LOCKEXT): - self.RunCmd('cp -f %s %s' % (ident+LOCKEXT, dir)) - - # With the new lockfiles in place force the server to reload - logger.info('... Syncing') - logger.debug('calling gpstop -u with env: %s' % str(env)) - self.RunCmd('gpstop -u -a -l %s' % self.logdir, env=env) - logger.info('Database Locked') - - # Check if there are any other sessions connected. Since we - # have modified the pg_hba.conf and run gpstop -u no new sessions - # will be allowed, but any that were already connected still - # need to be booted. - active = self.Select("SELECT count(*) FROM pg_stat_activity") - if active[0] > 1: - [env, utility] = self.dbup - logger.info('Active sessions detected, restarting server') - self.Shutdown() - self.Startup(env, utility) - - - def CheckClusterReady(self, databases, port): - ''' - Pre-flight checks - ''' - for db in databases: - xact = self.Select("SELECT count(*)::numeric FROM " + - "pg_prepared_xacts", - db=db['dbname'], port=port); - if xact[0] > 0: - raise UpgradeError("Database contains prepared transactions"); - - #------------------------------------------------------------ - def ReleaseLockdown(self, env): - ''' - Revert pg_hba.conf to pre-lockdown state - ''' - - # Two main cases: - # 1) Release lockdown on old environment - # 2) Release lockdown on newly installed environment - # Both cases have the database installed under MASTER_DATA_DIRECTORY - # so we actually treat them exactly the same - - if self.cmd == 'MASTER': - - # nothing to do if we haven't setup an array object yet. - if not self.array: - return - - logger.info('Unlocking database') - - # See comment about disabling lockdown on segments in SetupLockdown() - # self.CallSlaves('UNLOCK') - - datadirs = [self.masterdir] - else: - datadirs = self.datadirs - - for dir in datadirs: - hba = os.path.join(dir, 'pg_hba.conf') - ident = os.path.join(dir, 'pg_ident.conf') - if os.path.exists(hba + LOCKEXT): - forward = False - if self.upgrade and env == self.newenv: - forward = True - elif not self.upgrade and env == self.oldenv: - forward = True - self.move_rewrite_hba(hba+LOCKEXT, hba, forward) - - if os.path.exists(ident + LOCKEXT): - self.RunCmd('mv -f %s %s' % (ident+LOCKEXT, ident)) - - - # if we don't think the database is up, double check! - if not self.dbup: - env = None - try: - env = self.CheckUp() - except UpgradeError: - pass - if env: - self.dbup = [env, False] - - # Re-source the conf files if the database is up. - if self.dbup: - self.RunCmd('gpstop -u -l %s' % self.logdir, env=env) - - # MPP-10107 - The server is still running with PGUSER set in its - # environment, which is wrong since that user doesn't exist. - # Ideally that shouldn't matter, but unfortunately it does. We - # deal with this by restarting the server. - # - # Note: the step above is still required because otherwise we - # can't even run this gpstop due to pg_hba.conf lock. - # - # A better solution would be to have gpstart make a constrained - # environment when it starts and stops the server, but that is - # more than a one-line change, and this solves this case with - # minimal impact to anything else. - self.RunCmd('gpstop -ral %s' % self.logdir, env=env) - - # We would love to actually delete the user, but it's not actually - # safe to do so here because we unlock the source database before we - # are done with the upgrade. - return - - - #------------------------------------------------------------ - def ResetXLogs(self): - ''' - Resets the xlog prior to recreating the schema - ''' - if self.cmd == 'MASTER': - logger.info('Syncing XLogs') - datadirs = [self.masterdir] - self.CallSlaves('RESETXLOG') - else: - datadirs = self.datadirs - - # We should always have at least one data directory... - if len(self.datadirs) < 1: - return - - # Check the mapping in all data directories - for olddir in datadirs: - (d, seg) = os.path.split(olddir) - newdir = os.path.join(d, WORKDIR, UPGRADEDIR, seg) - - # Links are in place, but data will not be visible until - # we sync the pg_control info - oldsync = self.RunCmd('pg_resetxlog -n ' + olddir, - env=self.oldenv) - newsync = self.RunCmd('pg_resetxlog -n ' + newdir, - env=self.newenv) - - logger.debug('*****************************') - logger.debug(' - old xlog -') - logger.debug(oldsync) - logger.debug('*****************************') - logger.debug(' - new xlog -') - logger.debug(newsync) - logger.debug('*****************************') - - - # Nasty bit of code to turn the control data into a - # nice dict object - sync = {} - for line in oldsync.split('\n'): - if len(line) == 0: continue - (field, value) = line.split(':') - sync[field] = value.lstrip() - - # Set the next transaction id in new control - x = sync["Latest checkpoint's NextXID"].find('/') - if x < 0: - nextxid = sync["Latest checkpoint's NextXID"] - else: - nextxid = sync["Latest checkpoint's NextXID"][x+1:] - - self.RunCmd('pg_resetxlog -f -x %s %s' % (nextxid, newdir), - env=self.newenv) - - # Set the WAL archives - self.RunCmd('pg_resetxlog -l %s,%s,%s %s' % - (sync["Latest checkpoint's TimeLineID"], - sync["Current log file ID"], - sync["Next log file segment"], - newdir), env=self.newenv) - - # Set next oid - self.RunCmd('pg_resetxlog -o %s %s' % - (sync["Latest checkpoint's NextOID"], - newdir), env=self.newenv) - - # Set next multitransaction ID/offset - self.RunCmd('pg_resetxlog -m %s %s' % - (sync["Latest checkpoint's NextMultiXactId"], - newdir), env=self.newenv) - self.RunCmd('pg_resetxlog -O %s %s' % - (sync["Latest checkpoint's NextMultiOffset"], - newdir), env=self.newenv) - - # Replace the transaction logs with the old ones - # since the old logs actually correspond to the data - for f in ['pg_clog', - 'pg_distributedlog', - 'pg_subtrans', - 'pg_multixact']: - oldlogs = os.path.join(olddir, f) - newlogs = os.path.join(newdir, f) - - # the only one that wouldn't exist is pg_distributedlog - # and only if we were upgrading/downgrading to/from a - # version < 3.1.1.5 - if (os.path.exists(newlogs)): - self.RunCmd('rm -rf %s' % newlogs) - if (os.path.exists(oldlogs)): - self.RunCmd('cp -rf %s %s' % (oldlogs, newlogs)) - - #------------------------------------------------------------ - def DeepLink(self, style): - ''' - Makes a deep hard link copy of one directory into another - ''' - - if style == 'CLEANUP': - return - - if style not in ['BACKUP', 'INSTALL', 'REVERT', 'SKELETON']: - raise UpgradeError('FATAL', ' DeepLink(%s)!?' % style) - - if self.cmd == 'MASTER': - fsdirs = self.masterfilespace - if style == 'BACKUP': - logger.info('Linking backup') - logger.info('... This may take a while') - includeMirrors = True - elif style == 'INSTALL': - logger.info('Linking upgrade') - logger.info('... This may take a while') - includeMirrors = True - elif style == 'SKELETON': - logger.info('Linking skeleton') - logger.info('... This may take a while') - includeMirrors = False - else: - logger.info('Reverting to backup') - logger.info('... This may take a while') - includeMirrors = True - - - # Have all hosts perform the link first - self.CallSlaves('DEEPLINK', style, includeMirrors=includeMirrors) - - else: - fsdirs = self.filespaces - - for dir in fsdirs: - (location, content) = os.path.split(dir) - - # For Backup fsdirs is the source, for INSTALL it's the target - if style == 'BACKUP': - source = os.path.join(location, content) - destination = os.path.join(location, WORKDIR, BACKUPDIR, content) - if style == 'SKELETON': - source = os.path.join(location, content) - destination = os.path.join(location, WORKDIR, UPGRADEDIR, content) - if style == 'INSTALL': - source = os.path.join(location, WORKDIR, UPGRADEDIR, content) - destination = os.path.join(location, content) - if style == 'REVERT': - source = os.path.join(location, WORKDIR, BACKUPDIR, content) - destination = os.path.join(location, content) - - if not os.path.exists(source): - continue - - # Link the log file - if os.path.isfile(source + '.log'): - oldfile = source + '.log' - newfile = destination + '.log' - if os.path.exists(newfile): - if style == 'BACKUP': - logger.warn('File exists ' + newfile) - os.remove(newfile) - # logger.debug('Linking file %s => %s' % (oldfile, newfile)) - os.link(oldfile, newfile) - - # Remove any files from the destination location - # Not including the target directory itself - # Also not including the workdir and contents - #if os.path.exists(destination): - # for file in os.listdir(destination): - # If the file/directory also exists in the source - # then delete it in anticipation of receiving the - # replacement. - # if os.path.exists(os.path.join(source, file)): - # self.RunCmd('rm -rf ' + os.path.join(destination, file)) - - # if the source contains a gpperfmon/data directory we need to - # check that it hasn't grown to ridiculous sizes. If it has - # that implies that the perfmon is in a "runaway" state where - # the server is creating files and nothing is consuming them. - # in this case we should just delete the directory. - # - # Note: This is deleted in the SOURCE, it will NOT be restored - # if the upgrade fails or reverts. This is deemed acceptable - # because with this many files the system was almost certainly - # misconfigured to begin with. Removing the directory will - # prevent further acrual of files. - gpperfdir = os.path.join(source, 'gpperfmon', 'data') - if os.path.exists(gpperfdir): - nlink = os.stat(gpperfdir).st_nlink - - # We define "too large" as >10000 files - if nlink > 10000: - logger.info('Performance Monitor directory contains ' - '%s files - removing' % nlink) - self.RunCmd('rm -rf ' + gpperfdir) - - # now walk the source directory and link every file back into - # the target directory. - for root, dirs, files in os.walk(source, topdown = True): - if WORKDIR in dirs: - dirs.remove(WORKDIR) # don't visit the work directory - - newroot = root.replace(source, destination, 1) - if not os.path.exists(newroot): - os.mkdir(newroot) - self.RunCmd('chmod 700 ' + newroot) - - for name in files: - oldfile = os.path.join(root, name) - newfile = os.path.join(newroot, name) - if os.path.exists(newfile): - if style == 'BACKUP': - logger.warn('File exists ' + newfile) - os.remove(newfile) - # logger.debug('Linking file %s => %s ' % (oldfile, newfile)) - os.link(oldfile, newfile) - - #------------------------------------------------------------ - def FixConfig(self): - ''' - After moving the upgraded db from DATA_DIR/upgrade to just DATA_DIR - we need to go and fix the gp_configuration table to point to the - correct location. - ''' - - # We can do this in utility mode since the master host is the only - # host that contains gp_configuration information - logger.info('Adjusting configuration') - - if self.cmd == 'MASTER': - self.Startup(self.newenv, utility=True) - self.Update(""" - UPDATE pg_filespace_entry SET fselocation = - regexp_replace(fselocation, E'/gpmigrator/upgrade/','/') - """, forceutility=True, upgradeMode=True, modSysTbl=True) - self.Update("VACUUM FREEZE pg_filespace_entry", forceutility=True, upgradeMode=True, modSysTbl=True) - - self.Shutdown() - self.Startup(self.newenv, upgrade=True) - - self.CallSlaves('FIXCONFIG') - datadirs = [self.masterdir] - else: - datadirs = self.datadirs - - # Update the persistent table for each segment - for dir in datadirs: - # Find the last line in the conf file that matches our expression - conf = os.path.join(dir, 'postgresql.conf') - def getconf(x): - conf_re = re.compile('^\s*%s\s*=\s*(\d+)' % x) - try: - conf_str = self.RunCmd('grep %s %s' % (x, conf)) - except CmdError: - conf_str = "" # grep returns errorcode on no match - value = None - for line in conf_str.split('\n'): - match = conf_re.search(line) - if match: - value = int(match.group(1)) - return value - - # Find the port for this segment: - port = getconf('port') - if port == None: - raise UpgradeError('Could not determine port from %s/postgresql.conf' % dir) - - self.Update(""" - UPDATE gp_persistent_filespace_node SET location_1 = - regexp_replace(location_1, E'/gpmigrator/upgrade/','/')||' ' - where persistent_state = 2 - """, port=port, forceutility=True, upgradeMode=True, modSysTbl=True) - - if self.cmd == 'MASTER': - self.Shutdown() - self.newenv['MASTER_DATA_DIRECTORY'] = \ - self.oldenv['MASTER_DATA_DIRECTORY'] - - - #------------------------------------------------------------ - def TouchupConfig(self): - ''' - Touch up the config for a transform method upgrade/downgrade - ''' - logger.info('Adjusting configuration') - - if self.cmd == 'MASTER': - # Before we can start the master we need to tag it with a gp_dbid file. - # We will tag all the other segment directories after re-mirroring. - GpCreateDBIdFile.local('create master gp_dbid file', - directory=self.newmaster, dbid=1) - - # Use upgrade mode to tell the backend code to do some trick, - # since at this point the catalog has not yet updated so there - # are some discrepancy between code and catalog. The code handles - # that case by looking at gp_upgrade_mode flag. - self.Startup(self.newenv, utility=True, upgrade=True) - - # Move datadirectory location to gpmigrator/upgrade subdirectories - self.Update(""" - UPDATE pg_filespace_entry SET fselocation = - regexp_replace(fselocation, E'(/[^/]*)$', - E'/gpmigrator/upgrade\\\\1') - """, forceutility=True, upgradeMode=True, modSysTbl=True) - self.Update("VACUUM FREEZE pg_filespace_entry", forceutility=True, upgradeMode=True) - - # Update the gp_segment configuration table to correctly fill in - # the hostname and replication ports. - for db in self.array.getDbList(): - dbid = db.getSegmentDbId() - hostname = db.getSegmentHostName() or db.getSegmentAddress() - rport = db.getSegmentReplicationPort() or "null" - - self.Update(""" - UPDATE gp_segment_configuration - SET hostname='%s', replication_port=%s - WHERE dbid=%s - """ % (hostname, rport, dbid), forceutility=True, upgradeMode=True, modSysTbl=True) - self.Update("VACUUM FREEZE gp_segment_configuration", forceutility=True, upgradeMode=True) - - self.Shutdown() - self.Startup(self.newenv, upgrade=True) - - self.CallSlaves('TOUCHUPCONFIG') - datadirs = [ self.masterdir ] - - else: - datadirs = self.datadirs - - # Update the persistent table for each segment - for dir in datadirs: - # Find the last line in the conf file that matches our expression - conf = os.path.join(dir, 'postgresql.conf') - def getconf(x): - conf_re = re.compile('^\s*%s\s*=\s*(\d+)' % x) - try: - conf_str = self.RunCmd('grep %s %s' % (x, conf)) - except CmdError: - conf_str = "" # grep returns errorcode on no match - value = None - for line in conf_str.split('\n'): - match = conf_re.search(line) - if match: - value = int(match.group(1)) - return value - - # Find the port for this segment: - port = getconf('port') - if port == None: - raise UpgradeError('Could not determine port from %s/postgresql.conf' % dir) - - self.Update(""" - UPDATE gp_persistent_filespace_node SET location_1 = - regexp_replace(location_1, E'(/[^/]*) $', - E'/gpmigrator/upgrade\\\\1') - where persistent_state = 2 - """, port=port, forceutility=True, upgradeMode=True, modSysTbl=True) - - if self.cmd == 'MASTER': - self.Shutdown() - - - #------------------------------------------------------------ - def ExtractCatFiles(self): - ''' - Get the set of data files which are necessary to bring up a skeleton - system: just those necessary for bringing up the system in utility - mode and querying the catalogs -- no user data. - - We write the set of files to the file system for later use. - ''' - - logger.info('Extracting list of catalog files') - - if self.cmd == 'MASTER': - datadirs = [ self.masterdir ] - self.CallSlaves('EXTRACTCATFILES') - else: - datadirs = self.datadirs - - for dir in datadirs: - # The directory for this segment - (d, seg) = os.path.split(dir) - logdir = os.path.join(d, WORKDIR) - - # Open the mapping file - mapfilename = os.path.join(logdir, '%s_catfiles' % seg) - mapfile = open(mapfilename, 'w') - - # Find the last line in the conf file that matches our expression - conf = os.path.join(dir, 'postgresql.conf') - def getconf(x): - conf_re = re.compile('^\s*%s\s*=\s*(\d+)' % x) - try: - conf_str = self.RunCmd('grep %s %s' % (x, conf)) - except CmdError: - conf_str = "" # grep returns errorcode on no match - value = None - for line in conf_str.split('\n'): - match = conf_re.search(line) - if match: - value = int(match.group(1)) - return value - - # Find the port for this segment: - port = getconf('port') - if port == None: - raise UpgradeError('Could not determine port from %s/postgresql.conf' % dir) - - # Get a list of databases: - # Ideally we would like switch to using self.dbs instead - # of re-issuing this query, but that requires establishing - # a method of passing the database dictionary to the slaves. - databases = self.Select( - ''' - SELECT - datname - FROM pg_database where datname != 'template0' - ORDER BY datname - ''', port=port) - - # Build up the list of catalog entries - files = {} - for db in databases: - l = self.Select( - ''' - SELECT d.oid as datoid, - ts.spcfsoid, - case when d.dattablespace = 1663 - then '%s/base' - else rtrim(fse.location_1)||'/'||(d.dattablespace::text) - end || '/'||d.oid as catloc, - c1.relname as rel1, c1.relfilenode as node1, - c2.relname as rel2, c2.relfilenode as node2, - c3.relname as rel3, c3.relfilenode as node3 - FROM pg_class c1 - left outer join pg_namespace n on (c1.relnamespace = n.oid) - left outer join pg_class c2 on (c1.reltoastrelid = c2.oid) - left outer join pg_class c3 on (c2.reltoastidxid = c3.oid), - pg_database d, - pg_tablespace ts, - pg_filespace fs - left outer join gp_persistent_filespace_node fse on (fs.oid = fse.filespace_oid) - WHERE (nspname = 'pg_catalog' or - nspname = 'pg_aoseg' or - nspname = 'information_schema') - and c1.relkind in ('r', 'i', 'o') - and not c1.relisshared - and d.dattablespace = ts.oid - and ts.spcfsoid = fs.oid - and d.datname = '%s'; - ''' % (dir, db), port=port, db=db) - - first = True - f = [] - nlist = ["1", "2", "3"] - for row in l: - if first: - first = False - file = os.path.join(row.get("catloc"), - 'PG_VERSION') - f.append(file) - - for n in nlist: - if row.get("rel" + n): - dfile = os.path.join(row.get("catloc"), - str(row['node' + n])) - if not os.path.exists(dfile): - raise UpgradeError( -'found path "%s" for table "%s" but file does not exist' % (dfile, row['rel'+n])) - f.append(dfile) - files[db] = f - - # make sure tblspc dir is empty - pgtblspc = os.path.join(dir, 'pg_tblspc') - if os.path.exists(pgtblspc): - if os.listdir(pgtblspc): - raise UpgradeError('cannot upgrade system using ' + - 'tablespaces') - - # meta data directories, like commit log - meta = ['global', 'pg_clog', 'pg_distributedlog', - 'pg_distributedxidmap', 'pg_multixact', 'pg_subtrans', - 'pg_twophase', 'pg_utilitymodedtmredo', 'pg_xlog', - 'pg_tblspc', - 'postgresql.conf', - 'pg_hba.conf', 'pg_ident.conf', - 'PG_VERSION'] - - for name, fs in files.iteritems(): - for f in fs: - # Prefix each name with the database it's in - mapfile.write('db:%s\n' % f) - - # add template0 - r = self.Select("""select oid from pg_database where - datname = 'template0'""", port=port) - if len(r) > 0: - mapfile.write('dir:%s\n' % os.path.join('base', str(r[0]))) - - for f in meta: - mapfile.write('meta:%s\n' % f) - - # files to just link, like SSL key files, backup - # postgresql.conf, other stuff admins leave lying around - nolink = ['pg_hba.conf', - 'pg_ident.conf', - 'pg_hba.conf'+LOCKEXT, - 'pg_ident.conf'+LOCKEXT, - 'postmaster.pid', - 'postmaster.opts'] - for f in os.listdir(dir): - if (os.path.isfile(os.path.join(dir, f)) and - f not in meta and - f not in nolink): - # logger.debug('adding link for file %s' % f) - mapfile.write('link:%s\n' % f) - - mapfile.close() - - - #------------------------------------------------------------ - def ExtractAosegs(self): - ''' - Build AO seg upgrade/downgrade files for this node - - Also builds the persistent file upgrade/downgrade files for this node. - ''' - - logger.info('Building append only segment upgrader') - - if self.cmd == 'MASTER': - self.CallSlaves('EXTRACTAOSEGS') - datadirs = [ self.masterdir ] - else: - datadirs = self.datadirs - - for dir in datadirs: - (d, _) = os.path.split(dir) - logdir = os.path.join(d, WORKDIR) - - # We want to do this once per directory location, if a previous - # datadir already handled this we can skip doing it again. - fname = os.path.join(logdir, 'aoseg_template0_rewrite.sql') - if os.path.exists(fname): - continue - - conf = os.path.join(dir, 'postgresql.conf') - def getconf(x): - conf_re = re.compile('^\s*%s\s*=\s*(\d+)' % x) - try: - conf_str = self.RunCmd('grep %s %s' % (x, conf)) - except CmdError: - conf_str = "" # grep returns errorcode on no match - value = None - for line in conf_str.split('\n'): - match = conf_re.search(line) - if match: - value = int(match.group(1)) - return value - - # Find the port for this segment: - port = getconf('port') - if port == None: - raise UpgradeError('Could not determine port ' + - 'from %s/postgresql.conf' % dir) - - # Get a list of databases: - # Ideally we would like switch to using self.dbs instead - # of re-issuing this query, but that requires establishing - # a method of passing the database dictionary to the slaves. - databases = self.Select('''select datname from pg_database - where datname != 'template0';''', - port=port) - - # do it for every database - for db in databases: - fname = os.path.join(logdir,'aoseg_%s_rewrite.sql' % db) - logger.debug('creating AO rewrite: %s' % fname) - f = open(fname, 'w') - - # The addition of eofuncompressed was 32 => 33 - # - # Because this version of the upgrader does not support upgrades - # directly from 32 the old query is removed. - # - # See the version of gpmigrator that shipped with 33 for - # details. - f.close() - fname = os.path.join(logdir, 'aoseg_template0_rewrite.sql') - f = open(fname, 'w') - f.write('vacuum freeze;\n') - f.close() - - - # upgrade to persistent tables: 33 => 40 - # - # Produce upgrade files for persistent tables: - # - template1 must be processed first and handles the updates for - # the shared tables - # - the master is always called with gp_persistent_build_db(false) - # - segments call gp_persistent_build_db(true) when there are - # mirrors - fname = os.path.join(logdir, 'persistent_master.sql') - f = open(fname, 'w') - f.write('select gp_persistent_build_db(false);\n') - f.close() - - fname = os.path.join(logdir, 'persistent_segment.sql') - f = open(fname, 'w') - if len(self.mirrors) > 0: - f.write('select gp_persistent_build_db(true);\n') - else: - f.write('select gp_persistent_build_db(false);\n') - f.close() - - #------------------------------------------------------------ - def BuildSkel(self): - ''' - Copy data files to build a skeleton database. - ''' - - logger.info('Building skeleton cluster') - - if self.cmd == 'MASTER': - - # It would be nice to avoid having to make a full deep link, - # but the gp_persistent creation requires reading in a bunch - # of user tables to handle part of the ao conversion. - #self.DeepLink('SKELETON') - # Once that is done make a REAL copy (not hardlink) of the - # actual catalog tables - self.CallSlaves('BUILDSKEL', self.highestoid, self.gpperfmon) - datadirs = [ self.masterdir ] - else: - datadirs = self.datadirs - self.highestoid = int(self.option) - self.gpperfmon = str(self.option2) - if self.gpperfmon == 'None': - self.gpperfmon = None - - - def do_copy(src, dst, f): - ''' - Copy a directory tree - ''' - #logger.debug("copying %s to %s" % (os.path.join(src, f), - # os.path.join(dst, f))) - if os.path.isdir(os.path.join(src, f)): - if os.path.exists(os.path.join(dst,f)): - logger.debug('rmtree(%s)' % os.path.join(dst,f)) - shutil.rmtree(os.path.join(dst,f)) - logger.debug('cptree(%s)' % os.path.join(dst,f)) - shutil.copytree(os.path.join(src, f), os.path.join(dst, f)) - else: - if os.path.exists(os.path.join(dst,f)): - logger.debug('rm(%s)' % os.path.join(dst,f)) - os.remove(os.path.join(dst,f)) - logger.debug('cp(%s)' % os.path.join(dst,f)) - shutil.copy2(os.path.join(src, f), os.path.join(dst, f)) - - # for this is a file, copy all segments - if os.path.isfile(os.path.join(src, f)): - for i in range(1, 66000): - i = '.' + str(i) - fseg = f + i - if os.path.exists(os.path.join(src, fseg)): - if os.path.exists(os.path.join(dst,fseg)): - logger.debug('rm(%s)' % os.path.join(dst,fseg)) - os.remove(os.path.join(dst,fseg)) - logger.debug('cp(%s)' % os.path.join(dst,fseg)) - shutil.copy2(os.path.join(src, fseg), - os.path.join(dst, fseg)) - else: - break - - for src in datadirs: - (d, seg) = os.path.split(src) - dst = os.path.join(d, WORKDIR, UPGRADEDIR, seg) - logdir = os.path.join(d, WORKDIR) - - # Open the mapping file - mapfilename = os.path.join(logdir, '%s_catfiles' % seg) - mapfile = open(mapfilename, 'r') - - # build list for directories to create - dirs = [] - - for line in mapfile: - line = line[:-1] - (maptype, f) = line.split(':') - # f is in the form of filespace/seg/tablespace/dbid/filenode if maptype is db - # destination dir is filespace/WORKDIR/UPGRADEDIR/seg/tablespace/dbid - if (maptype=='db'): - (filespace, segid, tablespace, dbid, filenode) = f.rsplit("/", 4) - path = os.path.join(filespace, WORKDIR, UPGRADEDIR, segid, tablespace, dbid) - dirs.append(path) - - dirs = list(set(dirs)) # make unique - logger.debug('dirs = %s' % str(dirs)) - - # make sure dst exists - if not os.path.exists(dst): - os.mkdir(dst, 0700) - - dirs.append(os.path.join(dst, 'pg_log')) - dirs.append(os.path.join(dst, 'pg_changetracking')) - for dir in dirs: - logger.debug("making dir %s" % dir) - if not os.path.exists(dir): - os.makedirs(dir, 0700) - - logger.debug("copying files") - - # rewind - mapfile.seek(0) - for line in mapfile: - (maptype, f) = line.split(':') - f = f[:-1] - if maptype == 'link': - left = os.path.join(src, f) - right = os.path.join(dst, f) - logger.debug('linking %s to %s' % (left, right)) - - if os.path.exists(right): - os.remove(right) - os.link(left, right) - elif maptype == 'meta': - do_copy(src, dst, f) - elif maptype == 'dir': - do_copy(src, dst, f) - else: - (filespace, segid, tablespace, dbid, filenode) = f.rsplit("/", 4) - srcdir = os.path.join(filespace, segid, tablespace, dbid) - dstdir = os.path.join(filespace, WORKDIR, UPGRADEDIR, segid, tablespace, dbid) - do_copy(srcdir, dstdir, filenode) - - def overwrite(frm, to): - if not os.path.exists(frm): return - - f = open(frm, 'r') - t = open(to, 'w') - for line in f: - t.write(line) - f.close() - t.close() - - # remove the two pg_hba.conf hardlinks from DeepLink('SKELETON') - # otherwise they get in the way of the recreate logic below. - if os.path.exists(os.path.join(dst, 'pg_hba.conf'+LOCKEXT)): - os.remove(os.path.join(dst, 'pg_hba.conf'+LOCKEXT)) - if os.path.exists(os.path.join(dst, 'pg_hba.conf')): - os.remove(os.path.join(dst, 'pg_hba.conf')) - - # master needs locked pg_hba.conf - if self.cmd == 'MASTER': - self.copy_rewrite_hba(os.path.join(src, 'pg_hba.conf'+LOCKEXT), - os.path.join(dst, 'pg_hba.conf'+LOCKEXT), - self.upgrade) - - overwrite(os.path.join(src, 'pg_ident.conf'+LOCKEXT), - os.path.join(dst, 'pg_ident.conf'+LOCKEXT)) - - self.copy_rewrite_hba(os.path.join(src, 'pg_hba.conf'), - os.path.join(dst, 'pg_hba.conf'), - self.upgrade) - overwrite(os.path.join(src, 'pg_ident.conf'), - os.path.join(dst, 'pg_ident.conf')) - - - # Perform updates to the postgresql.conf file - newconf = os.path.join(dst, 'postgresql.conf') - oldconf = os.path.join(dst, 'postgresql.conf.bak') - if os.path.exists(oldconf): - os.remove(oldconf) - shutil.move(newconf, oldconf) - f = open(oldconf, 'r') - t = open(newconf, 'w') - foundPerfmon = False - conf_re = re.compile(r"^\s*(\w+)\s*=\s*([^#]*)(.*)$") - for line in f: - m = conf_re.match(line) - if m: - setting = m.group(1) - value = m.group(2) - rest = m.group(3) - - # Comment out defunct gucs - if setting in ('gp_fault_action'): - t.write('#%s = %s # defunct setting\n' % (setting, value)) - continue - - # Bump the value of max_resource_queues by one - if setting in ('max_resource_queues'): - value = str(int(value)+1) - t.write('%s = %s %s\n' % (setting, value, rest)) - continue - - # Make perfmon setting consistent for all segments - if setting in ('gp_enable_gpperfmon'): - if self.gpperfmon == None: - t.write('#%s = %s %s\n' % (setting, value, rest)) - else: - t.write('%s = %s %s\n' % (setting, self.gpperfmon, rest)) - foundPerfmon = True - continue - - t.write(line) - - # If gp_enable_gpperfmon was not mentioned anywhere in the file - # and the setting exists on the master then add the line. - if (foundPerfmon == False and self.gpperfmon != None): - t.write('gp_enable_gpperfmon = %s\n' % self.gpperfmon) - - f.close() - t.close() - - # set highest oid - # XXX: should check that this oid is higher! - self.RunCmd("pg_resetxlog -o %d %s" % \ - (self.highestoid, dst), env=self.oldenv) - - #------------------------------------------------------------ - def TransformDbs(self): - ''' - Transform the databases. - ''' - - logger.info('Performing catalog transformation') - - def transform(dbs, tfile): - - # MPP-10166 - because some of our transformations involve creates - # of views with unstable oids we must make sure that we process - # the databases in exactly the same order on every segment, so - # sorting the oids is essential. - oids = sorted(dbs.keys()) - for dboid in oids: - db = dbs[dboid] - - logger.info('... ' + db) - transform_db(db, dboid, tfile) - # Test failure during transformation - if self.faultinjection == 1: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - - def transform_db(dbname, dboid, tfile): - ''' - Transform the specified database for a single database. - ''' - logger.debug("transforming db = %s dboid = %s tfile = %s" % \ - (str(dbname), str(dboid), tfile)) - - # Execute the upgrade script - cmd = ' '.join(["PGOPTIONS='-c gp_maintenance_conn=true'", 'psql', '-f', tfile, dbname, self.user]) - logger.debug('running ' + cmd) - p = subprocess.Popen(cmd, shell = True, - close_fds = True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.newenv) - result = p.communicate() - - if p.returncode == 0: - logger.debug('successfully transformed ' + dbname) - logger.debug(result[0].strip()) - logger.debug(result[1].strip()) - else: - logger.debug(result[0]) - raise CmdError(str(cmd), result[0], result[1]) - - # run the upgrade file - runfile = "%s/share/postgresql/upgrade/upg2_catupgrade.sql" % self.newhome - transform(self.dbs, runfile) - - - #------------------------------------------------------------ - def SetCatVersion(self, frm, to): - ''' - Set the catalog version to something different. Cluster should be down. - ''' - - logger.info('Modifying catalog version of skeleton cluster') - releases = {"3.0": "200703112", - "3.1": "200712072", - "3.2": "200808253", - "3.3": "200905011", - "4.0": "201005134", - "4.1": "201101130", - "4.2": "201109210"} - - self.CheckDown() - - def release2catverno(rno): - if not rno in releases.keys(): - raise Exception("unknown version %s" % rno) - return releases[rno] - - def get_control_data(datadir): - ''' - Parse the output of pg_controldata run on data directory, returning - catalog version and state - ''' - cmd = ' '.join(['pg_controldata', datadir]) - p = subprocess.Popen(cmd, shell=True, close_fds=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self.newenv) - result = p.communicate() - - if p.returncode != 0: - raise CmdError(cmd, result[0], result[1]) - - out = result[0].strip() - ver = "" - state = "" - for line in out.split('\n'): - s = line.split(':') - if s[0] == 'Catalog version number': - ver = s[1].strip() - elif s[0] == 'Database cluster state': - state = s[1].strip() - - return [ver, state] - - def setcatversion(datadir, frm, to): - ''' - Set catalog version to 'to' from 'frm'. Make sure the database - is already set to frm. - - We use gpmodcontrol to update pg_control file since in this - version (4.3) we change the ControlFile structure. If only - catalog_version_no needs to change, we could use gpmodcatversion - as was previously. In case of gpmodcontrol (but not the case - for gpmodcatversion), the newer environment is responsible to - take care of downgrade too since older version does not have - knowledge of the newer structure. - ''' - (ver, state) = get_control_data(datadir) - - frmcatverno = release2catverno(frm) - if ver != frmcatverno: - raise Exception("Expected version %s but found %s" % (frmcatverno, ver)) - - - # It is an error if the current state is not "shut down", that - # means that there may be pending xlog records which will cause - # problems for the upgrade. - if state != "shut down": - raise UpgradeError("Cannot upgrade: Database did not shutdown cleanly") - - # gpmodcatversion is only in the 3.3 env so we must use that - - cmd = ['%s/bin/lib/gpmodcontrol' % self.newhome] - if not self.upgrade: - cmd.append('--downgrade') - cmd.append(datadir) - p = subprocess.Popen(cmd, - shell=False, - close_fds=True, - env=self.newenv, # always newenv - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - result = p.communicate() - if p.returncode != 0: - raise Exception("could not update catalog to %s" % to) - - - if self.cmd == 'MASTER': - datadirs = [ self.masterdir ] - self.CallSlaves('SETCATVERSION') - else: - datadirs = self.datadirs - - for olddir in datadirs: - (d, seg) = os.path.split(olddir) - newdir = os.path.join(d, WORKDIR, UPGRADEDIR, seg) - setcatversion(newdir, frm, to) - #------------------------------------------------------------ - def GetHighestOid(self): - ''' - Get the highest OID declared on the node, according to the control - file. We use this number to reset all the control files in the - new cluster we're building, so that all the new objects will be - added with the same OID. Since we're not dispatching the SQL to create - the objects, this is our only way to synchronise the OIDs. - - Database must be down. - ''' - - logger.info('Looking for highest OID') - self.CheckDown(self.revert) - self.highestoid = 16384 # first normal OID - oids = [] - if self.cmd == 'MASTER': - datadirs = [ self.masterdir ] - out = self.CallSlaves('GETHIGHESTOID') - for l in out.splitlines(): - logger.debug("GETHIGHESTOID> '%s'" % l) - a = l.split() - if a[0] == "highestoid": - oids.append(int(a[1])) - else: - datadirs = self.datadirs - - for dir in datadirs: - cmd = "pg_controldata %s" % dir - out = self.RunCmd(cmd, env=self.oldenv) - for l in out.splitlines(): - a = l.split(':') - if a[0] == "Latest checkpoint's NextOID": - oids.append(int(a[1])) - - logger.debug("oids: %s" % str(oids)) - for o in oids: - # XXX: what about overflow ? - if o > self.highestoid: - self.highestoid = o - - if self.cmd != 'MASTER': - print "highestoid " + str(self.highestoid) - - logger.debug("highest oid found: %d" % self.highestoid) - - - #------------------------------------------------------------ - def StampDBId(self): - ''' - Populates the gp_dbid file for every segment - ''' - logger.info("Adding gp_dbid file") - - # Already handled the master during TouchupConfig() - for seg in self.segments: - c = GpCreateDBIdFile('create gp_dbid file', - seg.getSegmentDataDirectory(), - seg.getSegmentDbId(), - ctxt=base.REMOTE, - remoteHost=seg.getSegmentAddress()) - self.pool.addCommand(c) - for seg in self.mirrors: - c = GpCreateDBIdFile('create gp_dbid file', - seg.getSegmentDataDirectory(), - seg.getSegmentDbId(), - ctxt=base.REMOTE, - remoteHost=seg.getSegmentAddress()) - self.pool.addCommand(c) - - try: - self.pool.join() - except: - self.pool.haltWork() - self.pool.joinWorkers() - - failure = False - for cmd in self.pool.getCompletedItems(): - if not cmd.was_successful(): - msg = cmd.get_results().stdout.strip() - log_literal(logger, logging.ERROR, msg) - failure = True - if failure: - raise UpgradeError("Fatal Segment Error") - - - #------------------------------------------------------------ - def Run(self): - """ - The main execution method - switches between submodes. - - The main frontend will execute the main PerformUpgrade() function, - during which it will execute CallSlaves with various backend operations. - For each one of the main backend operations we will spawn the backend - executables, this switches between each of these operations. - """ - - # Execution on the master - if self.cmd == 'MASTER': - self.PerformUpgrade() - - # The rest of the options are executed on remote hosts - elif self.cmd == 'SETSTATE': - self.SetState(self.option) - elif self.cmd == 'CHKDIR': - try: - self.CheckDirectories() - except UpgradeError, e: - if str(e) == 'Directories exist': - sys.exit(1) - raise e - elif self.cmd == 'CHKDOWN': - self.CheckDown(self.revert) - elif self.cmd == 'LOCKDOWN': - self.SetupLockdown() - elif self.cmd == 'UNLOCK': - self.ReleaseLockdown(None) - elif self.cmd == 'MKDIR': - self.CreateDirectories() - elif self.cmd == 'RMDIR': - if self.option == 'False': - self.RemoveDirectories(False) - else: - self.RemoveDirectories(True) - elif self.cmd == 'RESETXLOG': - self.ResetXLogs() - elif self.cmd == 'DEEPLINK': - self.DeepLink(self.option) - elif self.cmd == 'EXTRACTCATFILES': - self.ExtractCatFiles() - elif self.cmd == 'BUILDSKEL': - self.BuildSkel() - elif self.cmd == 'TOUCHUPCONFIG': - self.TouchupConfig() - elif self.cmd == 'FIXCONFIG': - self.FixConfig() - elif self.cmd == 'SETCATVERSION': - self.SetCatVersion(self.oldversion.getVersionRelease(), - self.newversion.getVersionRelease()) - elif self.cmd == 'TRANSFORMCAT': - self.TransformDbs() - elif self.cmd == 'EXTRACTAOSEGS': - self.ExtractAosegs() - elif self.cmd == 'GETHIGHESTOID': - self.GetHighestOid() - else: - raise Exception('Unknown cmd: ' + str(self.cmd)) - - if self.pool: - t = self.pool - self.pool = None - del t - - #------------------------------------------------------------ - def PerformUpgrade(self): - """ - This is the main method of the upgrader which controls the overallflow - of the upgrade. - """ - logger.info('gpmigrator version ' + __version__) - logger.info('Python version %d.%d.%d %s %d' % sys.version_info) - - # Check that we can do the upgrade between the versions - self.CheckVersions() - - # User requested a revert to previous version - if self.revert: - self.Revert() - sys.exit(0) - - # Read the statefile if we are trying to restart - current_state = {} - statefile = os.path.join(self.workdir, 'state') - if os.path.exists(statefile): - logger.info("Found previous upgrade") - self.state = open(statefile, 'r') - for line in self.state: - tup = line.split(':', 1) - if len(tup) == 1: - current_state[line.strip()] = True - else: - current_state[tup[0].strip()] = tup[1].strip() - self.state = open(statefile, 'a') - - # Be careful if the last upgrade died in an unstable state - if 'BACKUP_VALID' in current_state: - self.Revert() - - # Determine if it is safe to continue from where we left off, or - # if we need to start over from the beginning. - # 1) We only try to resume if we made it to NEWDB_VALID - # 2) We only try to resume if the Latest checkpoint's NextXid - # maches our expectations. - if 'NEWDB_VALID' not in current_state: - if os.path.exists(statefile): - logger.info("... State not resumable, restarting upgrade") - current_state = {} - else: - control_data = \ - GpControlData.local('Get Control Data', self.masterdir) - current_xid = control_data["Latest checkpoint's NextXID"] - - # We expect gpupgrade mirrors to bump the latest xid by exactly - # one as it starts the database and extracts the configuration - # information. - old_xid = current_state['NEWDB_VALID'] - expected_xid = current_state.get('REMIRROR_XID') - - logger.info("... Old XID = %s" % old_xid) - logger.info("... New XID = %s" % current_xid) - logger.info("... Expected XID = %s" % expected_xid) - - if current_xid not in [old_xid, expected_xid]: - logger.info("... Latest checkpoint mismatch, restarting upgrade") - current_state = {} - else: - logger.info("... Latest checkpoint matches, resuming upgrade") - - # -------- - # Stage 1: - # -------- - # We have 1 main restart state which is the state of NEWDB_VALID. This - # is the state we record right before we start remirroring. After the - # first phase of remirroring we stop the script to allow for the - # possibility of a zfs snapshot (which isn't really feasible before - # remirroring due to the amount of data written during the remirroring - # step). - if 'NEWDB_VALID' in current_state: - self.ReadInfo() - else: - - # Test failure before we bring up the database - if self.faultinjection == 1: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # Test failure due to a bad lockfile in /tmp - # For this injection we are assuming that the segments are running - # on the local machine and using port 50000, not necessarily true - # but should be good enough for testing. - if self.faultinjection == 2: # XXX - try: - env = self.CheckUp() - self.dbup = [env, False] - self.Shutdown() - except: - pass - logger.fatal("faultinjection=%d" % self.faultinjection) - open('/tmp/.s.PGSQL.50001.lock', 'w') - - # We always pull the current configuration from the old instance. - # (It may be nice to rewrite to allow caching state information) - shutdown = None - lockfile = os.path.join(self.oldenv['MASTER_DATA_DIRECTORY'], - 'pg_hba.conf'+LOCKEXT) - if os.path.exists(lockfile): - raise UpgradeError("migration lock file already exists at: %s" - % lockfile) - try: - env = self.CheckUp(True) - shutdown = False - self.dbup = [env, False] - if env == self.newenv: - logger.fatal('Cannot prepare with running new db') - except: - shutdown = True - self.Startup(self.oldenv) - - # Test failure after we bring up the database - # - we should stop it again - if self.faultinjection == 3: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # The way things are structured we will only issue selects against - # the database that's "up" so mark the oldenv as running. - self.ExtractInfo() - - # Create working directories on all hosts - # + Requires configuration info collected in ExtractInfo - # + Requires Execution on all hosts - if 'CREATE_DIRS' not in current_state: - self.CreateDirectories() - self.SetState('BOOTSTRAP: ' + self.RunCmd('date +%Y%m%d:%H:%M:%S')) - self.SetState('CREATE_DIRS') - - - # Run the common pre-upgrade check - if 'CHECKCAT' not in current_state: - self.PreUpgradeCheck() - self.SetState('CHECKCAT') - - # If we're in checkcat only mode, return here. Don't do the actual upgrade - if self.checkonly: - return - - # SetupLockdown: - # + Requires the database to be up - # + Creates the upgrade user - # + Shuts the database down - # + Adjusts hba_conf to only allow upgrade user access - # + Database is left down - self.SetupLockdown() - - # Test failure after we have locked the database - # - we should unlock it - if self.faultinjection == 4: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # Write configuration files - # + Required for Revert() to work correctly - if 'CONFIG_FILES' not in current_state: - self.GenerateConfigFiles() - self.SetState('CONFIG_FILES') - - if 'EXTRACTCATFILES' not in current_state: - self.ExtractCatFiles() - self.SetState('EXTRACTCATFILES') - - #if 'EXTRACTAOSEGS' not in current_state: - # self.ExtractAosegs() - # self.SetState('EXTRACTAOSEGS') - - # All information collected, shutdown old database - # - We can't release the lockdown yet because we rely on copying - # the lockdown files when building the upgraded version - self.Shutdown() - - # Test failure - # - unlock database - if self.faultinjection == 5: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - if 'GETHIGHESTOID' not in current_state: - self.GetHighestOid() - self.SetState('GETHIGHESTOID') - - # Note: Because the source array is already in a lockdown state - # the skeleton we build will also automatically be in a lockdown - # state, so there is no need to Explicitly lock it. - if 'BUILDSKEL' not in current_state: - self.BuildSkel() - self.SetState('BUILDSKEL') - - if 'SETCATVERSION' not in current_state: - sov = self.oldversion.getVersionRelease() - snv = self.newversion.getVersionRelease() - self.SetCatVersion(sov, snv) - self.SetState('SETCATVERSION') - - # TouchupConfig() performs final modifications to the catalog - # to get the configuration information in line with the upgrade. - # This involves adjusting the replication ports, hostnames, - # and other configuration information. - if 'TOUCHUPCONFIG' not in current_state: - self.TouchupConfig() - self.SetState('TOUCHUPCONFIG') - - if 'TRANSFORMCAT' not in current_state: - self.Startup(self.newenv, upgrade=True) - self.TransformDbs() - self.SetState('TRANSFORMCAT') - self.Shutdown() - - # Test failure - # - unlock database - if self.faultinjection == 6: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # Release the lockdown on the old database - self.ReleaseLockdown(self.oldenv) - - # Test failure - # - unlock database - if self.faultinjection == 7: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # We are now at the state where we have a fully upgraded version - # but we haven't yet installed it or brought the mirrors in sync. - # This is an important state because we may try to resume from - # this point. - control_data = \ - GpControlData.local('Get Control Data', self.masterdir) - current_xid = control_data["Latest checkpoint's NextXID"] - self.SetState('NEWDB_VALID:%s' % str(current_xid)) - - logger.info("--------------------------------------------------") - logger.info("Stage 1 complete") - logger.info("--------------------------------------------------") - - # -------- - # Stage 2: - # -------- - # We have no fully upgraded the existing old configuration into a - # new configuration. - # - # The current state of the world is as follows - # DATA_DIR/ = untouched original db - # DATA_DIR/upgrade = fully upgraded new db - # DATA_DIR/backup = empty except for the state and flag files. - # - # We haven't modified anything under the master directory yet. - # If anything has gone wrong up to this point it is pretty much - # safe to blow away everything under 'upgrade/' and 'backup/' - # (The only thing to be careful about is that we shutdown the - # new db before we remove the storage from under it.) - - # Test failure after releasing the lockdown - if self.faultinjection == 8: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # Add the gp_dbid file to every segment - # + Requires all mirror directories to exist - if 'GP_DBID' not in current_state: - self.StampDBId() - self.SetState('GP_DBID') - - # To install the new db under the old master path we go through - # pains to make it as robust as possible. This is done via a - # combination of redundancy and hard links. - # - # => Make a deep hardlink copy of DATA_DIR -> DATA_DIR/backup - if 'BACKUP_VALID' not in current_state: - self.DeepLink('BACKUP') - self.SetState('BACKUP_VALID') - - try: - # Test failure after installing the backup - # - Revert is necessary - if self.faultinjection == 9: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # Test failure during Revert - # - See Revert() - if self.faultinjection == -1: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # -------- - # Stage 3: - # -------- - # The current state of the world is as follows - # DATA_DIR/ = contents unreliable - # DATA_DIR/upgrade = fully upgraded new db - # DATA_DIR/backup = untouched original db - # => Make a deep hard link copy of DATA_DIR/upgrade -> DATA_DIR - # => Adjust new pg_configuration to point to DATA_DIR - self.DeepLink('INSTALL') - self.FixConfig() - - # Test failure after installing the newdb - # - Revert is necessary - if self.faultinjection == 10: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - # Everything upgraded and validated - # + Mark install valid - self.Startup(self.newenv) - - # Install the gp_toolkit schema - # + Requires a fully running 4.0 system including dispatch - # and thus must occur after remirroring. - if 'POSTUPGRADE' not in current_state: - self.PerformPostUpgrade() - self.SetState('POSTUPGRADE') - - # VACUUM FREEZE template1, since we haven't already - self.Update('VACUUM FREEZE;', db='template1') - - - # Finally, drop the upgrade user: - # - The migrator user shouldn't own anything, but if the - # "dumprestore" method was ever used on this database - # then it might. - logger.info("Removing upgrade user"); - try: - self.Update('DROP USER ' + MIGRATIONUSER, forceUseEnvUser=True) - except: - logger.warn('Objects owned by gpmigrator reassigned to dba') - admin = self.Select('''select rolname from pg_authid - where oid = 10''', forceUseEnvUser=True)[0]; - oids = sorted(self.dbs.keys()) - for dboid in oids: - db = self.dbs[dboid] - - if db == 'template0': - continue - self.Update('REASSIGN OWNED BY gpmigrator TO ' + admin, db, forceUseEnvUser=True) - self.Update('DROP USER ' + MIGRATIONUSER) - - # Test failure after installing the newdb - # - Revert is necessary - # - gpmigrator user no longer exists - if self.faultinjection == 11: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - if self.warnings: - logger.warn('***************************************') - logger.warn('Warnings encountered during upgrade.') - logger.warn('***************************************') - - # Release the lockdown - self.ReleaseLockdown(self.newenv) - - except Exception, e: - logger.fatal(str(e)) - self.Revert() - raise e - - # -------- - # Stage 4: - # -------- - # The current state of the world is as follows - # DATA_DIR/ = fully upgraded new db - # DATA_DIR/upgrade = fully upgraded new db - # DATA_DIR/backup = untouched original db - # => Remove redundant DATA_DIR/upgrade - # => Remove historical DATA_DIR/backup - self.RemoveDirectories(force=True) - - # ----------------- - # Upgrade Complete: - # ----------------- - # The current state of the world is as follows - # DATA_DIR/ = fully upgraded new db - logger.info("---------------------------------------------------") - logger.info("Upgrade Successful") - logger.info("---------------------------------------------------") - logger.info("Please consult release notes for post-upgrade ") - logger.info("instructions to complete environment configuration") - logger.info("---------------------------------------------------") - return - - - -#============================================================ -if __name__ == '__main__': - coverage = GpCoverage() - coverage.start() - - # Create a new GPUpgrade - should never throw exception - u = GPUpgrade() - - # parses and validates input - try: - u.Setup() - except Exception, e: - logger.fatal(str(e)) - sys.exit(1) - - # Execute the main upgrade routine, the Run() function itself is - # just a switch on the "cmd", which switches between the main - # PerformUpgrade() function and all the various subcommands that - # are upgraded by backend executions. - try: - u.Run() - - # Any error that can happen - except KeyboardInterrupt: - try: u.Cleanup() - except Exception, e2: - logger.fatal(str(e2)) - logger.fatal('***************************************') - logger.fatal('== Upgrade interrupted by user == <<<<< ') - logger.fatal('***************************************') - sys.exit(1) - - except ConnectionError, e: - try: u.Cleanup() - except Exception, e2: - logger.fatal(str(e2)) - logger.fatal('***************************************') - logger.fatal(str(e)) - logger.fatal('***************************************') - sys.exit(1) - - except UpgradeError, e: - try: u.Cleanup() - except Exception, e2: - logger.fatal(str(e2)) - logger.fatal('***************************************') - logger.fatal(str(e)) - logger.fatal('***************************************') - sys.exit(1) - - except CmdError, e: - try: u.Cleanup() - except Exception, e2: - logger.fatal(str(e2)) - logger.fatal('***************************************') - logger.fatal('Error executing command:') - logger.fatal(' ' + str(e.cmd.strip())) - logger.fatal(' ' + str(e.stderr.strip())) - logger.fatal('***************************************') - sys.exit(1) - - # Shouldn't get any of these, if they occur it is probably a bug in - # the upgrader. - except Exception, e: - try: u.Cleanup() - except Exception, e2: - logger.fatal(str(e2)) - logger.fatal('***************************************') - logger.fatal(str(e)) - logger.fatal('***************************************') - logger.fatal(traceback.format_exc()) - sys.exit(1) - finally: - coverage.stop() - coverage.generate_report() - - # This shouldn't need to be explicit, but if something goes wrong in - # thread shutdown it forces the issue. - sys.exit(0) diff --git a/gpMgmt/bin/gpmigrator_mirror b/gpMgmt/bin/gpmigrator_mirror deleted file mode 100755 index 6e406e4a21ebfb774c835116b6ae475d5fd713a4..0000000000000000000000000000000000000000 --- a/gpMgmt/bin/gpmigrator_mirror +++ /dev/null @@ -1,1318 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -''' -gpmigrator_mirror [options] old-gphome new-gphome - -Options: - -h, --help show this help message and exit - -v, --version show the program's version number and exit - -q quiet mode - -d DIRECTORY the master host data directory - -l DIRECTORY log file directory - --debug debugging information -''' - -#============================================================ -import sys, os - -# Python version 2.6.2 is expected, must be between 2.5-3.0 -if sys.version_info < (2, 5, 0) or sys.version_info >= (3, 0, 0): - sys.stderr.write("Error: %s is supported on Python versions 2.5 or greater\n" - "Please upgrade python installed on this machine." - % os.path.split(__file__)[-1]) - sys.exit(1) - -try: - from gppylib.operations.gpMigratorUtil import * -except ImportError, e: - sys.exit('Error: unable to import module: ' + str(e)) - -libdir = os.path.join(sys.path[0], 'lib/') - -logger = get_default_logger() -EXECNAME = os.path.split(__file__)[-1] -MIGRATIONUSER = 'gpmigrator' -LOCKEXT = '.gpmigrator_orig' -WORKDIR = 'gpmigrator' -BACKUPDIR = 'backup' -UPGRADEDIR = 'upgrade' -PARALLELISM = 16 - - - - -#============================================================ -__version__ = '$Revision: #1 $' - - - - -#============================================================ -def makeCommand(oldHome, newHome, - command, dataDirectories, option1): - # space separated list of directories - datadirs = base64.urlsafe_b64encode(pickle.dumps(dataDirectories)) - cmd = [ - EXECNAME, - oldHome, - newHome, - '--internal-command=' + str(command), - '--internal-dirs=' + urllib.quote(datadirs), - '--internal-option=' + urllib.quote(str(option1)), - '--debug', - '--quiet', - ] - return cmd - -#============================================================ -def logGenericErrorMessage(): - logger.fatal('Upgrade Failed..!!!') - logger.fatal('...Database might be in an unusable state') - logger.fatal('...Please check the logs, resolve the issue an re-run gpmigrator_mirror') - - -#============================================================ -class GPUpgrade(GPUpgradeBase): - ''' - Greenplum Database Upgrade Utility - ''' - - # INTERNAL command: - # MASTER - default, run on master, [not included in list] - # CHKDOWN - check the db is down - # SETCATVERSION - set catalog version - commands = ['CHKDOWN', 'SETCATVERSION'] - - #------------------------------------------------------------ - def __init__(self): - ''' - The most basic of initialization - ''' - super(GPUpgrade, self).__init__() - - # Environment and system info - self.datadirs = None # Set of all data and mirror directories (Segment only) - - - # Mirrored Upgrade State - self.mirror_upgrade_state = None - self.mirror_upgrade_state_file = None - - #------------------------------------------------------------ - def Setup(self): - ''' - Basic initialization, separate from __init__ for exception - handling purposes. - ''' - - # GPHOME, PYTHONPATH must be setup properly - # GPHOME must match the location of this file - # The first PYTHONPATH must be based on the proper gphome - gphome_bin = os.path.realpath(os.path.split(__file__)[0]) - gphome = os.path.split(gphome_bin)[0] - env_GPHOME = os.path.realpath(os.environ.get('GPHOME')) - if (env_GPHOME != gphome): - logger.fatal(" $GPHOME is set to %s which is not newhome" % env_GPHOME) - logger.fatal(' source the greenplum.sh from the newhome to setup env ') - raise UpgradeError('Initialization failed') - - pythonpath = os.path.join(gphome, "lib", "python") - env_PYTHONPATH = os.path.realpath(os.environ.get('PYTHONPATH').split(':')[0]) - if (env_PYTHONPATH != pythonpath): - logger.fatal(' $PYTHONPATH is incorrect ') - logger.fatal(' source the greenplum.sh from the newhome to setup env ') - raise UpgradeError('Initialization failed') - - # This is the same path used by gpinitsystem - self.path = '/usr/kerberos/bin:/usr/sfw/bin:/opt/sfw/bin' - self.path += ':/usr/local/bin:/bin:/usr/bin:/sbin:/usr/sbin:/usr/ucb' - self.path += ':/sw/bin' - - # Set defaults - self.user = os.environ.get('USER') or os.environ.get('LOGNAME') - self.host = self.RunCmd('hostname') - self.masterdir = os.environ.get('MASTER_DATA_DIRECTORY') - - self.ParseInput() - - # Setup worker pool - self.pool = base.WorkerPool(numWorkers=PARALLELISM); - - # Extract path from input master directory - if self.cmd == 'MASTER': - logger.info('Beginning upgrade') - logger.info('Checking configuration') - - if not self.masterdir or len(self.masterdir) == 0: - raise UpgradeError('MASTER_DATA_DIRECTORY is not defined') - self.masterdir = self.masterdir.rstrip('/') - - # The configuration file - conf = os.path.join(self.masterdir, 'postgresql.conf') - - # Simple function to look for settings in the conf file - def getconf(x): - conf_re = re.compile('^\s*%s\s*=\s*(\w+)' % x) - try: - conf_str = self.RunCmd('grep %s %s' % (x, conf)) - except CmdError: - conf_str = "" # grep returns errorcode on no match - value = None - for line in conf_str.split('\n'): - match = conf_re.search(line) - if match: - value = match.group(1) - return value - - # Find the port for this segment: - self.masterport = getconf('port') - if self.masterport == None: - raise UpgradeError('Could not determine master port from ' + conf) - self.masterport = int(self.masterport) - - self.sock_dir = getconf('unix_socket_directory') - if not self.sock_dir: - self.sock_dir = '/tmp/' - - # Verify that (max_connections == superuser_reserved_connections) - # max_conn = getconf('max_connections') - # reserved = getconf('superuser_reserved_connections') - - masterpath, masterdir = os.path.split(self.masterdir) - self.workdir = os.path.join(self.masterdir, WORKDIR) - self.oldenv = self.SetupEnv(self.oldhome, self.masterdir) - self.newenv = self.SetupEnv(self.newhome, self.masterdir) - - else: - self.oldenv = self.SetupEnv(self.oldhome, None) - self.newenv = self.SetupEnv(self.newhome, None) - - #------------------------------------------------------------ - def ParseInput(self): - ''' - Parses and validates input to the script - ''' - - try: - parser = optparse.OptionParser(usage=(cli_help(EXECNAME) or __doc__), add_help_option=False) - parser.add_option('-v', '--version', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('-h', '--help', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('-d', dest='directory', - help=optparse.SUPPRESS_HELP) - parser.add_option('-l', dest='logdir', - help=optparse.SUPPRESS_HELP) - parser.add_option('-c', '--check-only', dest='checkonly', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('-q', '--quiet', action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('--debug', action='store_true', - help=optparse.SUPPRESS_HELP) - - parser.add_option('--internal-command', dest='command', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-dirs', dest='datadirs', - help=optparse.SUPPRESS_HELP) - parser.add_option('--internal-option', dest='option', - help=optparse.SUPPRESS_HELP) - parser.add_option('--skip-checkcat', dest='skipcheckcat', - action='store_true', - help=optparse.SUPPRESS_HELP) - parser.add_option('--fault-injection', dest='faultinjection', - type='int', help=optparse.SUPPRESS_HELP) - (options, args) = parser.parse_args() - - if options.version: - print EXECNAME + ' ' + __version__ - sys.exit(0) - - if options.help: - usage(EXECNAME) - sys.exit(0) - - if len(args) != 2: - usage(EXECNAME) - msg = "incorrect number of arguments" - if len(args) > 0: - msg += ": %s" % str(args) - parser.error(msg) - except Exception, e: - usage(EXECNAME) - raise UpgradeError('Error parsing input: ' + str(e)) - - if options.directory: - self.masterdir = options.directory - if options.command: - self.cmd = options.command - self.oldhome = args[0].rstrip('/') - self.newhome = args[1].rstrip('/') - if options.datadirs: - self.datadirs = pickle.loads(base64.urlsafe_b64decode((urllib.unquote(options.datadirs)))) - if options.option: - self.option = urllib.unquote(options.option) - if options.checkonly: - self.checkonly = True - if options.debug: - self.debug = True - enable_verbose_logging() - if options.quiet: - quiet_stdout_logging() - if options.logdir: - self.logdir = options.logdir - else: - self.logdir = os.path.join(os.environ['HOME'], 'gpAdminLogs') - - try: - setup_tool_logging(EXECNAME, - unix.getLocalHostname(), - unix.getUserName(), - self.logdir) - except OSError, e: - logger.fatal('cannot log to %s: %s' % (self.logdir, str(e))) - exit(1) - - self.skipcheckcat = options.skipcheckcat - self.faultinjection = options.faultinjection - - if self.cmd == 'MASTER': - - today = date.today().strftime('%Y%m%d') - if not os.path.isdir(self.logdir): - os.makedirs(self.logdir, 0700) - logname = os.path.join(self.logdir, '%s_%s.log' % (EXECNAME, today)) - self.logfile = open(logname, 'a') - - if self.masterdir == None: - logger.fatal('MASTER_DATA_DIRECTORY parameter not set') - raise UpgradeError('Initialization failed') - if not os.path.exists(self.masterdir): - logger.fatal('MASTER_DATA_DIRECTORY: %s not found' - % self.masterdir) - raise UpgradeError('Initialization failed') - if not os.path.isabs(self.masterdir): - self.masterdir = os.path.abspath(self.masterdir) - if not os.path.exists(self.oldhome): - logger.fatal(' directory not found: ' + self.oldhome) - raise UpgradeError('Initialization failed') - if not os.path.exists(self.newhome): - logger.fatal(' directory not found: ' + self.newhome) - raise UpgradeError('Initialization failed') - if not os.path.isabs(self.oldhome): - self.oldhome = os.path.abspath(self.oldhome) - if not os.path.isabs(self.newhome): - self.newhome = os.path.abspath(self.newhome) - - - #------------------------------------------------------------ - def CallSlaves(self, cmd, option=''): - ''' - Calls every host to execute the given command with the given option - value - ''' - - logger.debug("Remote call: %s" % cmd) - - # Check for things that should never happen - if self.cmd != 'MASTER': - raise UpgradeError("Recursive communication error") - if not self.array: - raise UpgradeError("Failure initializing array") - if not self.hostcache: - raise UpgradeError("Failure initializing host cache") - if not self.pool: - raise UpgradeError("Failure initializing worker pool") - - # Construct the commands to pass to the worker pool - hosts = self.hostcache.get_hosts() - for host in hosts: - hostname = host.hostname - - # Skip any hosts in the cache that contain no segments for this - # configuration. - if len(host.dbs) == 0: - continue - - # Get the data directories for this host: - datadirs = [] - for seg in host.dbs: - datadirs.append(seg.getSegmentDataDirectory()) - - # Skip any hosts that have no applicable data directories - if len(datadirs) == 0: - continue - - cmdList = makeCommand(oldHome=self.oldhome, - newHome=self.newhome, - command=cmd, - dataDirectories=datadirs, - option1=option) - - c = GpUpgradeCmd("gpmigrator_mirror remote call", - cmdList, - ctxt=base.REMOTE, - remoteHost=hostname) - self.pool.addCommand(c) - - # Wait for the segments to finish - try: - self.pool.join() - self.pool.haltWork() - except: - self.pool.haltWork() - self.pool.joinWorkers() - - failure = False - results = [] - for cmd in self.pool.getCompletedItems(): - r = cmd.get_results() - - # Going through the gppylib Command interface all stderr from the - # remote calls gets redirected to stdout, which is unfortunate - # because we'd like to be able to differentiate between the two. - # - # We keep the stdout chatter to a minimum by passing --quiet to - # the remote calls which performs quiet stdout logging. - - # sys.stderr.write(r.stderr) - msg = r.stdout.strip() - results.append(msg) - - if not cmd.was_successful(): - log_literal(logger, logging.ERROR, msg) - failure=True - - if failure: - raise UpgradeError("Fatal Segment Error") - - # Warning this output contains everything written to stdout, - # which unfortunately includes some of the logging information - return "\n".join(results) - - - #------------------------------------------------------------ - def CheckVersions(self): - ''' - Validates that the upgrade from old->new is okay - ''' - - # Log the OS type info - os_type = os.uname()[0] - os_ver = os.uname()[2] - logger.info('Operating System: %s %s' % (os_type, os_ver)) - - # Log version checking - logger.info('Checking version compatibility') - - # If we got a version, but it isn't recognized by the GpVersion class - # then it is likely not a Greenplum database. - oldversion = self.getversion(self.oldhome, self.oldenv) - newversion = self.getversion(self.newhome, self.newenv) - try: - oldversion = GpVersion(oldversion) - except: - raise UpgradeError('Source not a Greenplum Database: ' + oldversion) - try: - newversion = GpVersion(newversion) - except: - raise UpgradeError('Target not a Greenplum Database: ' + newversion) - - logger.info('Source Version: (Greenplum Database) %s' % str(oldversion)) - logger.info('Target Version: (Greenplum Database) %s' % str(newversion)) - - if newversion == oldversion: - raise UpgradeError("Greenplum Database is already version '%s'" - % str(newversion)) - elif newversion.isVersionRelease(oldversion): - raise UpgradeError("Upgrade not needed to go from version '%s' to version '%s'" - % (str(oldversion), str(newversion))) - - is_supported_version(oldversion) - is_supported_version(newversion) - - # We don't support downgrade - if newversion < oldversion: - raise UpgradeError("Downgrade to Greenplum Database %s not supported from gpmigrator" - % str(newversion)) - - if not newversion.isVersionCurrentRelease(): - main = GpVersion('main') - raise UpgradeError( - "Upgrade from '%s' to '%s' not supported. Target version should be Greenplum Database %s" - % (str(oldversion), str(newversion), main.getVersionRelease())) - - # Compare old version catalog number with the catalog in - # MASTER_DATA_DIRECTORY - catalog_re = re.compile('Catalog version number: *(.*)') - oldcat = self.RunCmd('postgres --catalog-version', env=self.oldenv) - logger.info("Source %s" % oldcat) - newcat = self.RunCmd('postgres --catalog-version', env=self.newenv) - logger.info("Target %s" % newcat) - - control = self.RunCmd('pg_controldata ' + self.masterdir, env=self.oldenv) - for line in control.split('\n'): - m = catalog_re.match(line) - if m: - logger.info("Data %s " % line) - if (line != oldcat): - logger.debug('catalog mismatch: expected %s, found %s' - % (oldcat, line)) - msg = 'Catalog in %s does not match source binary' % self.masterdir - raise UpgradeError(msg) - break - - # For the moment everything goes: - # - Additional checks will be made once the old database is up - logger.info('Versions are compatible') - - - #------------------------------------------------------------ - def GenerateConfigFiles(self): - ''' - Creates 'gp_databases' 'gp_array_config' and 'gp_config' files - ''' - - logger.info('Generating config files') - - # Write a dump of the databases hash - db_file = os.path.join(self.workdir, 'gp_databases') - o = open(db_file, 'w') - for (oid, name) in self.dbs.iteritems(): - o.write('%s:%s\n' % (oid, name)) - o.close() - - # Write a dump of the gparray object - array_file = os.path.join(self.workdir, 'gp_array_config') - self.array.dumpToFile(array_file) - - logger.info('Configuration files generated') - - - #------------------------------------------------------------ - def ReadInfo(self): - ''' - When restarting an upgrade this method reads the configuration - information from the cached status files. - - It corresponds directly with ExtractInfo() which gathers the same - information from an active database. - - It is also the inverse of GenerateConfigFiles() which creates the files - that this function reads. - - In normal processing usually ExtractInfo() is called rather than - ReadInfo(). This function is used /instead/ when we are doing Revert() - processing, or when resuming an upgrade. E.g. cases when we do not - want to start the database to get the information we need because the - database may not be in a stable state. - ''' - - # self.array - configuration information - array_config = os.path.join(self.workdir, 'gp_array_config') - if not os.path.exists(array_config): - raise UpgradeError(" '%s' not found" % array_config) - self.array = GpArray.initFromFile(array_config) - - # self.dbs - database information - db_file = os.path.join(self.workdir, 'gp_databases') - if not os.path.exists(db_file): - raise UpgradeError(" '%s' not found" % db_file) - f = open(db_file, 'r') - for line in f: - (oid, name) = line.split(':', 1) - self.dbs[oid] = name.strip() - f.close() - - # self.hostcache - cache of hostnames - self.hostcache = GpHostCache(self.array, self.pool) - failed_pings = self.hostcache.ping_hosts(self.pool) - if len(failed_pings) > 0: - raise UpgradeError( - "Cannot upgrade while there are unreachable hosts") - - # self.mirrors - dbs = self.array.getSegDbList() - self.mirrors = filter(GpDB.isSegmentMirror, dbs) - - # Setup the master catalog dirs - self.mastercatdir = self.array.master.catdirs - - - - #------------------------------------------------------------ - def ExtractInfo(self): - ''' - Get all the information we need from the old instance - ''' - - # Can only extract with a running database - env = self.CheckUp() - - # -- Get the array information - logger.info('Extracting configuration information') - port = self.masterport - user = env['USER'] - url = dbconn.DbURL(port=port, dbname='template1', username=user) - array = GpArray.initFromCatalog(url, utility=True) - self.array = array - - # -- Determine if there are any invalid segments - logger.info('Checking that all segments are valid') - invalid = array.get_invalid_segdbs() - if len(invalid) > 0: - for db in invalid: - logger.fatal('INVALID SEGMENT: %s:/%s' % - (db.getSegmentHostName(), - db.getSegmentDataDirectory())) - raise UpgradeError('Cannot upgrade database with invalid segments') - - - # -- Get a list of available databases, note this includes "template0"! - logger.info('Creating list of databases') - databases = self.Select("SELECT oid, datname from pg_database") - for row in databases: - self.dbs[str(row['oid'])] = row['datname'] - - # -- Older releases didn't have hostname as part of configuration - # make sure we have an acurate list of address->host lookups. - logger.info('Validating hosts') - self.hostcache = GpHostCache(self.array, self.pool) - for host in self.hostcache.get_hosts(): - for seg in host.dbs: - seg.setSegmentHostName(host.hostname) - failed_pings = self.hostcache.ping_hosts(self.pool) - if len(failed_pings) > 0: - raise UpgradeError("Cannot upgrade while there are unreachable " - "hosts") - - # Setup the master catalog dirs - self.mastercatdir = self.array.master.catdirs - - master = array.master - standbymaster = array.standbyMaster - - # Oddly gparray doesn't have methods for fetching primary segments - # specifically - dbs = array.getSegDbList() - segments = filter(GpDB.isSegmentPrimary, dbs) - self.mirrors = filter(GpDB.isSegmentMirror, dbs) - - # Internal sanity checking - if len(segments) == 0: - raise UpgradeError('No segments found') - if standbymaster > 0: - raise UpgradeError('Cannot upgrade while standbymaster is running') - - # Check for unsupported index types. - found = False - logger.info("Validating indexes") - oids = sorted(self.dbs.keys()) - for dboid in oids: - db = self.dbs[dboid] - - if db == "template0": - continue - indexes = self.Select(''' - SELECT quote_ident(n.nspname) || '.' || - quote_ident(c.relname) as index, - m.amname as kind - FROM pg_class c - join pg_namespace n on (c.relnamespace = n.oid) - join pg_am m on (c.relam = m.oid) - WHERE c.relkind = 'i' and m.amname in ('gin', 'hash') - ''', db=db) - if len(indexes) > 0: - if found == False: - logger.fatal("Unable to upgrade the following indexes") - found = True - logger.fatal(" Database: %s" % db) - for row in indexes: - logger.fatal(" %s [%s]" % (row['index'], row['kind'])) - if found: - raise UpgradeError("Deprecated index types must be removed prior " - "to upgrade") - - # Check for SQL_ASCII database encoding - logger.info("Validating database encoding") - encoding = self.Select("SELECT datname FROM pg_database WHERE encoding=0") - if len(encoding) > 0: - logger.error("Deprecated database encodings found:") - for datname in encoding: - logger.error(" %s [SQL_ASCII]" % datname) - raise UpgradeError("Deprecated database encodings found - contact " - "Greenplum Support") - - logger.info('Configuration acquired') - - #------------------------------------------------------------ - def ExecuteSQLFileOnAllDBs(self, runfile): - ''' - Execute "runfile" in upgrade mode on all databases, including template0 - ''' - def transform_db(dbname, dboid, tfile): - ''' - Run the script on the specified database - ''' - logger.debug("Executing on db = %s dboid = %s tfile = %s" % \ - (str(dbname), str(dboid), tfile)) - - # Execute the upgrade script - cmd = ' '.join(["PGOPTIONS='-c gp_maintenance_conn=true'", 'psql', '-f', tfile, dbname, self.user]) - logger.debug('running ' + cmd) - p = subprocess.Popen(cmd, shell = True, - close_fds = True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.newenv) - result = p.communicate() - - if p.returncode == 0: - logger.debug('successfully transformed ' + dbname) - logger.debug(result[0].strip()) - logger.debug(result[1].strip()) - else: - logger.debug(result[0]) - raise CmdError(str(cmd), result[0], result[1]) - - - logger.info('Performing catalog transformation') - # MPP-10166 - because some of our transformations involve creates - # of views with unstable oids we must make sure that we process - # the databases in exactly the same order on every segment, so - # sorting the oids is essential. - oids = sorted(self.dbs.keys()) - for dboid in oids: - db = self.dbs[dboid] - - logger.info('... ' + db) - transform_db(db, dboid, runfile) - # Test failure during transformation - if self.faultinjection == 1: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - #------------------------------------------------------------ - def SetCatVersion(self, tonewver): - ''' - Upgrade/downgrade pg_control file to another version. Cluster should - be clean shutdown. - ''' - - logger.info('Modifying catalog version') - - if (tonewver==True): - curenv = self.newenv - curhome = self.newhome - else: - curenv = self.oldenv - curhome = self.oldhome - toversion = GpVersion(self.getversion(curhome, curenv)) - to = toversion.getVersionRelease() - - logger.info("To %s"%to) - self.CheckDown() - - def get_control_data(datadir): - ''' - Parse the output of pg_controldata run on data directory, returning - catalog version and state - ''' - cmd = ' '.join(['pg_controldata', datadir]) - p = subprocess.Popen(cmd, shell=True, close_fds=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=curenv) - result = p.communicate() - - if p.returncode != 0: - raise CmdError(cmd, result[0], result[1]) - - out = result[0].strip() - ver = "" - state = "" - for line in out.split('\n'): - s = line.split(':') - if s[0] == 'Catalog version number': - ver = s[1].strip() - elif s[0] == 'Database cluster state': - state = s[1].strip() - - return [ver, state] - - def setcatversion(datadir, to): - ''' - Set catalog version to 'to' - - We use gpmodcontrol to update pg_control file since in this - version (4.3) we change the ControlFile structure. If only - catalog_version_no needs to change, we could use gpmodcatversion - as was previously. In case of gpmodcontrol (but not the case - for gpmodcatversion), the newer environment is responsible to - take care of downgrade too since older version does not have - knowledge of the newer structure. - ''' - # It is an error if the current state is not "shut down", that - # means that there may be pending xlog records which will cause - # problems for the upgrade. - (ver, state) = get_control_data(datadir) - if state != "shut down": - raise UpgradeError("Cannot upgrade: Database did not shutdown cleanly") - - cmd = ['%s/bin/lib/gpmodcontrol' % self.newhome] - if not tonewver: - cmd.append('--downgrade') - cmd.append(datadir) - p = subprocess.Popen(cmd, - shell=False, - close_fds=True, - env=self.newenv, # always newenv - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - result = p.communicate() - if p.returncode != 0: - raise Exception("could not update catalog to %s" % to) - - - if self.cmd == 'MASTER': - datadirs = [ self.masterdir ] - self.CallSlaves('SETCATVERSION', tonewver) - if self.faultinjection == 2: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - else: - datadirs = self.datadirs - - for dir in datadirs: - setcatversion(dir, to) - if self.faultinjection == 3: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - - - - #------------------------------------------------------------ - def Run(self): - """ - The main execution method - switches between submodes. - - The main frontend will execute the main PerformUpgrade() function, - during which it will execute CallSlaves with various backend operations. - For each one of the main backend operations we will spawn the backend - executables, this switches between each of these operations. - """ - - # Execution on the master - if self.cmd == 'MASTER': - self.PerformMirroredUpgrade() - - # The rest of the options are executed on remote hosts - elif self.cmd == 'CHKDOWN': - self.CheckDown() - elif self.cmd == 'SETCATVERSION': - self.SetCatVersion((str(self.option)==str(True))) - else: - raise Exception('Unknown cmd: ' + str(self.cmd)) - - if self.pool: - t = self.pool - self.pool = None - del t - - # MIRROR UPGRADE CODE - #------------------------------------------------------------ - def ForceShutDown(self): - # The cleanest shutdown is to start the master in master only - # and then do gpstop -a. - # This simple thing couldn't shutdown segment if the master is dead. - try: - env = self.CheckUp() - self.dbup = [env, False] - self.Shutdown() - except: - pass - - - def CreateMirrorWorkDir(self): - # Create the workdir - # if the dir exists, clean it first - self.RunCmd('rm -rf ' + self.workdir) - os.mkdir(self.workdir) - return "ExtractInfoAndCopyMaster" - - #------------------------------------------------------------ - def ExtractInfoAndCopyMaster(self): - """ - Extract the database info and ceck for upgrade pre-requisite - Serialize the db info to a flat file for "resume" (if crashed) - Create a Master Copy - """ - self.ExtractandWriteInfo() - self.CreateMasterCopy() - return "TurnOffMirror" - - def ExtractandWriteInfo(self): - self.CheckVersions() - self.ForceShutDown() - self.Startup(self.oldenv) - - # Pre-requisite check - # 1. all segments must be up - #. 2. all mirror and segments must be in sync - # 3. all mirror/segment must be in their preferred role - # 4. it must have mirror - # 5. pass gpcheckcat - - # This query check 1, 2 and 3. - nonsynccnt = self.Select('''select count(*) - from pg_catalog.gp_segment_configuration - where status != 'u' or mode != 's' or role != preferred_role - ''' ) - if (nonsynccnt[0] > 0): - raise UpgradeError("Mirror or Segments are down or out of sync") - - # Now, extract the info before going on with more checks - self.ExtractInfo() - - # 4. it must have mirror - if (len(self.mirrors) ==0): - raise UpgradeError("This upgrade requires mirroring.") - - # 5. pass common pre-upgrade check - self.PreUpgradeCheck() - - # If we're in check only mode, then we exit here - if self.checkonly: - sys.exit(0) - - # Everything is ok. Write info to disk - self.GenerateConfigFiles() - - #------------------------------------------------------------ - def do_copy(self, src, dst, f): - ''' - Copy a directory tree - ''' - if os.path.isdir(os.path.join(src, f)): - if os.path.exists(os.path.join(dst,f)): - logger.debug('rmtree(%s)' % os.path.join(dst,f)) - shutil.rmtree(os.path.join(dst,f)) - logger.debug('cptree(%s)' % os.path.join(dst,f)) - shutil.copytree(os.path.join(src, f), os.path.join(dst, f)) - else: - if os.path.exists(os.path.join(dst,f)): - logger.debug('rm(%s)' % os.path.join(dst,f)) - os.remove(os.path.join(dst,f)) - logger.debug('cp(%s)' % os.path.join(dst,f)) - shutil.copy2(os.path.join(src, f), os.path.join(dst, f)) - - # for this is a file, copy all segments - if os.path.isfile(os.path.join(src, f)): - for i in range(1, 66000): - i = '.' + str(i) - fseg = f + i - if os.path.exists(os.path.join(src, fseg)): - if os.path.exists(os.path.join(dst,fseg)): - logger.debug('rm(%s)' % os.path.join(dst,fseg)) - os.remove(os.path.join(dst,fseg)) - logger.debug('cp(%s)' % os.path.join(dst,fseg)) - shutil.copy2(os.path.join(src, fseg), - os.path.join(dst, fseg)) - else: - break - #------------------------------------------------------------ - def DirIsIncludedFromBackup(self, d): - if (d != 'base' and d != 'gpmigrator' and - d != 'mirror_upgrade_state' and - d.startswith('gp_dump_') == False and - d.startswith('gp_cdatabase_') == False and - d.startswith('pg_log') == False ): - return True - return False - - #------------------------------------------------------------ - def CreateMasterCopy(self): - self.ForceShutDown() - - """ - Copy everthing under the master data, except base and gpmigrator, to gpmigrator/backup - Also, exclude gp_dump_*, gp_cdatabase_* and pg_log - """ - dst = os.path.join(self.masterdir, WORKDIR, BACKUPDIR) - self.RunCmd('rm -rf ' + dst) - os.mkdir(dst) - for d in os.listdir(self.masterdir): - if (self.DirIsIncludedFromBackup(d)): - self.do_copy(self.masterdir, dst, d) - - """ - Copy everything under catdir, except the backup dir. - """ - for src in self.mastercatdir: - dst = os.path.join(src, BACKUPDIR) - self.RunCmd('rm -rf ' + dst) - os.mkdir(dst) - for d in os.listdir(src): - if (d != BACKUPDIR): - self.do_copy(src, dst, d) - - #------------------------------------------------------------ - def DeleteMasterCatalogCopy(self): - # just print all the cat dirs - logger.info("Deleting Master Catalog Backup Copy: %s" % self.mastercatdir) - - """ - Delete catdir/backup - """ - for src in self.mastercatdir: - dst = os.path.join(src, BACKUPDIR) - self.RunCmd('rm -rf ' + dst) - - return "DeleteMasterDataDirCopy" - - #------------------------------------------------------------ - def RevertMasterCopyAndCatVersion(self): - self.ForceShutDown() - - """ - Delete everything which doesn't exists in the backup dir - Copy the backup to the cat dir - """ - for catdir in self.mastercatdir: - backupdir = os.path.join(catdir, BACKUPDIR) - for d in os.listdir(catdir): - if (d != BACKUPDIR): - backuploc = os.path.join(backupdir, d) - if (os.path.exists(backuploc)): - self.do_copy(backupdir, catdir, d) - else: - dst = os.path.join(catdir, d) - self.RunCmd('rm -rf ' + dst) - - """ - Delete everything under the master data, except base and gpmigrator, - mirror_upgrade_state file, and pg_log dir - to gpmigrator/backup - Copy things from backup - """ - backupdir = os.path.join(self.masterdir, WORKDIR, BACKUPDIR) - for d in os.listdir(self.masterdir): - if (self.DirIsIncludedFromBackup(d)): - backuploc = os.path.join(self.masterdir, WORKDIR, BACKUPDIR, d) - if (os.path.exists(backuploc)): - self.do_copy(backupdir, self.masterdir, d) - else: - dst = os.path.join(self.masterdir, d) - self.RunCmd('rm -rf ' + dst) - - """ - Set the catversion to the old one - """ - self.SetCatVersion(False) - - return "SwitchToMirror" - - #------------------------------------------------------------ - def SwitchToMirror(self): - # Set the mirror as primary - self.ForceShutDown() - self.Startup(self.oldenv, True) - self.Update("""begin; - set allow_system_table_mods='dml'; - update gp_segment_configuration set mode='c', role='p' where preferred_role='m'; - update gp_segment_configuration set mode='s', role='m', status='d' where preferred_role='p' and content >=0; - commit; - """) - self.Shutdown() - - return "RecoverPrimary" - - #------------------------------------------------------------ - def RecoverPrimary(self): - self.ForceShutDown() - self.Startup(self.oldenv) - logger.info("Running gprecoverseg to recover primary....") - - cmd = "$GPHOME/bin/gprecoverseg -a -F -l %s" % self.logdir - p = subprocess.Popen(cmd, shell = True, - close_fds = True, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - env=self.oldenv) - result = p.communicate() - if p.returncode != 0: - logger.debug(result[0]) - logger.warn("****************************************************") - logger.warn("Recovering Primary Failed...!!!!") - logger.warn(" Please examine the log in %s" % self.logdir) - logger.warn(" fix the problem and gprecoverseg -F to recover the primary") - logger.warn("****************************************************") - - return "AbortMirrorUpgrade" - - - #------------------------------------------------------------ - def DeleteMasterDataDirCopy(self): - # just print all the cat dirs and data dir - logger.info("Deleting Master Data Dir Backup Copy: %s" % self.masterdir) - - """ - Delete the work dir - """ - dst = os.path.join(self.masterdir, WORKDIR) - self.RunCmd('rm -rf ' + dst) - - return "FinishMirrorUpgrade" - - #------------------------------------------------------------ - def TurnOffMirror(self): - # Check that mirror and primary are in sync again..!!! - # Set the mirror off and put the primary on changetracking - self.ForceShutDown() - self.Startup(self.oldenv, True) - - nonsynccnt = self.Select('''select count(*) - from pg_catalog.gp_segment_configuration - where status != 'u' or mode != 's' or role != preferred_role - ''' ) - if (nonsynccnt[0] > 0): - raise UpgradeError("Mirror or Segments are down or out of sync") - - self.Update("""begin; - set allow_system_table_mods='dml'; - update gp_segment_configuration set mode='c' where preferred_role='p' and content >=0; - update gp_segment_configuration set status='d' where preferred_role='m'; - commit; - """) - self.Shutdown() - return "CatUpgrade" - - #------------------------------------------------------------ - def CatUpgrade(self): - newversion = GpVersion(self.getversion(self.newhome, self.newenv)) - snv = newversion.getVersionRelease() - - self.ForceShutDown() - - try: - # Upgrade pg_control first, and start up the database with - # the upgrade flag on. - self.SetCatVersion(True) - self.Startup(self.newenv, False, True) - runfile = "%s/share/postgresql/upgrade/upg2_catupgrade.sql" % self.newhome - self.ExecuteSQLFileOnAllDBs(runfile) - self.Shutdown() - self.Startup(self.newenv, False, False) - self.PerformPostUpgrade() - except: - logger.error(traceback.format_exc()) - return "RevertMasterCopyAndCatVersion" - - return "RecoverMirror" - - #------------------------------------------------------------ - def RecoverMirror(self): - self.ForceShutDown() - self.Startup(self.newenv) - logger.info("Running gprecoverseg to recover mirror....") - - cmd = ' '.join(["$GPHOME/bin/gprecoverseg", '-a', '-l', self.logdir]) - p = subprocess.Popen(cmd, shell = True, - close_fds = True, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - env=self.newenv) - result = p.communicate() - - if p.returncode != 0: - logger.debug(result[0]) - logger.warn("****************************************************") - logger.warn("Recovering Mirror Failed...!!!!") - logger.warn(" Please examine the log in %s" % self.logdir) - logger.warn(" fix the problem and re-run gpmigrator_mirror to recover the mirror") - logger.warn("****************************************************") - - return "DeleteMasterCatalogCopy" - - #------------------------------------------------------------ - def LoadState(self): - # Setup the current state and statefile - # if state file exists, the current state is the last state - self.mirror_upgrade_state = "CreateMirrorWorkDir" - statefile = os.path.join(self.masterdir, 'mirror_upgrade_state') - hasconfigfile = False - if os.path.exists(statefile): - logger.info("Found previous upgrade") - self.mirror_upgrade_state_file = open(statefile, 'r') - for line in self.mirror_upgrade_state_file: - line = line.strip() - if (line == "TurnOffMirror"): - hasconfigfile = True - if (line == 'DeleteMasterDataDirCopy'): - hasconfigfile = False - self.mirror_upgrade_state = line - self.mirror_upgrade_state_file.close() - self.mirror_upgrade_state_file = open(statefile, 'a') - - if (hasconfigfile): - self.ReadInfo() - - #------------------------------------------------------------ - def AbortMirrorUpgrade(self): - self.mirror_upgrade_state_file.close() - statefile = os.path.join(self.masterdir, 'mirror_upgrade_state') - open(statefile, 'w').truncate() - - logger.info("---------------------------------------------------") - logger.info("Upgrade Aborted") - logger.info("---------------------------------------------------") - logger.info("Please fix up the error and re-run upgrade") - logger.info("---------------------------------------------------") - - sys.exit(0) - #------------------------------------------------------------ - def FinishMirrorUpgrade(self): - self.mirror_upgrade_state_file.close() - statefile = os.path.join(self.masterdir, 'mirror_upgrade_state') - self.RunCmd('rm -rf ' + statefile) - - logger.info("---------------------------------------------------") - logger.info("Upgrade Successful") - logger.info("---------------------------------------------------") - logger.info("Please consult release notes for post-upgrade ") - logger.info("instructions to complete environment configuration") - logger.info("---------------------------------------------------") - - sys.exit(0) - - #------------------------------------------------------------ - def PerformMirroredUpgrade(self): - #========================================================== - # Upgrade strategy - # Use mirror as backup: start the db with mirror off and put primary in changetracking. - # Then start the db in upgrade mode to run the upgrade script as usual. - # When the script finished, use gprecoverseg to bring the mirror back in sync. - # In case of error when running the upgrade script, set the primary as down and bring - # the mirror up as primary. Do either full recovery (gprecoverseg -F) or delta - # (gpcheckmirrorseg, gprepairmirrorseg and gprecoverseg). - # - # Master is not mirrored. We'll copyed everything in the master to a backup location - #========================================================== - - # We use a state machine approach to handle fault and crash. - # Fault will be cought as exception and it'll go the fault hanlding - # state. - # Crash will not change the state. When restart, it'll re-do the current state - # agagin. - - # Initialize the current state - # if a state file already exists, the current state is in the state file - self.LoadState() - - faultinject_index = 100 - - while True: - logger.info(" ") - logger.info("**************************************************") - logger.info("**************************************************") - logger.info("Executing state:%s"%self.mirror_upgrade_state) - logger.info("**************************************************") - logger.info("**************************************************") - if (self.mirror_upgrade_state == "CreateMirrorWorkDir"): - self.mirror_upgrade_state = self.CreateMirrorWorkDir() - elif (self.mirror_upgrade_state == "ExtractInfoAndCopyMaster"): - self.mirror_upgrade_state = self.ExtractInfoAndCopyMaster() - elif (self.mirror_upgrade_state == "TurnOffMirror"): - self.mirror_upgrade_state = self.TurnOffMirror() - elif (self.mirror_upgrade_state == "CatUpgrade"): - self.mirror_upgrade_state = self.CatUpgrade() - elif (self.mirror_upgrade_state == "RecoverMirror"): - self.mirror_upgrade_state = self.RecoverMirror() - elif (self.mirror_upgrade_state == "DeleteMasterCatalogCopy"): - self.mirror_upgrade_state = self.DeleteMasterCatalogCopy() - elif (self.mirror_upgrade_state == "DeleteMasterDataDirCopy"): - self.mirror_upgrade_state = self.DeleteMasterDataDirCopy() - elif (self.mirror_upgrade_state == "RevertMasterCopyAndCatVersion"): - self.mirror_upgrade_state = self.RevertMasterCopyAndCatVersion() - elif (self.mirror_upgrade_state == "SwitchToMirror"): - self.mirror_upgrade_state = self.SwitchToMirror() - elif (self.mirror_upgrade_state == "RecoverPrimary"): - self.mirror_upgrade_state = self.RecoverPrimary() - elif (self.mirror_upgrade_state == "AbortMirrorUpgrade"): - self.mirror_upgrade_state = self.AbortMirrorUpgrade() - elif (self.mirror_upgrade_state == "FinishMirrorUpgrade"): - self.mirror_upgrade_state = self.FinishMirrorUpgrade() - else: - # can't get here - raise UpgradeError('Unknown Command : %s' % self.mirror_upgrade_state) - sys.exit(1) - - if self.faultinjection == faultinject_index: - raise UpgradeError("faultinjection=%d" % self.faultinjection) - faultinject_index = faultinject_index+1 - - self.mirror_upgrade_state_file.write("%s\n" % self.mirror_upgrade_state) - self.mirror_upgrade_state_file.flush() - sys.exit(0) - - -#============================================================ -if __name__ == '__main__': - coverage = GpCoverage() - coverage.start() - - # Create a new GPUpgrade - should never throw exception - u = GPUpgrade() - - # parses and validates input - try: - u.Setup() - except Exception, e: - logger.fatal(str(e)) - sys.exit(1) - - # Execute the main upgrade routine, the Run() function itself is - # just a switch on the "cmd", which switches between the main - # PerformUpgrade() function and all the various subcommands that - # are upgraded by backend executions. - try: - u.Run() - # Any error that can happen - except KeyboardInterrupt: - logger.fatal('***************************************') - logger.fatal('== Upgrade interrupted by user == <<<<< ') - logGenericErrorMessage() - logger.fatal('***************************************') - sys.exit(1) - - except ConnectionError, e: - logger.fatal('***************************************') - logger.fatal(str(e)) - logGenericErrorMessage() - logger.fatal('***************************************') - sys.exit(1) - - except UpgradeError, e: - logger.fatal('***************************************') - logger.fatal(str(e)) - logGenericErrorMessage() - logger.fatal('***************************************') - sys.exit(1) - - except CmdError, e: - logger.fatal('***************************************') - logger.fatal('Error executing command:') - logger.fatal(' ' + str(e.cmd.strip())) - logger.fatal(' ' + str(e.stderr.strip())) - logGenericErrorMessage() - logger.fatal('***************************************') - sys.exit(1) - - # Shouldn't get any of these, if they occur it is probably a bug in - # the upgrader. - except Exception, e: - logger.fatal('***************************************') - logger.fatal(str(e)) - logger.fatal('***************************************') - logger.fatal(traceback.format_exc()) - sys.exit(1) - finally: - coverage.stop() - coverage.generate_report() - - sys.exit(0) diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index 4670cf25b366f269be744bcdbc9528c0c9401ed1..18f5b897fffc8939a879ef10aca08e6e4f21d656 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -1684,7 +1684,6 @@ def createTempDirectoryName(masterDataDirectory, tempDirPrefix): #------------------------------------------------------------------------- # gp_dbid methods moved to gp_dbid.py, but this class was left here -# to avoid changing gpmigrator and gpmigrator_mirror (which is the only caller). # class GpCreateDBIdFile(Command): diff --git a/gpMgmt/bin/gppylib/gp_dbid.py b/gpMgmt/bin/gppylib/gp_dbid.py index 4c55eda0288377f4573a42f1c7e1baabfb7950ed..d0d429c414219d0fa1f0bd4bc87722c913fd75a2 100644 --- a/gpMgmt/bin/gppylib/gp_dbid.py +++ b/gpMgmt/bin/gppylib/gp_dbid.py @@ -19,8 +19,8 @@ STANDBY_DBID_RE = re.compile(r"standby_dbid\s*=\s*(\d+)") class GpDbidFile: """ - Used by gpstart, gpinitstandby, gpactivatestandby and indirectly - by gpmigrator via gpsetdbid.py to manage the gp_dbid file. + Used by gpstart, gpinitstandby, and gpactivatestandby + via gpsetdbid.py to manage the gp_dbid file. """ def __init__(self, datadir, do_read=False, logger=None): diff --git a/gpMgmt/bin/gppylib/gparray.py b/gpMgmt/bin/gppylib/gparray.py index cb5016f936e6a7564f480e5825f40c37fa8700dd..e829866d87d31e45cf9eda7ce88a35395bda9367 100755 --- a/gpMgmt/bin/gppylib/gparray.py +++ b/gpMgmt/bin/gppylib/gparray.py @@ -1494,7 +1494,6 @@ class GpArray: (called by gpexpand.) Note: Currently this is only used by the gpexpand rollback facility, - and by gpmigrator utility, there is currently NO expectation that this file format is saved on disk in any long term fashion. diff --git a/gpMgmt/bin/gppylib/operations/gpMigratorUtil.py b/gpMgmt/bin/gppylib/operations/gpMigratorUtil.py deleted file mode 100644 index 098e52677853d55ac61018173fef97b166737f37..0000000000000000000000000000000000000000 --- a/gpMgmt/bin/gppylib/operations/gpMigratorUtil.py +++ /dev/null @@ -1,850 +0,0 @@ -#!/usr/bin/env python -# -# This is a common util lib for gpmigrator and gpmigrator_mirror. -# -# Copyright (c) Greenplum Inc 2010. All Rights Reserved. -# -import pickle, base64 -import subprocess, csv -import optparse, tempfile, re, traceback -from datetime import date -from time import localtime, strftime, sleep, time -from threading import Thread -from Queue import Queue -import urllib # for passing strings across gpssh -import shutil # for copying -from pygresql import pg # Database interaction -from gppylib.gplog import * # Greenplum logging facility -from gppylib.commands import base # Greenplum layer for worker pools -from gppylib.commands import unix # Greenplum layer for unix interaction -from gppylib.commands.gp import GpCreateDBIdFile -from gppylib.db import dbconn -from gppylib.gpversion import GpVersion -from gppylib.gparray import GpArray, GpDB -from gppylib.gphostcache import GpHostCache -from gppylib.gpcoverage import GpCoverage -from gppylib.operations.gpMigratorUtil import * - -libdir = os.path.join(sys.path[0], 'lib/') -logger = get_default_logger() -MIGRATIONUSER = 'gpmigrator' -LOCKEXT = '.gpmigrator_orig' -WORKDIR = 'gpmigrator' -BACKUPDIR = 'backup' -UPGRADEDIR = 'upgrade' -PARALLELISM = 16 - -#============================================================ -def make_conn(ug, user, db, options, port, sockdir): - retries = 5 - for i in range(retries): - try: - logger.debug("making database connection: user = %s, " - "dbname = %s port = %i" - % (user, db, port)) - conn = pg.connect(user=user, - dbname=db, - opt=options, - port=port) - break - except pg.InternalError, e: - if 'too many clients already' in str(e) and i < retries: - logger.warning('Max Connection reached, attempt %d / %d' % (i+1, retries)) - sleep(2) - continue - raise ConnectionError(str(e)) - return conn - -#============================================================ -def cli_help(execname): - help_path = os.path.join(sys.path[0], '..', 'docs', 'cli_help', execname + '_help'); - f = None - try: - try: - f = open(help_path); - return f.read(-1) - except: - return '' - finally: - if f: f.close() - -#============================================================ -def usage(execname): - print cli_help(execname) or __doc__ - -#============================================================ -class ConnectionError(StandardError): pass -class UpgradeError(StandardError): pass -class CmdError(StandardError): - def __init__(self, cmd, stdout, stderr): - self.cmd = cmd - self.stdout = stdout - self.stderr = stderr - def __str__(self): - return self.stderr - -#============================================================ -def is_supported_version(version, upgrade=True): - ''' - Checks that a given GpVersion Object is supported for upgrade/downgrade. - We do not support: - Versions < PREVIOUS MINOR RELEASE - Versions > main - ''' - - if upgrade: - upstr = "upgrade" - else: - upstr = "downgrade" - - build = version.getVersionBuild() - if not re.match(r"(.*)", build): - raise UpgradeError( - "Greenplum Version '%s' is not supported for %s" - % (str(version), upstr)) - - main = GpVersion('main') - if version > main: - raise UpgradeError( - "To %s Greenplum Version '%s' use the %s tool " - "shipped with that release" - % (upstr, str(version), upstr)) - if version < (main << 1): - raise UpgradeError( - "Greenplum Version '%s' is not supported for %s" - % (str(version), upstr)) - return True - - -#============================================================ -class GpUpgradeCmd(base.Command): - def __init__(self, name, cmd, ctxt=base.LOCAL, remoteHost=None): - cmdStr = ' '.join(cmd) - base.Command.__init__(self, name, cmdStr, ctxt, remoteHost) - - -#============================================================ -class GPUpgradeBase(object): - def __init__(self): - self.cmd = 'MASTER' # Command being run - self.array = None - self.dbs = {} # dict of databases in the array - self.dbup = None - self.debug = False - self.faultinjection = None - self.hostcache = None - self.interrupted = False # SIGINT recieved - self.logdir = None - self.logfile = None - self.masterdir = None # Master Data Directory - self.masterport = 5432 # Master Port - self.mirrors = [] # list of mirrors - self.newenv = None # enviornment: new gp env - self.newhome = None # New gphome/bin directory - self.oldenv = None # enviornment: old gp env - self.oldhome = None # Old gphome/bin directory - self.option = None # Argument passed to cmd - self.path = None # Default path - self.pool = None # worker pool - self.skipcheckcat = False - self.user = None # db admin user - self.workdir = None # directory: masterdir/gpmigrator - self.checkonly = False # checkcat only; do not run upgrade - - #------------------------------------------------------------ - def RunCmd(self, cmd, env=None, utility=False, supressDebug=False): - ''' - Runs a single command on this machine - ''' - - if type(cmd) in [type('string'), type(u'unicode')]: - cmdstr = cmd - cmd = cmd.split(' ') - elif type(cmd) == type(['list']): - cmdstr = ' '.join(cmd) - else: - logger.warn("Unknown RunCmd datatype '%s'" % str(type(cmd))) - cmdstr = str(cmd) - - if self.debug and not supressDebug: - logger.debug("ENV: " + str(env)) - logger.debug("CMD: " + cmdstr) - - # If utility mode has been specified add role to env - if env == None: - env = {'PATH': self.path} - - if utility: - env['PGOPTIONS'] = '-c gp_session_role=utility' - - try: - pipe = subprocess.Popen(cmd, env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=True) - result = pipe.communicate(); - except OSError, e: - raise CmdError(cmdstr, '', str(e)) - - if self.debug and not supressDebug: - logger.debug(result[1]) - - if pipe.returncode: - raise CmdError(' '.join(cmd), result[0], result[1]) - else: - return result[0].strip(); - - - #------------------------------------------------------------ - def SetupEnv(self, gphome, masterdir): - ''' - Sets up environment variables for Greenplum Administration - ''' - home = os.environ.get('HOME') - lpath = os.environ.get('LD_LIBRARY_PATH') - dypath = os.environ.get('DYLD_LIBRARY_PATH') - - # Add $GPHOME/bin to the path for this environment - path = '%s/bin:%s/ext/python/bin:%s' % (gphome, gphome, self.path) - - if lpath: - lpath = '%s/lib:%s/ext/python/lib:%s' % (gphome, gphome, lpath) - else: - lpath = '%s/lib:%s/ext/python/lib' % (gphome, gphome) - if dypath: - dypath = '%s/lib:%s/ext/python/lib:%s' % (gphome, gphome, dypath) - else: - dypath = '%s/lib:%s/ext/python/lib' % (gphome, gphome) - - env = {} - env['HOME'] = home - env['USER'] = self.user - env['LOGNAME'] = self.user - env['GPHOME'] = gphome - env['PATH'] = path - env['LD_LIBRARY_PATH'] = lpath - env['DYLD_LIBRARY_PATH'] = dypath - env['PYTHONPATH'] = os.path.join(gphome, 'lib', 'python') - env['PYTHONHOME'] = os.path.join(gphome, 'ext', 'python') - if masterdir: - env['MASTER_DATA_DIRECTORY'] = masterdir - env['PGPORT'] = str(self.masterport) - return env - - - #------------------------------------------------------------ - def Select(self, qry, db='template1', port=None, forceutility=False, forceUseEnvUser=False): - # forceUseEnvUser: user will be GPMIGRATOR if we're in lockdown mode(gpmigrator); otherwise - # it'll be the env user. Setting forceuseEnvUser will always use env user, - # even though we're in gpmigrator and in lockdown. - ''' - Execute a SQL query and return an array of result rows - + Single columns will be returned as an array of values - + Multiple columns will be returned as an array of tuples - ''' - - # - # If we are executing from one of the segments then we don't - # actually know where the data directory is for the segment - # is so we can't check if we are in lockdown mode - # - # Fortunately we never actually lock the segments, so we can - # always run as current user. - # - if self.cmd == 'MASTER': - if not self.dbup: - raise UpgradeError('database is down') - [env, utility] = self.dbup - - locked = os.path.join(env['MASTER_DATA_DIRECTORY'], - 'pg_hba.conf'+LOCKEXT) - if os.path.exists(locked) and not forceUseEnvUser: - user = MIGRATIONUSER - else: - user = env['USER'] - - else: - - # If we lockdown on segments this needs to change, some method - # would need to be made to determine if the segment in question - # is locked - user = self.user - utility = True - - if port == None: - port = self.masterport - elif port != self.masterport: - utility = True - - if utility or forceutility: - options = '-c gp_session_role=utility' - else: - options = '' - - if self.__dict__.get('sock_dir'): - sockdir = self.sock_dir - else: - sockdir = '/tmp/' - conn = make_conn(self, user, db, options, port, sockdir) - - curs = None - logger.debug('executing query ' + qry) - try: - curs = conn.query(qry) - except Exception, e: - conn.close() - logger.fatal('Error executing SQL') - raise ConnectionError('%s\nsql> %s' % (str(e), qry)) - - rows = [] - for tuple in curs.dictresult(): - if len(tuple) == 1: - rows.append(tuple.values()[0]) - else: - rows.append(tuple) - conn.close() - return rows - - #------------------------------------------------------------ - def Update(self, qry, db='template1', port=None, forceutility=False, upgradeMode=False, modSysTbl=False, forceUseEnvUser=False): - # forceUseEnvUser: user will be GPMIGRATOR if we're in lockdown mode(gpmigrator); otherwise - # it'll be the env user. Setting forceuseEnvUser will always use env user, - # even though we're in gpmigrator and in lockdown. - - ''' - Execute a SQL query expecting no output - ''' - - # If we are executing from one of the segments then we don't - # have the current database state, so we have to know what to - # do via other means. - # - # The assumption is: - # A) utility = true - # B) pguser = MIGRATIONUSER - # - # If we ever need to run as the non-upgrade user from a segment - # this code will obviously need to change - # - if self.cmd == 'MASTER': - if not self.dbup: - raise UpgradeError('database is down') - [env, utility] = self.dbup - - locked = os.path.join(env['MASTER_DATA_DIRECTORY'], - 'pg_hba.conf'+LOCKEXT) - if os.path.exists(locked) and not forceUseEnvUser: - user = MIGRATIONUSER - else: - user = env['USER'] - else: - - # If we lockdown on segments this needs to change, some method - # would need to be made to determine if the segment in question - # is locked - user = self.user - utility = True - - if port == None: - port = self.masterport - - options = ' ' - if modSysTbl: options += ' -c allow_system_table_mods=dml' - if utility or forceutility: options += ' -c gp_session_role=utility' - if upgradeMode: options += ' -c gp_maintenance_conn=true' - - if self.__dict__.get('sock_dir'): - sockdir = self.sock_dir - else: - sockdir = '/tmp/' - conn = make_conn(self, user, db, options, port, sockdir) - - logger.debug('executing query ' + qry) - try: - conn.query(qry) - except Exception, e: - logger.fatal('Error executing SQL') - raise ConnectionError('%s\nsql> %s' % (str(e), qry)) - - finally: - conn.close() - - #------------------------------------------------------------ - def CheckUp(self, tryconn = False): - ''' - Checks if one of the postmasters is up and running. - Return the environment of the running server - ''' - olddir = self.oldenv['MASTER_DATA_DIRECTORY'] - newdir = self.newenv['MASTER_DATA_DIRECTORY'] - env = None - - if olddir and os.path.exists('%s/postmaster.pid' % olddir): - logger.debug('CheckUp: found old postmaster running') - env = self.oldenv - if newdir and os.path.exists('%s/postmaster.pid' % newdir): - logger.debug('CheckUp: found new postmaster running') - env = self.newenv - - if not env: - logger.info('Postmaster not running') - raise UpgradeError("Postmaster failed to start") - - if tryconn: - try: - logger.debug('trying to establish connection to database') - logger.debug('user = %s, db = template1, port = %i, dir = %s' - % (self.user, self.masterport, self.sock_dir)) - conn = make_conn(self, self.user, 'template1', '', self.masterport, - self.sock_dir) - except Exception: - raise UpgradeError("Found a postmaster.pid but postmaster " + - "running") - conn.close() - return env - - #------------------------------------------------------------ - def CheckDown(self, warn=False): - ''' - Checks that neither postmaster is running and that the database - was cleanly shutdown. - ''' - if self.cmd == 'MASTER': - datadirs = [ self.masterdir ] - else: - datadirs = self.datadirs - - shutdown_re = re.compile('Database cluster state: *(.*)') - - for oldseg in datadirs: - (d, content) = os.path.split(oldseg) - newseg = os.path.join(d, WORKDIR, UPGRADEDIR, content) - - # Only check the upgrade directory if its not legacy - if self.datadirs: - dirs = [oldseg, newseg] - else: - dirs = [oldseg] - - for dir in dirs: - if dir == oldseg: env = self.oldenv - else: env = self.newenv - - # upgrade directory might not actually exist - if not os.path.isdir(dir): - continue - - pid = os.path.join(dir, 'postmaster.pid') - if os.path.exists(pid): - raise UpgradeError("Greenplum process running: " + pid) - - shutdown = self.RunCmd('pg_controldata ' + dir, env=env) - for line in shutdown.split('\n'): - m = shutdown_re.match(line) - if m: - if m.group(1) == 'shut down': - break - msg = 'pg_controldata: "Database cluster state: %s"\n' % m.group(1) - msg += 'Greenplum segment %s did not shutdown cleanly' % dir - if warn: - logger.warn(msg) - else: - raise UpgradeError(msg) - - if self.cmd == 'MASTER' and self.datadirs: - self.CallSlaves('CHKDOWN') - - return True - - #------------------------------------------------------------ - def Startup(self, env, utility=False, upgrade=False): - ''' - Starts up the specified database - ''' - - if self.dbup: - raise UpgradeError('database already started') - self.dbup = [env, utility] - - isu = "" - if utility: - isu = " in utility mode" - elif upgrade: - isu = " in upgrade mode" - - if (env == self.oldenv): - logger.info('Starting old Greenplum postmaster%s' % isu) - else: - logger.info('Starting new Greenplum postmaster%s' % isu) - - try: - cmd = "gpstart -a" - if utility: - cmd = cmd +' -m' - env['GPSTART_INTERNAL_MASTER_ONLY'] = '1' - - if upgrade: cmd = cmd +' -U upgrade' - - cmd += ' -l %s' % self.logdir - - locked = os.path.join(env['MASTER_DATA_DIRECTORY'], - 'pg_hba.conf'+LOCKEXT) - try: - if os.path.exists(locked): - env['PGUSER'] = MIGRATIONUSER - logger.debug("lockfile: '%s' exists" % locked) - else: - logger.debug("lockfile: '%s' does not exist" % locked) - - logger.debug("Starting cluster with env = %s" % str(env)) - pid = subprocess.Popen(cmd, preexec_fn=os.setpgrp, - env=env, shell=True, - stdout=self.logfile, - stderr=self.logfile, - close_fds=True) - finally: - if os.path.exists(locked): - del env['PGUSER'] - - # Ignore interrupt requests until startup is done - error = None - retcode = None - while retcode == None: - try: - retcode = pid.wait(); - - except KeyboardInterrupt, e: - if not self.interrupted: - logger.fatal('***************************************') - logger.fatal('SIGINT-: Upgrade Aborted') - logger.fatal('***************************************') - logger.info( 'Performing clean abort') - self.interrupted = True - error = e - else: - logger.info( 'SIGINT-: Still processing shutdown') - - if retcode < 0: - raise UpgradeError("Startup terminated by signal"); - - today = date.today().strftime('%Y%m%d') - logname = os.path.join(self.logdir, 'gpstart_%s.log' % today) - if retcode == 1: - logger.warn('***************************************') - logger.warn('Warnings generated starting cluster') - logger.warn('Check %s for detailed warnings' % logname) - logger.warn('***************************************') - if retcode > 1: - logger.fatal('***************************************') - logger.fatal('Startup failed with error code %d' % retcode) - logger.fatal('Check %s for detailed warnings' % logname) - logger.fatal('***************************************') - raise UpgradeError('Startup failed') - - # If we recieved an interrupt, resignal it now that the startup is done - if error: - raise error - - except OSError, e: - logger.fatal(str(e)) - raise UpgradeError('Startup failed') - - self.CheckUp(); - - - #------------------------------------------------------------ - def Shutdown(self): - ''' - Stops the specified database - ''' - if not self.dbup: - return - [env, utility] = self.dbup - - dir = env['MASTER_DATA_DIRECTORY'] - if not os.path.exists('%s/postmaster.pid' % dir): - logger.warn( - 'Shutdown skipped - %s/postmaster.pid not found' % dir) - return - - if (env == self.oldenv): - logger.info('Shutting down old Greenplum postmaster') - else: - logger.info('Shutting down new Greenplum postmaster') - - locked = os.path.join(env['MASTER_DATA_DIRECTORY'], - 'pg_hba.conf'+LOCKEXT) - - try: - - # Note on arguments to gpstop: - # This code has gone back and forth on -s vs -f for shutdown: - # - # -f aborts active connections. This is a good thing. If - # a user or script snuck in a connection before we were able - # to establish the lockdown then we want to abort that - # connection otherwise it will cause the upgrade to fail and - # that is bad. - # - # Prior versions would sometimes issue a kill for fast - # shutdown, this is a problem since we need the database - # shutdown cleanly with no pending xlog transactions. - # Because of that we switched to -s. - # - # -s causes problems because it is a stop that will fail if a - # session is connected and we want to abort active sessions. - # Because of that we switched back to -f. - # - # The belief is currently that the current version of gpstop - # should be good with passing -f. To help safeguard this belief - # there is a check when we set the catalog version to ensure - # that the database shutdown cleanly. - # - # If this needs to be changed again please read the above, - # consider what happens if you try to upgrade with an active - # connection open to the database, and procede cautiously. - - if utility: cmd = 'gpstop -a -f -m' - else: cmd = 'gpstop -a -f' - - cmd += ' -l %s' % self.logdir - - locked = os.path.join(env['MASTER_DATA_DIRECTORY'], - 'pg_hba.conf'+LOCKEXT) - try: - if os.path.exists(locked): - env['PGUSER'] = MIGRATIONUSER - logger.debug('handling locked shutdown') - logger.debug('shutting down with env: %s' % str(env)) - logger.debug('shutting down with command: %s' % cmd) - - pid = subprocess.Popen(cmd, preexec_fn=os.setpgrp, - env=env, shell=True, - stdout=self.logfile, - stderr=self.logfile, - close_fds=True) - finally: - if os.path.exists(locked): - del env['PGUSER'] - self.dbup = None - - # Ignore interrupt requests until shutdown is done - error = None - retcode = None - while retcode == None: - try: - retcode = pid.wait(); - - except KeyboardInterrupt, e: - if not self.interrupted: - logger.fatal('***************************************') - logger.fatal('SIGINT-: Upgrade Aborted') - logger.fatal('***************************************') - logger.info( 'Performing clean abort') - self.interrupted = True - error = e - else: - logger.info( 'SIGINT-: Still processing shutdown') - - if retcode < 0: - raise UpgradeError("Shutdown terminated by signal"); - today = date.today().strftime('%Y%m%d') - logname = os.path.join(self.logdir, 'gpstop_%s.log' % today) - if retcode == 1: - logger.warn('***************************************') - logger.warn('Warnings generated stopping cluster') - logger.warn('Check %s for detailed warnings' % logname) - logger.warn('***************************************') - if retcode > 1: - logger.fatal('***************************************') - logger.fatal('Shutdown failed with error code %d' % retcode) - logger.fatal('Check %s for detailed error' % logname) - logger.fatal('***************************************') - raise UpgradeError('Shutdown failed') - - # If we recieved an interrupt, resignal it now that the startup is done - if error: - raise error - - except OSError, e: - logger.fatal(str(e)) - raise UpgradeError('Shutdown failed') - - self.CheckDown(); - - - #------------------------------------------------------------ - def PreUpgradeCheck(self): - ''' - Check master/segment binary version, GUC, free space, catalog - ''' - self.CheckBinaryVersion() - self.CheckGUCs() - self.CheckFreeSpace() - if not self.skipcheckcat: - self.CheckCatalog() - else: - logger.info('Skip Checking Catalog') - - #------------------------------------------------------------ - def CheckBinaryVersion(self): - ''' - Validate that the correct binary is installed in all segments - ''' - logger.info('Checking Segment binary version') - hosts = self.Select("select distinct hostname from gp_segment_configuration"); - masterversion = self.getversion(self.newhome, self.newenv) - - cmdStr = '%s/bin/pg_ctl --gp-version' % self.newhome - for uh in hosts: - cmd = base.Command(uh, cmdStr, base.REMOTE, uh) - self.pool.addCommand(cmd) - self.pool.join() - items = self.pool.getCompletedItems() - for i in items: - if i.results.rc: - logger.error("error on host %s with error: %s" % (i.remoteHost, i.results.stderr)) - raise UpgradeError('Cannot verify segment GPDB binary') - if not i.results.stdout: - logger.error("could not find version string from host %s with command: %s" % (i.remoteHost, cmdStr)) - raise UpgradeError('Cannot verify segment GPDB binary') - version_string = i.results.stdout.strip() - if version_string != masterversion: - logger.error("version string on host %s: '%s' does not match expected: '%s'" % (i.remoteHost, version_string, masterversion)) - raise UpgradeError('Master/Segment binary mismatch') - - #------------------------------------------------------------ - def CheckGUCs(self): - ''' - Validate that the GUCs are set properly - - gp_external_enable_exec must be set to true - ''' - logger.info('Checking GUCs') - disable_exec_ext = self.Select("select current_setting('gp_external_enable_exec')"); - if (disable_exec_ext[0] == 'off'): - logger.fatal('***************************************') - logger.fatal("gp_external_enable_exec is set to 'false'.") - logger.fatal("Please set gp_external_enable_exec to 'true'.") - logger.fatal('***************************************') - raise UpgradeError('Invalid GUC value') - - #------------------------------------------------------------ - def CheckFreeSpace(self): - ''' - Validate that we've enough space for upgrade. - Right now, we require 2GB. It's arbitrary. - ''' - logger.info('Checking Free Space') - nospacesegs = self.Select("select dfsegment from gp_toolkit.gp_disk_free where dfspace::float/1024/1024 < 2") - if (len(nospacesegs)>0): - logger.fatal('***************************************') - logger.fatal('The following segment ID has less than 2GB of free space.') - logger.fatal('Please make sure that there is at least 2GB of free space on each segment before upgrade.') - logger.fatal(nospacesegs) - logger.fatal('***************************************') - raise UpgradeError('Insufficient Space on Segment') - - df=unix.DiskFree.get_disk_free_info_local('gpmigrator_check_freespace', self.masterdir) - mdf = float(df[3])/1024/1024 - if (mdf < 2): - logger.fatal('***************************************') - logger.fatal('The Master data directory has only %sGB of free space.' % mdf) - logger.fatal('Please make sure that there is at least 2GB of free space on the master.') - logger.fatal('***************************************') - raise UpgradeError('Insufficient Space on Master') - - #------------------------------------------------------------ - def CheckCatalog(self): - ''' - Validate that the created catalog looks good - ''' - logger.info('Checking Catalog') - self.CheckUp() - [env, utility] = self.dbup - - locked = os.path.join(env['MASTER_DATA_DIRECTORY'], - 'pg_hba.conf'+LOCKEXT) - if os.path.exists(locked): - user = MIGRATIONUSER - else: - user = env['USER'] - - exec_cmd = libdir + '/gpcheckcat -p %d -U %s -B %i ' % \ - (self.masterport, user, PARALLELISM) - - oids = sorted(self.dbs.keys()) - for dboid in oids: - db = self.dbs[dboid] - - if db == 'template0': - continue - - cmd = exec_cmd + db - logger.info('... Checking ' + db) - - logger.debug("CMD: " + cmd) - - outfilename = os.path.join(self.workdir, 'gpcheckcat_%s.log' % db) - outfilename = outfilename.replace(' ', '_') - outfile = open(outfilename, 'w') - - pipe = subprocess.Popen(cmd, shell=True, stdout=outfile, stderr=outfile,close_fds=True) - retcode = pipe.wait() - - if retcode < 0: - raise UpgradeError('Catalog Check terminated by signal') - if retcode > 0: - raise UpgradeError('Catalog Check Failed - see %s for details' % outfilename) - - - #------------------------------------------------------------ - def PerformPostUpgrade(self): - ''' - Handles various post upgrade tasks including: - - Populates the gp_toolkit schema - - Performs updates for the gpperfmon database - ''' - try: - - # Get the admin role and all the databases - logger.info("Installing gp_toolkit") - rolname = self.Select("select rolname from pg_authid where oid=10")[0] - - # Read the toolkit sql file into memory - fname = '%s/share/postgresql/gp_toolkit.sql' % self.newhome - sql = open(fname, 'r').read() - - # Set our role to the admin role then execute the sql script - sql = "SET SESSION AUTHORIZATION %s;\n%s" % (rolname, sql) - oids = sorted(self.dbs.keys()) - for dboid in oids: - db = self.dbs[dboid] - - if db == 'template0': - continue - self.Update(sql, db=db) - - except BaseException, e: - sys.stderr.write(traceback.format_exc()) - sys.stderr.write(str(e)) - raise e - - #------------------------------------------------------------ - def getversion(self, home,env): - binary = os.path.join(home, 'bin', 'pg_ctl') - if not os.path.exists(binary): - raise UpgradeError(binary + ' not found') - - try: - return self.RunCmd('pg_ctl --gp-version', env=env) - except CmdError: - pass - conf = '%s/include/pg_config.h' % home - if os.path.exists(conf): - cmd = 'grep PG_VERSION_STR ' + conf - try: - return self.RunCmd(cmd, env=self.oldenv) - except Exception, e: - logger.fatal(str(e)) - raise UpgradeError('Unable to determine version of %s' % home) - diff --git a/gpMgmt/bin/gppylib/operations/test/test_migrator.py b/gpMgmt/bin/gppylib/operations/test/test_migrator.py deleted file mode 100644 index 5dee0771ff156e90dbb431255de7f504aa738d4b..0000000000000000000000000000000000000000 --- a/gpMgmt/bin/gppylib/operations/test/test_migrator.py +++ /dev/null @@ -1,20 +0,0 @@ -import unittest2 as unittest -from gppylib.operations.gpMigratorUtil import is_supported_version, UpgradeError -from gppylib.gpversion import GpVersion - -class MigratorTests(unittest.TestCase): - def test_is_supported_version_weird(self): - v = GpVersion('4.4.0.0 build monkey') - with self.assertRaisesRegexp(UpgradeError, - "Greenplum Version '4.4.0.0 build monkey' is not supported for upgrade"): - is_supported_version(v) - def test_is_supported_version_low(self): - v = GpVersion('main') << 2 - with self.assertRaisesRegexp(UpgradeError, - "Greenplum Version '4.1.0.0 build dev' is not supported for upgrade"): - is_supported_version(v) - def test_is_supported_version_high(self): - v = GpVersion('99.99') - with self.assertRaisesRegexp(UpgradeError, - "To upgrade Greenplum Version '99.99 build dev' use the upgrade tool shipped with that release"): - is_supported_version(v) diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_gpmigrator_util.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_gpmigrator_util.py deleted file mode 100755 index 01600f86996f88af11c9571f0e985e6b6ee35bc5..0000000000000000000000000000000000000000 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_gpmigrator_util.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) Pivotal Inc 2014. All Rights Reserved. -# - -import os -import unittest2 as unittest -from gppylib.gpversion import GpVersion -from gppylib.operations.gpMigratorUtil import is_supported_version -from mock import patch, MagicMock, Mock - -class IsSupportedVersionTestCase(unittest.TestCase): - def test_is_supported_version_major_version_upgrade(self): - v = GpVersion('4.3.0.0') - self.assertTrue(is_supported_version(v)) - - def test_is_supported_minor_version_upgrade(self): - v = GpVersion('4.3.8.0') - self.assertTrue(is_supported_version(v)) - - def test_is_supported_dev_build_version_upgrade(self): - v = GpVersion('4.3 build dev') - self.assertTrue(is_supported_version(v)) - - def test_is_supported_version_hotfix_upgrade(self): - v = GpVersion('4.3.7.3EC7') - self.assertTrue(is_supported_version(v)) - - def test_is_supported_version_major_version_unsupported_upgrade(self): - v = GpVersion('4.0.0.0') - with self.assertRaisesRegexp(Exception, "Greenplum Version '4.0.0.0 build dev' is not supported for upgrade"): - is_supported_version(v) - - def test_is_supported_version_awesome_build_upgrade(self): - v = GpVersion('4.3.3.4__AWESOME_BUILD__') - self.assertTrue(is_supported_version(v)) - - def test_is_supported_version_space_in_build_upgrade(self): - v = GpVersion('4.3.3.4 EC7') - self.assertTrue(is_supported_version(v)) - - def test_is_supported_version_major_version_downgrade(self): - v = GpVersion('4.3.0.0') - self.assertTrue(is_supported_version(v, False)) - - def test_is_supported_minor_version_downgrade(self): - v = GpVersion('4.3.8.0') - self.assertTrue(is_supported_version(v, False)) - - def test_is_supported_dev_build_version_downgrade(self): - v = GpVersion('4.3 build dev') - self.assertTrue(is_supported_version(v, False)) - - def test_is_supported_version_hotfix_downgrade(self): - v = GpVersion('4.3.7.3EC7') - self.assertTrue(is_supported_version(v, False)) - - def test_is_supported_version_major_version_unsupported_downgrade(self): - v = GpVersion('4.0.0.0') - with self.assertRaisesRegexp(Exception, "Greenplum Version '4.0.0.0 build dev' is not supported for downgrade"): - is_supported_version(v, False) - - def test_is_supported_version_awesome_build_downgrade(self): - v = GpVersion('4.3.3.4__AWESOME_BUILD__') - self.assertTrue(is_supported_version(v, False)) - - def test_is_supported_version_space_in_build_downgrade(self): - v = GpVersion('4.3.3.4 EC7') - self.assertTrue(is_supported_version(v)) - diff --git a/gpMgmt/bin/lib/gpenv.sh b/gpMgmt/bin/lib/gpenv.sh index bd31e252e5485c4c11443af09c38fb261b2afd06..7305ed085af86c6f16a73cf97c28af58b5c49549 100755 --- a/gpMgmt/bin/lib/gpenv.sh +++ b/gpMgmt/bin/lib/gpenv.sh @@ -1,9 +1,5 @@ #!/bin/bash # -# gpmigrator requires calling itself on individual hosts, but it is a python script -# so it needs certain environment variables to be setup before it can successfully -# execute itself. -# # This is a bash script (since /bin/bash is guaranteed) that sets up the environment # needed to execute a python script. It uses the path to itself as the basis from # which it determines GPHOME. diff --git a/gpMgmt/doc/gpbitmapreindex_help b/gpMgmt/doc/gpbitmapreindex_help index c81311eef64e19b0f8af4cc7a8fd6b7a6e7a9bf2..6054301c5daedca6e2142c59d2d92269eef3e24c 100644 --- a/gpMgmt/doc/gpbitmapreindex_help +++ b/gpMgmt/doc/gpbitmapreindex_help @@ -11,9 +11,6 @@ gpbitmapreindex -m { r | d | {l [-o ]} } [-h ] [-p ] [-n ] [-v] -gpmigrator --version - -gpmigrator --help | -? ***************************************************** diff --git a/gpMgmt/doc/gpmigrator_help b/gpMgmt/doc/gpmigrator_help deleted file mode 100644 index d5fbd979a201a1fa0dd335e3b69104d685846840..0000000000000000000000000000000000000000 --- a/gpMgmt/doc/gpmigrator_help +++ /dev/null @@ -1,150 +0,0 @@ -COMMAND NAME: gpmigrator - -Upgrades an existing Greenplum Database 4.2.x system -without mirrors to 4.3.x. - -Use gpmigrator_mirror to upgrade a 4.2.x system that -has mirrors. - -NOTE: Using gpmigrator on a system with mirrors causes - an error. - -***************************************************** -SYNOPSIS -***************************************************** - -gpmigrator - [-d ] - [-l ] [-q] - [--check-only] [--debug] [-R] - -gpmigrator --version | -v - -gpmigrator --help | -h - - -***************************************************** -PREREQUISITES -***************************************************** - -The following tasks should be performed prior to executing an upgrade: - -* Make sure you are logged in to the master host as the Greenplum Database - superuser (gpadmin). -* Install the Greenplum Database 4.3 binaries on all Greenplum hosts. -* Copy any custom modules you use into your 4.3 installation. Make sure - you obtain the latest version of any custom modules and that they are - compatible with Greenplum Database 4.3. -* Copy or preserve any additional folders or files (such as backup folders) - that you have added in the Greenplum data directories or $GPHOME directory. - Only files or folders strictly related to Greenplum Database operations are - preserved by the migration utility. -* (Optional) Run VACUUM on all databases, and remove old server log files - from pg_log in your master and segment data directories. This is not required, - but will reduce the size of Greenplum Database files to be backed up and migrated. -* Check for and recover any failed segments in your current Greenplum Database - system (gpstate, gprecoverseg). -* (Optional, but highly recommended) Backup your current databases (gpcrondump - or ZFS snapshots). If you find any issues when testing your upgraded system, - you can restore this backup. -* Remove the standby master from your system configuration (gpinitstandby -r). -* Do a clean shutdown of your current system (gpstop). -* Update your environment to source the 4.3 installation. -* Inform all database users of the upgrade and lockout time frame. Once the - upgrade is in process, users will not be allowed on the system until the - upgrade is complete. - - -***************************************************** -DESCRIPTION -***************************************************** - -The gpmigrator utility upgrades an existing Greenplum Database 4.2.x.x -system without mirrors to 4.3. This utility updates the system catalog -and internal version number, but not the actual software binaries. -During the migration process, all client connections to Greenplum -Database will be locked out. - - -***************************************************** -OPTIONS -***************************************************** - - - - Required. The absolute path to the current version of Greenplum - Database software you want to migrate away from. - - - - - Required. The absolute path to the new version of Greenplum Database - software you want to migrate to. - - --d - - Optional. The current master host data directory. If not specified, - the value set for $MASTER_DATA_DIRECTORY will be used. - - --l - - The directory to write the log file. Defaults to ~/gpAdminLogs. - - --q (quiet mode) - - Run in quiet mode. Command output is not displayed on the screen, but is - still written to the log file. - - --R (revert) - - In the event of an error during upgrade, reverts all changes made by gpmigrator. - - ---check-only - - Runs pre-migrate checks to verify that your database is healthy. - Checks include: - * Check catalog health - * Check that the Greenplum Database binaries on each segment match - those on the master - * Check for a minimum amount of free disk space - - NOTE: Performing a pre-migration check of your database should done - during a database maintenance period. If the utility detects catalog - errors, the utility stops the database. - - ---help | -h - Displays the online help. - - ---debug - Sets logging level to debug. - - ---version | -v - Displays the version of this utility. - - -***************************************************** -EXAMPLE -***************************************************** - -Upgrade to version 4.3.x from version 4.2.x (make sure you are using the -4.3 version of gpmigrator). This example upgrades to version 4.3.0.0 -from version 4.2.6.3: - -/usr/local/greenplum-db-4.3.0.0/bin/gpmigrator \ - /usr/local/greenplum-db-4.2.6.3 \ - /usr/local/greenplum-db-4.3.0.0 - - -***************************************************** -SEE ALSO -***************************************************** - -gpmigrator_mirror, gpstop, gpstate, gprecoverseg, gpcrondump \ No newline at end of file diff --git a/gpMgmt/doc/gpmigrator_mirror_help b/gpMgmt/doc/gpmigrator_mirror_help deleted file mode 100644 index 381923b2f14bc2a42739b25484a84ccd8cff9831..0000000000000000000000000000000000000000 --- a/gpMgmt/doc/gpmigrator_mirror_help +++ /dev/null @@ -1,148 +0,0 @@ -COMMAND NAME: gpmigrator_mirror - -Upgrades an existing Greenplum Database 4.2.x system -with mirrors to 4.3.x. - -Use gpmigrator to upgrade a 4.2.x system that does not -have mirrors. - -NOTE: Using gpmigrator_mirror on a system without mirrors - causes an error. - -***************************************************** -SYNOPSIS -***************************************************** - -gpmigrator_mirror - [-d ] - [-l ] [-q] - [--check-only] [--debug] - -gpmigrator_mirror --version | -v - -gpmigrator_mirror --help | -h - - -***************************************************** -PREREQUISITES -***************************************************** - -The following tasks should be performed prior to executing an upgrade: - -* Make sure you are logged in to the master host as the Greenplum Database - superuser (gpadmin). -* Install the Greenplum Database 4.3 binaries on all Greenplum hosts. -* Copy any custom modules you use into your 4.3 installation. Make sure - you obtain the latest version of any custom modules and that they are - compatible with Greenplum Database 4.3. -* Copy or preserve any additional folders or files (such as backup folders) - that you have added in the Greenplum data directories or $GPHOME directory. - Only files or folders strictly related to Greenplum Database operations are - preserved by the migration utility. -* (Optional) Run VACUUM on all databases, and remove old server log files - from pg_log in your master and segment data directories. This is not required, - but will reduce the size of Greenplum Database files to be backed up and migrated. -* Check for and recover any failed segments in your current Greenplum Database - system (gpstate, gprecoverseg). -* (Optional, but highly recommended) Backup your current databases (gpcrondump - or ZFS snapshots). If you find any issues when testing your upgraded system, - you can restore this backup. -* Remove the standby master from your system configuration (gpinitstandby -r). -* Do a clean shutdown of your current system (gpstop). -* Update your environment to source the 4.3 installation. -* Inform all database users of the upgrade and lockout time frame. Once the - upgrade is in process, users will not be allowed on the system until the - upgrade is complete. - - -***************************************************** -DESCRIPTION -***************************************************** - -The gpmigrator utility upgrades an existing Greenplum Database 4.2.x.x -system with mirrors to 4.3. This utility updates the system catalog -and internal version number, but not the actual software binaries. -During the migration process, all client connections to Greenplum -Database will be locked out. - - -***************************************************** -OPTIONS -***************************************************** - - - - Required. The absolute path to the current version of Greenplum - Database software you want to migrate away from. - - - - - Required. The absolute path to the new version of Greenplum Database - software you want to migrate to. - - --d - - Optional. The current master host data directory. If not specified, - the value set for $MASTER_DATA_DIRECTORY will be used. - - --l - - The directory to write the log file. Defaults to ~/gpAdminLogs. - - --q (quiet mode) - - Run in quiet mode. Command output is not displayed on the screen, but is - still written to the log file. - - ---check-only - - Runs pre-migrate checks to verify that your database is healthy. - Checks include: - * Check catalog health - * Check that the Greenplum Database binaries on each segment match - those on the master - * Check for a minium amount of free disk space - - NOTE: Performing a pre-migration check of your database should done - during a database maintenance period. If the utility detects catalog - errors, the utility stops the database. - - ---help | -h - - Displays the online help. - - ---debug - - Sets logging level to debug. - - ---version | -v - - Displays the version of this utility. - - -***************************************************** -EXAMPLE -***************************************************** - -Upgrade to version 4.3.x from version 4.2.x with mirrors (make sure you -are using the 4.3 version of gpmigrator_mirror). This example upgrades -to 4.3.0.0 from 4.2.6.3: - -/usr/local/greenplum-db-4.3.0.0/bin/gpmigrator_mirror \ - /usr/local/greenplum-db-4.2.6.3 \ - /usr/local/greenplum-db-4.3.0.0 - - -***************************************************** -SEE ALSO -***************************************************** - -gpmigrator, gpstop, gpstate, gprecoverseg, gpcrondump \ No newline at end of file diff --git a/gpMgmt/sbin/gpupgrademirror.py b/gpMgmt/sbin/gpupgrademirror.py deleted file mode 100755 index 94c0443600913c410764d66cb205ee2b9aa337c2..0000000000000000000000000000000000000000 --- a/gpMgmt/sbin/gpupgrademirror.py +++ /dev/null @@ -1,3970 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) Greenplum Inc 2010. All Rights Reserved. -# - -from gppylib.commands.base import * -from optparse import Option, OptionGroup, OptionParser, OptionValueError, SUPPRESS_USAGE -from time import strftime, sleep -import bisect -import copy -import datetime -import os -import pprint -import signal -import sys -import tempfile -import threading -import traceback - - - -sys.path.append(os.path.join(os.path.dirname(__file__), "../bin")) -sys.path.append(os.path.join(os.path.dirname(__file__), "../bin/lib")) - - -try: - from pysync import * - from gppylib.commands.unix import * - from gppylib.commands.gp import * - from gppylib.commands.pg import PgControlData - from gppylib.gparray import GpArray, get_host_interface - from gppylib.gpparseopts import OptParser, OptChecker - from gppylib.gplog import * - from gppylib.db import dbconn - from gppylib.db import catalog - from gppylib.userinput import * - from pygresql.pgdb import DatabaseError - from pygresql import pg - from gppylib.gpcoverage import GpCoverage -except ImportError, e: - sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) - -# -# Constants -# - -EXECNAME = os.path.split(__file__)[-1] -FULL_EXECNAME = os.path.abspath( __file__ ) - -DESCRIPTION = ("""Upgrades 3.3.x.x mirrors to 4.0 mirrors.""") - -_help = [""" TODO add help """] - -_usage = """ TODO add usage """ - -TEN_MEG = 10485760 -ONE_GIG = 128 * TEN_MEG -TEN_GIG = 1024 * TEN_MEG - -INFO_DIR = "gpupgrademirrorinfodir" - -PID_FILE = "gpupgrademirror.pid" - -GPUPGRADEMIRROR_TEMP_DIR = "gpupgrademirrortempdir" - -GPMIGRATOR_TEMP_DIR = "gpmigrator" - - -#----------------------------------------------------------- -def Shutdown(env, utility=False): - """ - This function will attempt to stop a database system. - """ - - try: - if utility: - cmd = 'gpstop -a -m -f' - else: - cmd = 'gpstop -a -f' - - logger.debug("Stopping cluster with env = %s" % str(env)) - pid = subprocess.Popen( args = cmd - , preexec_fn = os.setpgrp - , env = env - , shell = True - , stdout = subprocess.PIPE - , stderr = subprocess.PIPE - , close_fds = True - ) - - retcode = None - - """ KAS note to self. Could the output be so big that it uses up too much memory? """ - stdoutdata, stderrdata = pid.communicate() - logger.debug("gpstop for master output = \n" + stdoutdata) - if len(stderrdata) > 0: - logger.debug("gpstop for master error = " + stderrdata) - retcode = pid.returncode - - if retcode < 0: - raise Exception("gpstop terminated by signal."); - - if retcode == 1: - logger.warn('***************************************') - logger.warn('Warnings generated stopping cluster') - logger.warn('***************************************') - if retcode > 1: - logger.fatal('***************************************') - logger.fatal('gpstop failed with error code %d' % retcode) - logger.fatal('***************************************') - raise Exception('gpstop failed') - - except OSError, e: - logger.fatal(str(e)) - raise Exception('gpstop failed') - - -#----------------------------------------------------------- -def Startup(env, utility=False): - ''' - Starts up the specified database - ''' - - try: - if utility: - cmd = 'gpstart -a -m' - env['GPSTART_INTERNAL_MASTER_ONLY'] = '1' - else: - cmd = 'gpstart -a' - - logger.debug("Starting cluster where cmd = %s, with env = %s" % (str(cmd), str(env))) - pid = subprocess.Popen( cmd - , preexec_fn = os.setpgrp - , env = env - , shell = True - , stdout = subprocess.PIPE - , stderr = subprocess.PIPE - , close_fds = True - ) - - retcode = None - - """ - while retcode == None: - retcode = pid.wait() - """ - stdoutdata, stderrdata = pid.communicate() - logger.debug("gpstart for master output = \n" + stdoutdata) - if len(stderrdata) > 0: - logger.debug("gpstart for master error = " + stderrdata) - retcode = pid.returncode - - if retcode < 0: - raise Exception("Startup terminated by signal"); - - if retcode == 1: - logger.warn('***************************************') - logger.warn('Warnings generated starting cluster') - logger.warn('***************************************') - if retcode > 1: - logger.fatal('***************************************') - logger.fatal('Startup failed with error code %d' % retcode) - logger.fatal('***************************************') - raise Exception('Startup failed') - - except OSError, e: - logger.fatal(str(e)) - raise Exception('Startup failed') - - -#------------------------------------------------------------------------------- -def StartupPrimaries(gparray): - """ Startup all the primary segments in the cluster """ - - logging.debug("Attempting to start primaries.") - - """ get all the primary segments """ - allSegDbList = gparray.getSegDbList() - - if len(allSegDbList) / 2 > 64: - maxThreads = 64 - else: - maxThreads = len(allSegDbList) / 2 - primarySegCmdList = [] - - try: - pool = WorkerPool(numWorkers = maxThreads); - - """ get all the primary segments """ - allSegDbList = gparray.getSegDbList() - for db in allSegDbList: - if db.isSegmentPrimary() == True: - segStartCmd = SegmentStart( name = "start command primary segment %s" + str(db.getSegmentDbId()) - , gpdb = db - , numContentsInCluster = 123 # unused here, so just pass any number - , mirroringMode = "mirrorless" - , utilityMode = True - , ctxt = REMOTE - , remoteHost = db.getSegmentAddress() - , noWait = False - , timeout = 60 - ) - logging.debug("attempting to start primary with: " + str(segStartCmd)) - pool.addCommand(segStartCmd) - - # Wait for the segments to finish - try: - pool.join() - pool.haltWork() - except: - pool.haltWork() - pool.joinWorkers() - - failure = False - results = [] - for cmd in pool.getCompletedItems(): - r = cmd.get_results() - if not cmd.was_successful(): - logging.error("Unable to start segment: " + str(r)) - failure = True - - if failure: - raise Exception("There was an issue starting the primary segments.") - except Exception, e: - ShutdownPrimaries(gparray) - raise Exception("Could not start primary segments: " + str(e)) - - -#------------------------------------------------------------------------------- -def ShutdownPrimaries(gparray): - """ Shutdown all the primary segments in the cluster """ - - logging.debug("Attempting to stop primaries.") - - """ get all the primary segments """ - allSegDbList = gparray.getSegDbList() - - if len(allSegDbList) / 2 > 64: - maxThreads = 64 - else: - maxThreads = len(allSegDbList) / 2 - primarySegCmdList = [] - - try: - pool = WorkerPool(numWorkers = maxThreads); - - """ get all the primary segments """ - allSegDbList = gparray.getSegDbList() - for db in allSegDbList: - if db.isSegmentPrimary() == True: - segStopCmd = SegmentStop( name = "stop command primary segment %s" + str(db.getSegmentDbId()) - , dataDir = db.getSegmentDataDirectory() - , mode = "fast" - , nowait = False - , ctxt = REMOTE - , remoteHost = db.getSegmentAddress() - , timeout = 60 - ) - logging.debug("attempting to stop primary with: " + str(segStopCmd)) - pool.addCommand(segStopCmd) - - # Wait for the segments to finish - try: - pool.join() - pool.haltWork() - except: - pool.haltWork() - pool.joinWorkers() - - failure = False - results = [] - for cmd in pool.getCompletedItems(): - r = cmd.get_results() - if not cmd.was_successful(): - logging.error("Unable to stop segment: " + str(r)) - failure = True - - if failure: - raise Exception("There was an issue stopping the primary segments.") - except Exception, e: - raise Exception("Could not stop primary segments: " + str(e)) - - - -#------------------------------------------------------------------------------- -def SetupEnv(gphome): - ''' - Sets up environment variables for Greenplum Administration - ''' - - if len(gphome) > 2 and gphome[0] == '"' and gphome[len(gphome) - 1] == '"': - gphome = gphome[1:len(gphome) - 1] - - thePath = '/usr/kerberos/bin:/usr/sfw/bin:/opt/sfw/bin' - thePath += ':/usr/local/bin:/bin:/usr/bin:/sbin:/usr/sbin:/usr/ucb' - thePath += ':/sw/bin' - user = os.environ.get('USER') or os.environ.get('LOGNAME') - home = os.environ.get('HOME') - lpath = os.environ.get('LD_LIBRARY_PATH') - dypath = os.environ.get('DYLD_LIBRARY_PATH') - - # Add $GPHOME/bin to the path for this environment - path = '%s/bin:%s/ext/python/bin:%s' % (gphome, gphome, thePath) - - if lpath: - lpath = '%s/lib:%s/ext/python/lib:%s' % (gphome, gphome, lpath) - else: - lpath = '%s/lib:%s/ext/python/lib' % (gphome, gphome) - if dypath: - dypath = '%s/lib:%s/ext/python/lib:%s' % (gphome, gphome, dypath) - else: - dypath = '%s/lib:%s/ext/python/lib' % (gphome, gphome) - - env = {} - env['HOME'] = home - env['USER'] = user - env['LOGNAME'] = user - env['GPHOME'] = gphome - env['PATH'] = path - env['LD_LIBRARY_PATH'] = lpath - env['DYLD_LIBRARY_PATH'] = dypath - env['PYTHONPATH'] = os.path.join(gphome, 'lib', 'python') - env['PYTHONHOME'] = os.path.join(gphome, 'ext', 'python') - env['MASTER_DATA_DIRECTORY'] = get_masterdatadir() - return env - - -#------------------------------------------------------------------------------- -def findRelFileDotNodes(relFileNode, sortedList): - """ - This function will take a relfilenode without a dot '.' suffix, and find all of its - dot suffix files in a sorted list (sortedList). Just the suffixes are returned. - relFileNode and sortedList are assumed to contain full path lists. - """ - retValue = [] - relFileNodeWithDot = relFileNode + "." - index = bisect.bisect_left(sortedList, relFileNode) - index = index + 1 - - while index < len(sortedList) and sortedList[index].startswith(relFileNodeWithDot): - fullName = sortedList[index] - dotIndex = len(relFileNodeWithDot) - suffix = fullName[dotIndex:] - retValue.append(suffix) - index = index + 1 - - return retValue - - -#------------------------------------------------------------------------------- -def sshBusy(cmd): - """ - This function will check the results of a Command to see if ssh was too busy. - It will return False if the command completed, successfully or not, - and a retry is not possible or necessary. - """ - retValue = False - results = cmd.get_results() - resultStr = results.printResult() - if results.rc != 0: - if resultStr.find("ssh_exchange_identification: Connection closed by remote host") != -1: - retValue = True - else: - retValue = False - else: - retValue = False - return retValue - - - -#------------------------------------------------------------------------------- -def runAndCheckCommandComplete(cmd): - """ - This function will run a Command and return False if ssh was too busy. - It will return True if the command completed, successfully or not, - if ssh wasn't busy. - """ - retValue = True - cmd.run(validateAfter = False) - if sshBusy(cmd) == True: - """ Couldn't make the connection. put in a delay, and return""" - self.logger.debug("gpupgrademirror ssh is busy... need to retry the command: " + str(cmd)) - time.sleep(1) - retValue = False - else: - retValue = True - return retValue - - -#------------------------------------------------------------------------------- - -def parseargs(): - parser = OptParser( option_class = OptChecker - , description = ' '.join(DESCRIPTION.split()) - , version = '%prog version $Revision: #12 $' - ) - parser.setHelp(_help) - parser.set_usage('%prog ' + _usage) - parser.remove_option('-h') - - parser.add_option('-r', '--rollback', action='store_true', - help='rollback failed expansion setup.', default=False) - parser.add_option('-c', '--continue-upgrade', action='store_true', - help='continue expansion.', default=False) - parser.add_option('-m', '--mode', default='S', - help='Valid values are S for safe mode and U for un-safe mode') - parser.add_option('-P', '--phase2', action='store_true', - help='phase 2 for upgrade of mirrors.', default=False) - parser.add_option('-v','--verbose', action='store_true', - help='debug output.', default=False) - parser.add_option('-h', '-?', '--help', action='help', - help='show this help message and exit.', default=False) - parser.add_option('-i', '--ids', default='', - help='comma separated list of dbids for use in upgrade') - parser.add_option('-g', '--gphome', default=get_gphome(), - help='location of gphome for 3.3.x system') - parser.add_option('-t', '--temp-dir', default='', - help='location for temp upgrade mirror data') - parser.add_option('-s', '--speed-network', default='', - help='speed of NIC on he network') - parser.add_option('--usage', action="briefhelp") - - - """ - Parse the command line arguments - """ - (options, args) = parser.parse_args() - - if len(args) > 0: - logger.error('Unknown argument %s' % args[0]) - parser.exit() - - """ - Make a directory to hold information about this run. We will need this information in the - event we run again in rollback or continue mode. We use temporary directory gpmigrator create - under the master data directory, or the one it creates under the segment data directory (i.e. - the directory passed in via the -t option. - """ - try: - if len(options.temp_dir) > 0: - options.info_data_directory = options.temp_dir - else: - mDir = os.path.split(get_masterdatadir()) - dirPrefix = mDir[0] - if len(dirPrefix) == 1 and str(dirPrefix) == str('/'): - """ Special case where the directory is '/'. """ - dirPrefix = "" - dirSuffix = mDir[1] - options.info_data_directory = dirPrefix + "/" + GPMIGRATOR_TEMP_DIR + "/" + dirSuffix + INFO_DIR - except GpError, msg: - logger.error(msg) - parser.exit() - - if not os.path.exists(options.info_data_directory): - MakeDirectoryWithMode.local( name = 'gpupgrademirror make info dir: %s' % options.info_data_directory - , directory = options.info_data_directory - , mode = 700 - ) - return options, args - -#------------------------------------------------------------------------------- -def sig_handler(sig, arg): - print "Handling signal..." - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGHUP, signal.SIG_DFL) - - # raise sig - os.kill(os.getpid(), sig) - - -#------------------------------------------------------------------------------- -def create_pid_file(info_data_directory): - """Creates gpupgradmirror pid file""" - try: - fp = open(info_data_directory + '/' + PID_FILE, 'w') - fp.write(str(os.getpid())) - except IOError: - raise - finally: - if fp: fp.close() - - -#------------------------------------------------------------------------------- -def remove_pid_file(info_data_directory): - """Removes upgrademirror pid file""" - try: - os.unlink(info_data_directory + '/' + PID_FILE) - except: - pass - - -#------------------------------------------------------------------------------- -def is_gpupgrademirror_running(info_data_directory): - """Checks if there is another instance of gpupgrademirror running""" - is_running = False - try: - fp = open(info_data_directory + '/' + PID_FILE, 'r') - pid = int(fp.readline().strip()) - fp.close() - is_running = check_pid(pid) - except IOError: - pass - except Exception, msg: - raise - - return is_running - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class InvalidStatusError(Exception): pass - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class ValidationError(Exception): pass - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class GPHostCache: - """ This class represents the information stored in the .gphostcache file """ - - def __init__(self, url): - self.hostcache = {} - self.readfile() - - #------------------------------------------------------------------------------- - def __str__(self): - tempStr = "self.hostcache = " + str(self.hostcache) + '\n' - return tempStr - - #------------------------------------------------------------------------------- - def readfile(self): - if len(options.ids) == 0: - FILEDIR = os.path.expanduser("~") - else: - FILEDIR = options.info_data_directory - self.FILENAME = ".gphostcache" - self.CACHEFILE = FILEDIR + "/" + self.FILENAME - - try: - file = None - file = open(self.CACHEFILE, 'r') - - for line in file: - (interface,hostname) = line.split(':') - self.hostcache[interface.strip()] = hostname.strip() - file.close() - except IOError, ioe: - logger.error("Can not read host cache file %s. Exception: %s" % (self.CACHEFILE, str(ioe))) - raise Exception("Unable to read host cache file: %s" % self.CACHEFILE) - finally: - if file != None: - file.close() - - #------------------------------------------------------------------------------- - def getInterfaces(self, hostName): - retList = [] - - for interface in self.hostcache: - if self.hostcache[interface] == hostName: - retList.append(interface) - return retList - - #------------------------------------------------------------------------------- - def getHost(self, interface): - return self.hostcache[interface] - - #------------------------------------------------------------------------------- - def getHostList(self): - retList = [] - - for (interface, host) in self.hostcache: - if host in retList: - continue - else: - retList.append(host) - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class RelFileNodes(): - """ This class represents a complete set of tables and dependencies for a given segment. """ - """ WARNING: This class isn't currently used, so it is not tested. Beware....""" - - def __init__(self, logger, mirrorInfo, database_list): - self.info_data_directory = mirrorInfo.mirror_dbid - self.mirrorInfo = mirrorInfo - self.databaseList = database_list - self.tableRelFileList = [] - - #------------------------------------------------------------------------------- - def __str__(self): - retValue = '' - - for tableFileList in self.tableRelFileList: - tempStr = ':'.join(tableFileList) - retValue = retValue + tempStr + '\n' - return retValue - -# #------------------------------------------------------------------------------- - def __repr__(self): - return self.__str__() - - #------------------------------------------------------------------------------- - def setup(self): - - """ Connect to each database on the segment, and get its list of tables and table dependencies. """ - for db in self.databaseList: - if str(db.databaseName) == str("template0"): - continue - segmentURL = dbconn.DbURL( hostname = self.mirrorInfo.primary_host - , port = self.mirrorInfo.primary_host_port - , dbname = db.databaseName - ) - conn = dbconn.connect( dburl = segmentURL - , utility = True - ) - - tablesCursor = dbconn.execSQL(conn, TableRelFileNodesRow.query()) - aTable = [] - for row in tablesCursor: - aTable = [] - """ KAS need to added code to look for *.1, *.2 files for this table """ - tableRelRow = TableRelFileNodesRow(row) - aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.table_file)) - if tableRelRow.toast_file != None: - aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.toast_file)) - if tableRelRow.toast_index_file != None: - aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.toast_index_file)) - if tableRelRow.ao_file != None: - aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.ao_file)) - if tableRelRow.ao_index_file != None: - aTable.append(str(db.databaseDirectory) + '/' + str(tableRelRow.ao_index_file)) - indexesCursor = dbconn.execSQL(conn, TableIndexRow.query(str(tableRelRow.table_file_oid))) - """ KAS note to self. Do I need to worry about *.1, *.2... files? """ - for indexRow in indexesCursor: - indexEntry = TableIndexRow(indexRow) - aTable.append(str(db.databaseDirectory) + '/' + str(indexEntry.relfilenode)) - self.tableRelFileList.append(aTable) - conn.close() - - #------------------------------------------------------------------------------- - def _read_rel_file_node_file(self): - """Reads in an existing relfile node file""" - self.logger.debug("Trying to read in a pre-existing relfile node file" + str(self._rel_file_node_filename)) - try: - self._fp = open(self._rel_file_node_filename, 'r') - self._fp.seek(0) - - for line in self._fp: - tableFileList = line.split(':') - tableRelFileList.append(tableFileList) - self._fp.close() - - except IOErrormm, e: - logger.error('Unable to read relfile node file: ' + str(e)) - logger.error('gpupgradmirror exiting') - exit("The gpupgrademirror log information can be found in " + str(get_logfile()) + "\n") - - #------------------------------------------------------------------------------- - def create_rel_file_node_file(self): - """Creates a new gpupgrademirror relfile node file""" - try: - self._fp = open(self._rel_file_node_filename, 'w') - except IOError, e: - logging.error('Unable to create relfile node file %s.' % self._rel_file_node_filename) - raise e - - #------------------------------------------------------------------------------- - def remove_rel_file_node_file(self): - """Closes and removes the gpupgrademirror relfile node file""" - if self._fp: - self._fp.close() - self._fp = None - if os.path.exists(self._rel_file_node_filename): - os.unlink(self._rel_file_node_filename) - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class TableRelFileNodesRow: - - def __init__(self, row): - self.table_file_oid = row[0] - self.table_file = row[1] - self.toast_file = row[2] - self.toast_index_file = row[3] - self.ao_file = row[4] - self.ao_index_file = row[5] - - #------------------------------------------------------------------------------- - @staticmethod - def query(): - exeSql = """SELECT - t1.oid AS table_file_oid - , t1.relfilenode AS table_file - , t2.relfilenode AS toast_file - , t3.relfilenode AS toast_index_file - , t4.relfilenode AS ao_file - , t5.relfilenode AS ao_index_file - FROM (((pg_class AS t1 left outer join pg_class AS t2 ON t2.oid = t1.reltoastrelid) - left outer join pg_class AS t3 ON t3.oid = t2.reltoastidxid) - left outer join pg_class AS t4 ON t4.oid = t1.relaosegrelid) - left outer join pg_class AS t5 ON t5.oid = t4.relaosegidxid - WHERE t1.relkind = 'r' or t1.relkind = 'x' - ORDER BY t1.relfilenode - """ - - return exeSql - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- - -class TableIndexRow: - - def __init__(self, row): - self.relfilenode = row[0] - - -#------------------------------------------------------------------------------- - @staticmethod - def query(table_oid): - exeSql = """ SELECT t2.relfilenode - FROM pg_catalog.pg_index AS t1 - INNER JOIN - pg_catalog.pg_class AS t2 - ON t1.indexrelid = t2.oid - WHERE t1.indrelid = """ + str(table_oid) - - return exeSql - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class MirrorNICRow: - - def __init__(self, row): - self.nic = row[0] - self.dbid = row[1] - - #------------------------------------------------------------------------------- - @staticmethod - def query(): - if options.phase2 == False: - exeStr = ''' SELECT hostname as nicaddress, dbid - FROM pg_catalog.gp_configuration - WHERE isprimary = 'f' - ORDER BY hostname ''' - else: - raise Exception('Internal Error. MirrorNICRow.qurey only for 3.x systems.') - return exeStr - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class MirrorHostsRow: - - def __init__(self, row): - self.hostname = row[0] - self.dbid = row[1] - - #------------------------------------------------------------------------------- - @staticmethod - def query(): - if options.phase2 == False: - raise Exception('Internal Error. MirrorHostsRow.qurey only for 3.3.x systems.') - else: - exeStr = ''' SELECT hostname, dbid - FROM pg_catalog.gp_segment_configuration - WHERE role = 'm' - ORDER BY hostname ''' - return exeStr - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class MirrorHostInfoRow: - - def __init__(self, row): - self.mirror_host = row[0] - self.mirror_host_port = row[1] - self.primary_host = row[2] - self.primary_host_port = row[3] - self.primary_data_directory = row[4] - self.mirror_data_directory = row[5] - self.mirror_dbid = row[6] - self.primary_dbid = row[7] - if options.phase2 == True: - self.nic_address = row[8] - - #------------------------------------------------------------------------------- - def __repr__(self): - return self.__str__() - - #------------------------------------------------------------------------------- - def __str__(self): - tempStr = 'mirror_host=' + str(self.mirror_host) + '\n' \ - + 'mirror_host_port=' + str(self.mirror_host_port) + '\n' \ - + 'primary_host=' + str(self.primary_host) + '\n' \ - + 'primary_host_port=' + str(self.primary_host_port) + '\n' \ - + 'primary_data_directory=' + str(self.primary_data_directory) + '\n' \ - + 'mirror_data_directory=' + str(self.mirror_data_directory) + '\n' \ - + 'mirror_dbid=' + str(self.mirror_dbid) + '\n' \ - + 'primary_dbid=' + str(self.primary_dbid) + '\n' - if options.phase2 == True: - tempStr = tempStr + 'nic_address=' + str(self.nic_address) + '\n' - return tempStr - - #------------------------------------------------------------------------------- - @staticmethod - def readListFromFile(nvFileName): - """ Static method to convert file with information in the __str__ method format """ - """ for this object into a list of rows. The rows are lists of column values. """ - retList = [] - - logger.debug("Trying to read in a pre-existing gpupgrademirror MirrorHostInfoRow file. " + str(nvFileName)) - - try: - fp = open(nvFileName,'r') - fp.seek(0) - - if options.phase2 == True: - colsPerRow = 9 - else: - colsPerRow = 8 - index = 0 - row = [] - for line in fp: - (name, value) = line.strip().rsplit('=', 1) - row.append(value) - index = index + 1 - if index == colsPerRow: - retList.append(MirrorHostInfoRow(row)) - row = [] - index = 0 - return retList - except IOError: - raise - finally: - try: - if fp != None: - fp.close() - except: - pass - - - #------------------------------------------------------------------------------- - @staticmethod - def writeListToFile(nvFileName, listOfRows): - """ Static method to write out a list of rows in the __str__ method format """ - - logger.debug("Trying to write a list of MirrorHostInfoRow to a file. " + str(nvFileName)) - - try: - fp = open(nvFileName, 'w') - fp.seek(0) - for row in listOfRows: - fp.write(str(row)) - fp.flush() - fp.close() - except IOError: - raise - - #------------------------------------------------------------------------------- - @staticmethod - def query(): - if options.phase2 == False: - exeStr = '''SELECT t2.hostname as mirror_host - , t2.port as mirror_host_port - , t1.hostname as primary_host - , t1.port as primary_host_port - , t1.datadir as primary_data_directory - , t2.datadir as mirror_data_directory - , t2.dbid as mirror_dbid - , t1.dbid as primary_dbid - FROM pg_catalog.gp_configuration as t1 - INNER JOIN - pg_catalog.gp_configuration as t2 - ON t1.content = t2.content - AND t2.hostname = '%s' - AND t1.content >= 0 - AND t1.isprimary = 't' - AND t2.isprimary = 'f' - ORDER BY primary_host ''' - else: - exeStr = """SELECT t2.hostname as mirror_host - , t2.port as mirror_host_port - , t1.hostname as primary_host - , t1.port as primary_host_port - , t3.fselocation as primary_data_directory - , t4.fselocation as mirror_data_directory - , t2.dbid as mirror_dbid - , t1.dbid as primary_dbid - , t2.address as nic_address - FROM ((pg_catalog.gp_segment_configuration as t1 - INNER JOIN - pg_catalog.gp_segment_configuration as t2 - ON t1.content = t2.content - AND t2.hostname = '%s' - AND t1.content >= 0 - AND t1.role = 'p' - AND t2.role = 'm' - ) INNER JOIN pg_catalog.pg_filespace_entry as t3 - ON t3.fsedbid = t1.dbid - ) INNER JOIN pg_catalog.pg_filespace_entry as t4 - ON t4.fsedbid = t2.dbid - ORDER BY primary_host""" - - return exeStr - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class DatabaseRow: - - def __init__(self, row): - self.databaseDirectory = row[0] - self.databaseName = row[1] - - @staticmethod - def query(): - return '''SELECT oid, datname - FROM pg_database - ORDER BY oid ''' - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class InformationSchemaTablesRow: - - def __init__(self, row): - self.filename = row[0] - - @staticmethod - def query(): - return '''SELECT c.relfilenode - FROM pg_class c JOIN pg_namespace n ON (c.relnamespace = n.oid) - WHERE n.nspname = 'information_schema' AND (relkind = 'r' OR relkind = 'i')''' - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class NonCatalogToastTablesRow: - - def __init__(self, row): - self.filename = row[0] - - @staticmethod - def query(): - return '''SELECT relfilenode - FROM pg_class - WHERE relnamespace = 11 - AND relisshared = 'f' - AND (relkind = 'r' OR relkind = 'i') - AND relstorage = 'h' - AND reltoastrelid = 0 - ORDER BY relfilenode ''' - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class CatalogToastTablesRow: - - def __init__(self, row): - self.filename = row[0] - self.toastfilename = row[1] - self.toastfileindex = row[2] - - @staticmethod - def query(): - return """SELECT t1.relfilenode, t2.relfilenode as toast_filenode, t3.relfilenode as toast_filenode_index - FROM pg_class as t1, pg_class as t2, pg_class as t3 - WHERE t1.relnamespace = 11 - AND t1.relisshared = 'f' - AND t1.reltoastrelid = t2.oid - AND t2.reltoastidxid = t3.oid - ORDER BY t1.relfilenode""" - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class Phase1and2MirrorInfo: - """ - This class contains information on all the mirror nodes in the cluster. - The way it gathers information depends on if we are in phase 1 or phase 2. - - If this process is a minion (i.e. it is a sub process of the overall gpupgrademirror), - it will just get those mirrors have have dbids in the options.ids list. - """ - - def __init__(self, url = None): - self.dbUrl = url - self.mirrorInfoFileUnqualified = 'gpupgrademirror.mirrorinfo' - self.mirrorInfoFile = options.info_data_directory + '/' + self.mirrorInfoFileUnqualified - self.mirrorNodeList = [] - self.allMirrorInfoList = [] - self.gpHostCacheInfo = GPHostCache(url = url) - conn = None - - if (options.rollback == True or options.continue_upgrade == True) and options.phase2 == False: - """ We are in a rollback or continue state in Phase 1. """ - """ Retrieve configuration information from previously saved file """ - self.allMirrorInfoList = MirrorHostInfoRow.readListFromFile(Phase1and2MirrorInfo.getMirrorInfoFile()) - - """ for each NIC in the list of mirror NICs, make a list of distinct mirror nodes """ - for mirrorInfo in self.allMirrorInfoList: - mirrorHostName = self.gpHostCacheInfo.getHost(mirrorInfo.mirror_host) - if mirrorHostName in self.mirrorNodeList: - pass - elif len(options.ids) == 0 or mirrorInfo.mirror_dbid in options.ids: - self.mirrorNodeList.append(mirrorHostName) - else: - """ We are in normal Phase1 or normal Phase 2 (no rollback or continue) """ - conn = dbconn.connect( dburl = dbUrl - , utility = True - ) - - ''' Get a list of mirror nodes ''' - if options.phase2 == True: - mirrorNodeCursor = dbconn.execSQL(conn, MirrorHostsRow.query()) - for row in mirrorNodeCursor: - mirror = MirrorHostsRow(row) - if len(options.ids) == 0 or str(mirror.dbid) in options.ids: - if mirror.hostname not in self.mirrorNodeList: - self.mirrorNodeList.append(mirror.hostname) - - ''' Get info on mirror segments for this host''' - mirrorSegInfoCursor = dbconn.execSQL(conn, MirrorHostInfoRow.query() % mirror.hostname) - for mirrorInfoRow in mirrorSegInfoCursor: - mirrorInfo = MirrorHostInfoRow(mirrorInfoRow) - if len(options.ids) == 0 or str(mirrorInfo.mirror_dbid) in options.ids: - self.allMirrorInfoList.append(mirrorInfo) - else: - ''' We are in phase1 ''' - ''' Get a list of all mirror NICs ''' - mirrorNicCursor = dbconn.execSQL(conn, MirrorNICRow.query()) - - ''' Make a list of distinct nodes associated with the mirror NICs ''' - for row in mirrorNicCursor: - nicRow = MirrorNICRow(row) - host = self.gpHostCacheInfo.getHost(nicRow.nic) - if host in self.mirrorNodeList: - pass - else: - if len(options.ids) == 0 or str(nicRow.dbid) in options.ids: - self.mirrorNodeList.append(host) - - for nodeName in self.mirrorNodeList: - ''' for each node, use its NICs to find its segments ''' - nicList = self.gpHostCacheInfo.getInterfaces(nodeName) - ''' Get info on mirror segments for this host''' - for nic in nicList: - mirrorSegInfoCursor = dbconn.execSQL(conn, MirrorHostInfoRow.query() % nic) - for mirrorInfoRow in mirrorSegInfoCursor: - mirrorInfo = MirrorHostInfoRow(mirrorInfoRow) - if len(options.ids) == 0 or str(mirrorInfo.mirror_dbid) in options.ids: - self.allMirrorInfoList.append(mirrorInfo) - - if conn != None: - conn.close() - - MirrorHostInfoRow.writeListToFile(Phase1and2MirrorInfo.getMirrorInfoFile(), self.allMirrorInfoList) - - #------------------------------------------------------------------------------- - def __str__(self): - tempStr = ' dbUrl = ' + str(self.dbUrl) + '\n' \ - + ' mirrorInfoFile = ' + str(self.mirrorInfoFile) + '\n' \ - + ' self.mirrorNodeList = \n' + str(self.mirrorNodeList) + '\n' \ - + ' self.allMirrorInfoList = ' + str(self.allMirrorInfoList) + '\n' \ - + ' self.gpHostCacheInfo = ' + str(self.gpHostCacheInfo) + '\n' - return tempStr - - #------------------------------------------------------------------------------- - def getAllMirrorInfoList(self): - return self.allMirrorInfoList - - #------------------------------------------------------------------------------- - def getMirrorNodeList(self): - return self.mirrorNodeList - - - #------------------------------------------------------------------------------- - def getMirrorInfo(self, mirrorName): - retList = [] - - if options.phase2 == True: - for mirrorSeg in self.allMirrorInfoList: - if mirrorSeg.mirror_host == mirrorName: - retList.append(mirrorSeg) - else: - nicNameList = self.gpHostCacheInfo.getInterfaces(mirrorName) - for nicName in nicNameList: - for mirrorSeg in self.allMirrorInfoList: - if nicName == mirrorSeg.mirror_host: - retList.append(mirrorSeg) - - return retList - - #------------------------------------------------------------------------------- - def getMirrorInfoFromMirrordbid(self, mirrordbid): - retValue = None - - for mirrorSeg in self.allMirrorInfoList: - if str(mirrorSeg.mirror_dbid) == str(mirrordbid): - retValue = mirrorSeg - break - return retValue - - #------------------------------------------------------------------------------- - def getInterfaceList(self, node_name): - retList = [] - - if options.phase2 == True: - for mirrorSeg in self.allMirrorInfoList: - if mirrorSeg.mirror_host == node_name: - retList.append(mirrorSeg.nic_address) - else: - allHostCacheInterfaces = self.gpHostCacheInfo.getInterfaces(node_name) - """ The host cache contains entries that are not actual nics, so """ - """ we must only return the real nics that are referenced from """ - """ from the database cluster. """ - for interface in allHostCacheInterfaces: - for seg in self.allMirrorInfoList: - if seg.mirror_host not in retList and interface == seg.mirror_host: - retList.append(interface) - - return retList - - #------------------------------------------------------------------------------- - @staticmethod - def getMirrorInfoFile(): - return options.info_data_directory + '/' + 'gpupgrademirror.mirrorinfo' - - - #------------------------------------------------------------------------------- - @staticmethod - def removeMirrorInfoFile(): - try: - os.remove(Phase1and2MirrorInfo.getMirrorInfoFile()) - except: - pass - - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class NetworkSpeed(Command): - """ - This class will attempt to get the network speed on the cluster. - It assumes the the 2 nodes being tested are different nodes, and - that no other traffic is present between the two nodes. - """ - - def __init__(self,name, host1, host2, tempDir1, tempDir2, ctxt=LOCAL, remoteHost=None): - self.host1 = host1 - self.host2 = host2 - self.tempDir1 = tempDir1 - self.tempDir2 = tempDir2 - self.remoteHost = remoteHost - cmdStr="gpcheckperf -h %s -h %s -r N -d %s -d %s" % (host1, host2, tempDir1, tempDir2) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - def getSpeed(self): - """ - Return the network speed in megabytes per second. - Don't ever fail, just say it is 0 if something goes wrong. - """ - retValue = 0 - results = "" - try: - results = self.get_results() - resultStr = results.printResult() - index = resultStr.find("median = ") - substring = resultStr[index:] - substringList = substring.split(" ") - retValue = int(float(substringList[2])) - except Exception, e: - logger.warning("There was a problem getting network speed: results = %s. exception = %s" % (str(results), str(e))) - pass - return retValue - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class MakeDirectoryWithMode(Command): - - def __init__(self,name,directory,ctxt=LOCAL,remoteHost=None,mode=None): - self.directory=directory - if mode == None: - pMode = '' - else: - pMode = "-m " + str(mode) - cmdStr="%s -p %s %s" % (findCmdInPath('mkdir'),pMode,directory) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - @staticmethod - def local(name,directory,mode=None): - mkdirCmd=MakeDirectoryWithMode(name,directory,mode = mode) - mkdirCmd.run(validateAfter=True) - - @staticmethod - def remote(name,remote_host,directory,mode=None): - mkdirCmd=MakeDirectoryWithMode(name,directory,ctxt=REMOTE,remoteHost=remote_host,mode=mode) - mkdirCmd.run(validateAfter=True) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class CreateDirIfNecessaryWithMode(Command): - def __init__(self,name,directory,ctxt=LOCAL,remoteHost=None,mode=None): - if mode == None: - cmdStr="""python -c "import os; os.mkdir('%s')" """ % (directory) - else: - if len(str(mode)) != 4: - raise Exception("Internal error: os.mkdir mode parameter must have length of 4") - pythonPg = """\"\"\"import os -if os.path.exists('%s') == False: - os.mkdir('%s', %s)\"\"\" -""" % (directory, directory, str(mode)) - cmdStr = """python -c %s""" % pythonPg - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - @staticmethod - def remote(name,remote_host,directory,mode=None): - cmd=CreateDirIfNecessaryWithMode(name,directory,ctxt=REMOTE,remoteHost=remote_host,mode=mode) - cmd.run(validateAfter=False) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class CopyToLocal(Command): - def __init__(self, name, srcHost, srcDirectory, dstDirectory, ctxt=LOCAL): - self.srcDirectory=srcDirectory - self.srcHost=srcHost - self.dstDirectory=dstDirectory - cmdStr="%s -r %s:%s %s" % (findCmdInPath('scp'), srcHost, srcDirectory, dstDirectory) - Command.__init__(self, name, cmdStr, ctxt=LOCAL) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class RemoteCopyPreserve(Command): - def __init__(self,name,srcDirectory,dstHost,dstDirectory,ctxt=LOCAL,remoteHost=None): - self.srcDirectory=srcDirectory - self.dstHost=dstHost - self.dstDirectory=dstDirectory - cmdStr="%s -rp %s %s:%s" % (findCmdInPath('scp'),srcDirectory,dstHost,dstDirectory) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class GPumDiskFree(Command): - """ This method checks for available space in a directory """ - """ The same method, without the GPum prefix is in gppylib, """ - """ but it doesn't work correctly in 3.3.x. """ - - def __init__(self,name,directory,ctxt=LOCAL,remoteHost=None): - self.directory=directory - dfCommand = SYSTEM.getDiskFreeCmd() - if options.phase2 == False and isinstance(SYSTEM, SolarisPlatform): - """ This is a work around for a bug in 3.3.x (see MPP-6647) """ - dfCommand = dfCommand + 'k' - if options.phase2 == False and isinstance(SYSTEM, LinuxPlatform): - """ This is a work around for a bug in 3.3.x (see MPP-6647) """ - dfCommand = dfCommand + 'Pk' - cmdStr="%s %s" % (dfCommand,directory) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - @staticmethod - def get_size(name,remote_host,directory): - dfCmd=GPumDiskFree( name = name - , directory = directory - , ctxt = REMOTE - , remoteHost = remote_host - ) - dfCmd.run(validateAfter=True) - return dfCmd.get_bytes_free() - - - def get_bytes_free(self): - '''expected output of the form: - Filesystem 512-blocks Used Available Capacity Mounted on - /dev/disk0s2 194699744 158681544 35506200 82% / - ''' - rawIn=self.results.stdout.split('\n')[1] - bytesFree=int(rawIn.split()[3])*1024 - return bytesFree - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class CreateHardLink(Command): - """ This class create a hard link or links. """ - - def __init__(self, name, srcFile, hardLink, ctxt = LOCAL, remoteHost = None, argsFile = None): - if argsFile == None: - self.srcFile = srcFile - self.hardLink = hardLink - cmdStr="%s %s %s" % (findCmdInPath('ln'), srcFile, hardLink) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - else: - cmdStr="%s %s | %s -n 2 %s" % (findCmdInPath("cat"), argsFile, findCmdInPath("xargs"), findCmdInPath('ln')) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class GPumMoveDirectory(Command): - """ This class moves a local directory.""" - - def __init__(self,name,srcDirectory,dstDirectory,ctxt=LOCAL,remoteHost=None): - self.srcDirectory=srcDirectory - self.dstDirectory=dstDirectory - cmdStr="%s -f %s %s" % (findCmdInPath('mv'),srcDirectory,dstDirectory) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class GPumMoveDirectoryContents(Command): - """ This class moves the contents of a local directory.""" - - def __init__(self,name,srcDirectory,dstDirectory,ctxt=LOCAL,remoteHost=None): - self.srcDirectory = srcDirectory - self.srcDirectoryFiles = self.srcDirectory + "." + 'dirfilelist' - self.dstDirectory = dstDirectory - ls = findCmdInPath("ls") - cat = findCmdInPath("cat") - xargs = findCmdInPath("xargs") - mv = findCmdInPath("mv") - cmdStr = "%s -1 %s > %s" % (ls, self.srcDirectory, self.srcDirectoryFiles) - cmdStr = cmdStr + ";%s %s" % (cat, self.srcDirectoryFiles) - cmdStr = cmdStr + " | %s -I xxx %s %s/xxx %s" % (xargs, mv, self.srcDirectory, self.dstDirectory) - cmdStr = cmdStr + "; rm %s" % (self.srcDirectoryFiles) - """ - ls -1 temp1 > temp1.list;cat temp1.list | xargs -I xxx mv temp1/xxx temp2 - cmdStr = "%s -1 %s > %s;%s %s | %s -I xxx %s %s/xxx %s" - cmdStr="%s -f %s %s" % (findCmdInPath('mv'),srcDirectory,dstDirectory) - """ - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class RemoveFileDirectoryFromList(Command): - """ - This class will remove a list of files from a given locaiton. - """ - def __init__(self,name,fileListLocation,ctxt=LOCAL,remoteHost=None): - self.fileListLocation = fileListLocation - self.ctxt = ctxt - self.remoteHost = remoteHost - cmdStr="%s %s | %s %s -rf" % \ - ( findCmdInPath("cat") - , fileListLocation - , findCmdInPath("xargs") - , findCmdInPath('rm') - ) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - def setupListFile(self, tempDir, theList): - fd, fdName = tempfile.mkstemp(prefix='tmpRFDFL', dir=tempDir) - - fd = open(fdName, "w") - for row in theList: - fd.write(row +"\n") - fd.close() - rmCmd = RemoteCopy( name = "gpupgrademirror copy RFDFL: %s to %s:%s" % (fdName, self.remoteHost, self.fileListLocation) - , srcDirectory = fdName - , dstHost = self.remoteHost - , dstDirectory = self.fileListLocation - , ctxt = LOCAL - , remoteHost = None - ) - rmCmd.run(validateAfter=True) - os.unlink(fdName) - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class CreateTarFromList(Command): - """ - This class will create a tar file from a list of files. - WARNING. This class is not used and is untested. - """ - def __init__(self, name, fileListLocation, dstTarFile, srcDirectory, ctxt=LOCAL, remoteHost=None): - self.fileListLocation = fileListLocation - self.dstTarFile = dstTarFile - self.srcDirectory = srcDirectory - self.ctxt = ctxt - self.remoteHost = remoteHost - tarCmd = SYSTEM.getTarCmd() - cmdStr="%s %s | %s %s rvPpf %s -C %s " % \ - ( findCmdInPath("cat") - , fileListLocation - , findCmdInPath('xargs') - , tarCmd - , self.dstTarFile - , srcDirectory - ) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - def setupListFile(self, tempDir, theList): - fd, fdName = tempfile.mkstemp(prefix='tmpCTFL', dir=tempDir) - - fd = open(fdName, "w") - for row in theList: - fd.write(row +"\n") - fd.close() - rmCmd = RemoteCopy( name = "gpupgrademirror copy CTFL: %s to %s:%s" % (fdName, self.remoteHost, self.fileListLocation) - , srcDirectory = fdName - , dstHost = self.remoteHost - , dstDirectory = self.fileListLocation - , ctxt = LOCAL - , remoteHost = None - ) - rmCmd.run(validateAfter=True) - os.unlink(fdName) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class FileDirectoryList(Command): - """ This class gets list of files based on a pattern used by ls. """ - - def __init__(self, name, filePattern, ctxt=LOCAL, remoteHost=None): - self.filePattern = filePattern - cmdStr="%s -1 %s" % (findCmdInPath('ls'), filePattern) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - @staticmethod - def get_list(name, remote_host, file_pattern): - lsCmd = FileDirectoryList(name, filePattern = file_pattern, ctxt = REMOTE, remoteHost = remote_host) - lsCmd.run(validateAfter=False) - return lsCmd.get_result_list() - - def get_result_list(self): - files = self.results.stdout.split('\n') - if files != None and len(files) > 0 and files[-1] == '': - ''' Remove any trailing empty string ''' - files.pop() - return files - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class FilesInDir(Command): - """ This class gets a list of files in a directory """ - - def __init__(self, name, filePattern, ctxt=LOCAL, remoteHost=None, remoteTempFile=None): - self.filePattern = filePattern - self.remoteHost = remoteHost - self.remoteTempFile = remoteTempFile - if remoteTempFile != None: - remoteTempFilePart = " > %s" % remoteTempFile - else: - remoteTempFilePart = "" - find = findCmdInPath('find') - cmdStr= "%s %s -type f %s" % (find, filePattern, remoteTempFilePart) - self.cmdStr = cmdStr - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - - def get_result_list(self, localTempFile = None): - retList = [] - - if localTempFile != None: - rcCmd = CopyToLocal( name = "gpupgrademirror get result list copy: %s:%s to %s" % (self.remoteHost, self.remoteTempFile, localTempFile) - , srcHost = self.remoteHost - , srcDirectory = self.remoteTempFile - , dstDirectory = localTempFile - ) - rcCmd.run(validateAfter = True) - rmCmd = RemoveFiles( name = 'gpupgrademirror remove remote temp file: %s:%s' % (self.remoteHost, self.remoteTempFile) - , directory = self.remoteTempFile - , ctxt = REMOTE - , remoteHost = self.remoteHost - ) - rmCmd.run(validateAfter = True) - - fd = open(localTempFile, "r") - fd.seek(0) - fileList = [] - for line in fd: - line = line.rstrip('\n') - fileList.append(line) - fd.close() - else: - fileList = self.results.stdout.split('\n') - - if fileList != None and len(fileList) > 0 and fileList[-1] == '': - ''' Remove any trailing empty string ''' - fileList.pop() - retList = fileList - - return retList - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class FileAndSize(Command): - """ - This class gets a list of files in a directory and their sizes in k-bytes - The results are sorted by file size. If "zeroSize is true" only files that - have 0 bytes are returned. - """ - - def __init__(self, name, filePattern, ctxt=LOCAL, remoteHost=None, zeroSize=False, remoteTempFile=None): - self.filePattern = filePattern - self.remoteHost = remoteHost - self.remoteTempFile = remoteTempFile - if remoteTempFile != None: - remoteTempFilePart = " > %s" % remoteTempFile - else: - remoteTempFilePart = "" - if zeroSize == True: - sizeArg = "-size 0c" - else: - sizeArg = '' - find = findCmdInPath('find') - xargs = findCmdInPath('xargs') - du = findCmdInPath('du') - sort = findCmdInPath('sort') - cmdStr= "%s %s -type f %s | %s %s -k | %s -n %s" % (find, filePattern, sizeArg, xargs, du, sort, remoteTempFilePart) - self.cmdStr = cmdStr - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - - def get_result_list(self, localTempFile = None): - retList = [] - - if localTempFile != None: - rcCmd = CopyToLocal( name = "gpupgrademirror get result list copy: %s:%s to %s" % (self.remoteHost, self.remoteTempFile, localTempFile) - , srcHost = self.remoteHost - , srcDirectory = self.remoteTempFile - , dstDirectory = localTempFile - ) - rcCmd.run(validateAfter = True) - rmCmd = RemoveFiles( name = 'gpupgrademirror remove remote temp file: %s:%s' % (self.remoteHost, self.remoteTempFile) - , directory = self.remoteTempFile - , ctxt = REMOTE - , remoteHost = self.remoteHost - ) - rmCmd.run(validateAfter = True) - - fd = open(localTempFile, "r") - fd.seek(0) - fileList = [] - for line in fd: - fileList.append(line) - fd.close() - else: - fileList = self.results.stdout.split('\n') - - if fileList != None and len(fileList) > 0 and fileList[-1] == '': - ''' Remove any trailing empty string ''' - fileList.pop() - for file in fileList: - sizef, filef = file.split() - sizef = sizef.strip() - filef = filef.strip() - ###print "sizef = " + str(sizef) - ###print "filef = >>>" + str(filef) + "<<<" - retList.append([sizef, filef]) - return retList - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class DirectoryList(Command): - """ This method gets a list of all directories and any sub-directories in a directory """ - - def __init__(self, name, dirLocation, ctxt=LOCAL, remoteHost=None): - self.dirLocation = dirLocation - find = findCmdInPath('find') - sort = findCmdInPath('sort') - cmdStr= "%s %s -type d | %s " % (find, dirLocation, sort) - Command.__init__(self,name,cmdStr,ctxt,remoteHost) - - @staticmethod - def get_list(name, remote_host, dir_location): - theCmd = DirectoryList(name, dirLocation = dir_location, ctxt = REMOTE, remoteHost = remote_host) - theCmd.run(validateAfter=False) - return theCmd.get_result_list() - - def get_result_list(self): - dirList = self.results.stdout.split('\n') - if dirList != None and len(dirList) > 0 and dirList[-1] == '': - ''' Remove any trailing empty string ''' - dirList.pop() - if dirList != None and len(dirList) > 0: - ''' The first element is the dirLcation itself. Remove it''' - dirList.pop(0) - return dirList - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class SpecialFileHandling(): - """ - This is a specialized class for dealing with Phase 2 special files that must be - copied from the primary to the mirror. It creats hard links from a directory to - files that must be copied, and then copies the entire directory contents over. - It assumes all the files exist under the same directory (i.e. base). - """ - def __init__(self, sourceList, seg): - self.sourceList = sourceList - self.seg = seg - self.fullPathLinkDir = seg.primaryTempDir + "/speciallinkdir" - - - def createLinks(self): - """ - Create a file containing a list of ln parameters locally, and the copy it - to the primary node where we will run ln and create the links. - """ - linkFile = "gpupgrademirror%s.linkspecialfile" % str(self.seg.primary_dbid) - sourceFullPathLinkFile = options.info_data_directory + "/" + linkFile - targetFullPathLinkFile = self.seg.primaryTempDir + "/" + linkFile - linkFP = open(sourceFullPathLinkFile, 'w') - linkFP.seek(0) - for sfile in self.sourceList: - fullPathSrcFile = self.seg.primary_data_directory + "/" + sfile - fullPathTargetFile = self.fullPathLinkDir + "/" + sfile - linkFP.write("%s %s\n" % (fullPathSrcFile, fullPathTargetFile)) - linkFP.flush() - linkFP.close() - - """ Copy the link file to the primary temp dir """ - cmdComplete = False - while cmdComplete == False: - linkCopy = RemoteCopyPreserve( name = "gpupgrademirror copy ln file for primary: %s:%s" % (self.seg.primary_host, self.fullPathLinkDir) - , srcDirectory = sourceFullPathLinkFile - , dstHost = self.seg.primary_host - , dstDirectory = targetFullPathLinkFile - , ctxt = LOCAL - ) - cmdComplete = runAndCheckCommandComplete(linkCopy) - linkCopy.validate() - - """ Create a directory to store all our links to the actual files """ - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror special file link dir: %s:%s' % (self.seg.primary_host, self.fullPathLinkDir) - , directory = self.fullPathLinkDir - , ctxt = REMOTE - , remoteHost = self.seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - cmdComplete = False - while cmdComplete == False: - rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror special file link dir: %s:%s' % (self.seg.primary_host, self.fullPathLinkDir) - , directory = self.fullPathLinkDir - , ctxt = REMOTE - , remoteHost = self.seg.primary_host - , mode = 700 - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - """ Create any subdirectories for our links """ - subDirList = [] - for candidate in self.sourceList: - candidateList = candidate.split("/") - if len(candidateList) > 1: - dirPart = "/".join(candidateList[:-1]) - if dirPart not in subDirList: - subDirList.append(dirPart) - for subDir in subDirList: - fullPathSubDir = self.fullPathLinkDir + "/" + subDir - cmdComplete = False - while cmdComplete == False: - rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror special file link dir: %s:%s' % (self.seg.primary_host, fullPathSubDir) - , directory = fullPathSubDir - , ctxt = REMOTE - , remoteHost = self.seg.primary_host - , mode = 700 - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - """ Create the links """ - cmdComplete = False - while cmdComplete == False: - link = CreateHardLink( name = "gpupgrademirror creating links for special files %s:%s." % (self.seg.primary_host, str(self.seg.primary_data_directory)) - , srcFile = None - , hardLink = None - , ctxt = REMOTE - , remoteHost = self.seg.primary_host - , argsFile = targetFullPathLinkFile - ) - cmdComplete = runAndCheckCommandComplete(link) - link.validate() - - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove links file: %s' % (targetFullPathLinkFile) - , directory = targetFullPathLinkFile - , ctxt = REMOTE - , remoteHost = self.seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class SegHardLinks(Command): - """ - This class will generate groups of hard links to files in a segment directory. - The overall design is to create a new temporary directory "gpupgrademirrorhardbase", which - will contain a many to one relationship between its sub-directories and - the real database directories (e.g. gpupgrademirrorhardbase/1.1, gpupgrademirrorhardbase/1.2 both contian - links that point to base/1 files). - - We will special case any file that has zero length. This is because pysync did not - handle zero based files correctly at the time this class was written. - """ - - """ These two values determin the upper limit as to the number and size of files copied with each copy. """ - _max_files_per_pysync = 500 - _max_size_kb_per_pysync = 10000000 - - def __init__(self, mirror_info, logger): - logger.debug("in SegHardLinks__init__() %s:%s primary seg id: %s" % (mirror_info.primary_host, mirror_info.primary_data_directory, str(mirror_info.primary_dbid))) - self.logger = logger - self.theMirror = mirror_info - self.hardBaseDir = self.theMirror.primary_data_directory + '/gpupgrademirrorhardbase' - - #------------------------------------------------------------------------------- - def createLinks(self): - self.logger.debug('in SegHardLinks.createLinks(). Primary seg id %s' % str(self.theMirror.primary_dbid)) - cmdComplete = False - while cmdComplete == False: - baseDir = DirectoryList( name = "gpupgrademirror get directory list under base for primary seg: " + str(self.theMirror.primary_dbid) - , dirLocation = self.theMirror.primary_data_directory + '/base' - , ctxt = REMOTE - , remoteHost = self.theMirror.primary_host - ) - cmdComplete = runAndCheckCommandComplete(baseDir) - baseDir.validate() - dirResultList = baseDir.get_result_list() - - """ Remove any reference to a dir called pgsql_tmp, which may exist in some database directories. """ - for aDir in dirResultList: - dirSuffixIndex = aDir.rfind("/") - candidateDir = aDir[dirSuffixIndex + 1:].rstrip() - if candidateDir == "pgsql_tmp": - dirResultList.remove(aDir) - - - """ Create a directory to store all our links to the actual files """ - cmdComplete = False - while cmdComplete == False: - rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror make soft base: %s' % self.hardBaseDir - , directory = self.hardBaseDir - , ctxt = REMOTE - , remoteHost = self.theMirror.primary_host - , mode = 700 - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - for dir in dirResultList: - """ For each database directory """ - self.logger.debug("working on dir %s:%s" % (self.theMirror.primary_host, str(dir))) - - """ Get a list of all files in the dir and their sizes in k-bytes """ - cmdComplete = False - while cmdComplete == False: - dirFileSizeAll = FileAndSize( name = 'gpupgradmirror get list of files and size in database dir' - , filePattern = str(dir) - , ctxt = REMOTE - , remoteHost = self.theMirror.primary_host - , remoteTempFile = self.theMirror.primary_data_directory + '/gpupgrademirrorhardbase/tempfileandsize' - ) - cmdComplete = runAndCheckCommandComplete(dirFileSizeAll) - dirFileSizeAll.validate() - dbFileListAll = dirFileSizeAll.get_result_list(localTempFile = options.info_data_directory + "/tempfileandsize" + str(self.theMirror.primary_dbid)) - - """ Get a sub-list of files that have zero length """ - cmdComplete = False - while cmdComplete == False: - dirFileSizeZero = FileAndSize( name = 'gpupgradmirror get list of files and size in database dir' - , filePattern = str(dir) - , ctxt = REMOTE - , remoteHost = self.theMirror.primary_host - , zeroSize = True - , remoteTempFile = self.theMirror.primary_data_directory + '/gpupgrademirrorhardbase/tempfileandsize' - ) - cmdComplete = runAndCheckCommandComplete(dirFileSizeZero) - dirFileSizeZero.validate() - dbFileListZero = dirFileSizeZero.get_result_list(localTempFile = options.info_data_directory + "/tempfileandsize" + str(self.theMirror.primary_dbid)) - dbFileListZeroLength = len(dbFileListZero) - - dbFileListZeroName = [] - for iii in range(len(dbFileListZero)): - dbFileListZeroName.append(dbFileListZero[iii][1]) - - """ Divide the two lists into distinct sub-lists """ - dbFileNonZeroList = [] - for candidate in dbFileListAll: - if candidate[0] == str(0): - """ size is less than 1k """ - if candidate[1] not in dbFileListZeroName: - dbFileNonZeroList.append(candidate) - else: - dbFileNonZeroList.append(candidate) - - """ Recombine the lists, putting the zero length file list on the front """ - dbFileList = dbFileListZero + dbFileNonZeroList - - newDirSuffix = 0 - fileIndex = 0 - dirIndex = dir.rfind('/') - theDir = dir[dirIndex + 1:] - while fileIndex < len(dbFileList): - """ For each file in the database directory """ - dirPlusSuffix = self.hardBaseDir + "/" + str(theDir) + "." + str(newDirSuffix) - - cmdComplete = False - while cmdComplete == False: - rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror make soft base sub-database dir: %s' % dirPlusSuffix - , directory = dirPlusSuffix - , ctxt = REMOTE - , remoteHost = self.theMirror.primary_host - , mode = 700 - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - numFiles = 0 - currentSize = 0 - srcFileList = [] - hardLinkList = [] - while (newDirSuffix == 0 and fileIndex < dbFileListZeroLength) \ - or \ - ( newDirSuffix != 0 and fileIndex < len(dbFileList) \ - and numFiles < self._max_files_per_pysync \ - and currentSize < self._max_size_kb_per_pysync \ - ): - """ Make groups of the files and create links to the grouped files """ - sizeSourceFile = dbFileList[fileIndex][0] - fullPathSourceFile = dbFileList[fileIndex][1] - subfileIndex = fullPathSourceFile.rfind('/') - theFile = fullPathSourceFile[subfileIndex + 1:] - sLink = dirPlusSuffix + "/" + theFile - srcFileList.append(fullPathSourceFile) - hardLinkList.append(sLink) - numFiles = numFiles + 1 - currentSize = currentSize + int(sizeSourceFile) - fileIndex = fileIndex + 1 - if newDirSuffix == 0 and fileIndex == dbFileListZeroLength: - zeroFilesFlag = False - - """ - Create a file containing a list of ln parameters locally, and the copy it - to the primary node where we will run ln and create the links. - """ - linkFile = "gpupgrademirror%s.linkfile" % str(self.theMirror.primary_dbid) - sourceFullPathLinkFile = options.info_data_directory + "/" + linkFile - targetFullPathLinkFile = self.hardBaseDir + "/" + linkFile - linkFP = open(sourceFullPathLinkFile, 'w') - linkFP.seek(0) - for lnIndex in range(len(srcFileList)): - linkFP.write("%s %s\n" % (srcFileList[lnIndex], hardLinkList[lnIndex])) - linkFP.flush() - linkFP.close() - cmdComplete = False - while cmdComplete == False: - linkCopy = RemoteCopyPreserve( name = "gpupgrademirror copy ln file for primary dbid: " + str(self.theMirror.primary_dbid) - , srcDirectory = sourceFullPathLinkFile - , dstHost = self.theMirror.primary_host - , dstDirectory = targetFullPathLinkFile - , ctxt = LOCAL - ) - cmdComplete = runAndCheckCommandComplete(linkCopy) - linkCopy.validate() - - cmdComplete = False - while cmdComplete == False: - link = CreateHardLink( name = "gpupgrademirror creating links for dbid %s directory %s ." % (self.theMirror.primary_dbid, str(newDirSuffix)) - , srcFile = srcFileList - , hardLink = hardLinkList - , ctxt = REMOTE - , remoteHost = self.theMirror.primary_host - , argsFile = targetFullPathLinkFile - ) - cmdComplete = runAndCheckCommandComplete(link) - link.validate() - - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove links file: %s' % (targetFullPathLinkFile) - , directory = targetFullPathLinkFile - , ctxt = REMOTE - , remoteHost = self.theMirror.primary_host - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - newDirSuffix = newDirSuffix + 1 - - #------------------------------------------------------------------------------- - def removeLinks(self): - self.logger.debug("in SegHardLinks.removeLinks %s:%s" % (self.theMirror.primary_host, self.hardBaseDir)) - - cmdComplete = False - while cmdComplete == False: - cmd = RemoveFiles( name = 'gpupgrademirror remove gpupgrademirrorhardbase dir' - , remoteHost = self.theMirror.primary_host - , directory = self.hardBaseDir - , ctxt = REMOTE - ) - cmdComplete = runAndCheckCommandComplete(cmd) - cmd.validate() - - #------------------------------------------------------------------------------- - def getHardDirList(self): - """ - Returns a list of the fully qualified link dirs - (e.g. [ '/db1/seg1/g0/base/2.1' , '/db1/seg1/g0/base/2.2' ,...] ) - """ - retList = [] - cmdComplete = False - while cmdComplete == False: - baseDir = DirectoryList( name = "gpupgrademirror get directory list under base for primary seg: " + str(self.theMirror.primary_dbid) - , dirLocation = self.hardBaseDir - , ctxt = REMOTE - , remoteHost = self.theMirror.primary_host - ) - cmdComplete = runAndCheckCommandComplete(baseDir) - baseDir.validate() - retList = baseDir.get_result_list() - return retList - - #------------------------------------------------------------------------------- - @staticmethod - def getUnqualifiedLinkDir(qualifiedDir): - retValue = '' - startIndex = qualifiedDir.rfind("/") - retValue = qualifiedDir[startIndex +1:] - return retValue - - #------------------------------------------------------------------------------- - @staticmethod - def getUnqualifiedRealDir(qualifiedDir): - retValue = '' - startIndex = qualifiedDir.rfind("/") - endIndex = qualifiedDir.rfind(".") - retValue = qualifiedDir[startIndex +1:endIndex] - return retValue - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class PySyncPlus(PySync): - """ - This class is really just PySync but it records all the parameters passed in. - """ - - def __init__(self,name,srcDir,dstHost,dstDir,ctxt=LOCAL,remoteHost=None, options=None): - self.namePlus = name - self.srcDirPlus = srcDir - self.dstHostPlus = dstHost - self.dstDirPlus = dstDir - self.ctxtPlus = ctxt - self.remoteHostPlus = remoteHost - self.optionsPlus = options - PySync.__init__( self - , name = name - , srcDir = srcDir - , dstHost = dstHost - , dstDir = dstDir - , ctxt = ctxt - , remoteHost = remoteHost - , options = options - ) - self.destinationHost = dstHost - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class GPUpgradeMirrorMinion(Command): - """ This class represents a gpugprademirror minion process. """ - - def __init__(self, name, options, segs, minionSeg, segmentDir, ctxt=LOCAL, remoteHost=None): - self.name = name - if options != None: - self.options = options - else: - options = "" - self.segList = segs - self.minionSeg = minionSeg - self.segmentDir = segmentDir - self.ctxt = ctxt - self.remoteHost = remoteHost - - mDir = os.path.split(self.segmentDir) - dirPrefix = mDir[0] - if len(dirPrefix) == 1 and str(dirPrefix) == str('/'): - """ Special case where the directory is '/'. """ - dirPrefix = "" - dirSuffix = mDir[1] - self.minionGpmigratorDir = dirPrefix + "/" + GPMIGRATOR_TEMP_DIR - self.minionDir = self.minionGpmigratorDir + "/" + dirSuffix + INFO_DIR + str(minionSeg) - - fp = os.environ.get("GP_COMMAND_FAULT_POINT") - if fp != None and len(fp) > 0: - preCommand = "export GP_COMMAND_FAULT_POINT='%s';" % str(fp) - else: - preCommand = "" - self.gpupgrademirror_executable = preCommand + "$GPHOME/sbin/" + EXECNAME - self.cmdStr = "%s %s -i %s -t %s" % ( self.gpupgrademirror_executable - , self.options - , segs - , self.minionDir - ) - Command.__init__( self - , name = self.name - , cmdStr = self.cmdStr - , ctxt = self.ctxt - , remoteHost = self.remoteHost - ) - - #------------------------------------------------------------------------------- - def createInfoDirectory(self): - """ Create a info directory for the minion """ - logger.debug("Create info directory for minion: %s:%s" % (self.remoteHost, self.minionDir)) - mkDirCmd = CreateDirIfNecessaryWithMode( name = 'Create the minion info dir: %s:%s' % (self.remoteHost, self.minionGpmigratorDir) - , directory = self.minionGpmigratorDir - , ctxt = REMOTE - , remoteHost = self.remoteHost - , mode = '0700' - ) - mkDirCmd.run(validateAfter=True) - mkDirCmd = CreateDirIfNecessaryWithMode( name = 'Create the minion info dir: %s:%s' % (self.remoteHost, self.minionDir) - , directory = self.minionDir - , ctxt = REMOTE - , remoteHost = self.remoteHost - , mode = '0700' - ) - mkDirCmd.run(validateAfter=True) - - #------------------------------------------------------------------------------- - def copyStatusFileToHost(self): - source = overallStatus._status_filename - target = self.minionDir + "/" + overallStatus._status_filename_unqualified - logger.debug("copy status file for minion: %s:%s" % (self.remoteHost, target)) - cpCmd = RemoteCopyPreserve( name = 'gpupgrademirror copy status file to minion : %s:%s' % (self.remoteHost, target) - , srcDirectory = source - , dstHost = self.remoteHost - , dstDirectory = target - , ctxt = LOCAL - , remoteHost = None - ) - cpCmd.run(validateAfter=True) - - - #------------------------------------------------------------------------------- - def copyInfoFileToHost(self): - source = p1MirrorInfo.mirrorInfoFile - target = self.minionDir + "/" + p1MirrorInfo.mirrorInfoFileUnqualified - logger.debug("copy info file for minion: %s:%s" % (self.remoteHost, target)) - cpCmd = RemoteCopyPreserve( name = 'gpupgrademirror copy info dir to minion : %s:%s' % (self.remoteHost, target) - , srcDirectory = source - , dstHost = self.remoteHost - , dstDirectory = target - , ctxt = LOCAL - , remoteHost = None - ) - cpCmd.run(validateAfter=True) - - - #------------------------------------------------------------------------------- - def copyGpHostCacheFileToHost(self): - source = p1MirrorInfo.gpHostCacheInfo.CACHEFILE - target = self.minionDir + "/" + p1MirrorInfo.gpHostCacheInfo.FILENAME - logger.debug("copy info file for minion: %s:%s" % (self.remoteHost, target)) - cpCmd = RemoteCopyPreserve( name = 'gpupgrademirror copy info dir to minion : %s:%s' % (self.remoteHost, target) - , srcDirectory = source - , dstHost = self.remoteHost - , dstDirectory = target - , ctxt = LOCAL - , remoteHost = None - ) - cpCmd.run(validateAfter=True) - - #------------------------------------------------------------------------------- - def copyLogFile(self, parentLogFile): - - self.parentLogFile = parentLogFile - minionSubstring = EXECNAME + str(self.minionSeg) - self.logFileName = self.parentLogFile.replace(EXECNAME, minionSubstring) - cmd = CopyToLocal( name = "gpupgrademirror copy minion log file up. %s:%s" % (self.remoteHost, self.logFileName) - , srcDirectory = self.logFileName - , srcHost = self.remoteHost - , dstDirectory = self.logFileName - ) - cmd.run(validateAfter=True) - - -#------------------------------------------------------------------------- -#------------------------------------------------------------------------- -class GpUpgradeMirrorStatus(): - """ - This class manages gpupgrademirror status files. - - There is one status file per mirror segment, and one overall status file. - Each status file has the segment dbid as part of its suffix. - """ - _safeModeStatusValues = { 'PHASE1_SAFE_MODE' : 2 - , 'START_SEG_SETUP' : 3 - , 'END_SEG_SETUP' : 4 - , 'START_PRIMARY_COPY' : 5 - , 'END_PRIMARY_COPY' : 6 - , 'START_NEW_MIRROR_FILE_FIXUP' : 7 - , 'END_NEW_MIRROR_FILE_FIXUP' : 8 - , 'START_MOVE_OLD_MIRROR' : 9 - , 'END_MOVE_OLD_MIRROR' : 10 - , 'START_MOVE_NEW_MIRROR' : 11 - , 'END_MOVE_NEW_MIRROR' : 12 - , 'START_REMOVE_OLD_MIRROR' : 13 - , 'END_REMOVE_OLD_MIRROR' : 14 - , 'PHASE1_DONE' : 15 - , 'START_PHASE2' : 16 - , 'PHASE2_DONE' : 17 - } - - _unsafeModeStatusValues = { 'PHASE1_UNSAFE_MODE' : 2 - , 'START_SEG_SETUP' : 3 - , 'END_SEG_SETUP' : 4 - , 'START_MOVE_OLD_MIRROR_SPECIAL_FILES' : 5 - , 'END_MOVE_OLD_MIRROR_SPECIAL_FILES' : 6 - , 'START_REMOVE_OLD_MIRROR' : 7 - , 'END_REMOVE_OLD_MIRROR' : 8 - , 'START_PRIMARY_COPY' : 9 - , 'END_PRIMARY_COPY' : 10 - , 'START_NEW_MIRROR_FILE_FIXUP' : 11 - , 'END_NEW_MIRROR_FILE_FIXUP' : 12 - , 'START_MOVE_NEW_MIRROR' : 13 - , 'END_MOVE_NEW_MIRROR' : 14 - , 'START_REMOVE_OLD_MIRROR_SPECIAL_FILES' : 15 - , 'END_REMOVE_OLD_MIRROR_SPECIAL_FILES' : 16 - , 'PHASE1_DONE' : 17 - , 'START_PHASE2' : 18 - , 'PHASE2_DONE' : 19 - } - - """ Note, Both safe and un-safe mode values are valid, therefore their values are the same '2' """ - _overallStatusValues = { 'PHASE1_SAFE_MODE' : 2 - , 'PHASE1_UNSAFE_MODE' : 2 - , 'START_PHASE1_SETUP' : 3 - , 'END_PHASE1_SETUP' : 4 - , 'MINION_START_SETUP' : 5 - , 'MINION_SETUP_DONE' : 6 - , 'PHASE1_DONE' : 7 - , 'START_PHASE2' : 8 - , 'START_PHASE2_SETUP' : 9 - , 'END_PHASE2_SETUP' : 10 - , 'PHASE2_DONE' : 11 - } - - - - def __init__(self, logger, info_data_directory, segment_dbid, phase, overall = False): - logger.debug("in __init__ for GpUpgradeMirrorStatus. dbid = %s" % str(segment_dbid)) - self.logger = logger - self.overall = overall - - if overall == True: - self._status_values = self._overallStatusValues - elif options.mode == 'S': - self._status_values = self._safeModeStatusValues - else: - self._status_values = self._unsafeModeStatusValues - - self._status = [] - self._status_info = [] - self._info_data_directory = info_data_directory - self._phase = phase - self.segment_dbid = segment_dbid - self._status_filename_unqualified = 'gpupgrademirror' + str(segment_dbid) + '.status' - self._status_filename = info_data_directory + "/" + self._status_filename_unqualified - self._fp = None - self._fp_standby = None - self._temp_dir = None - self._input_filename = None - self._original_primary_count = None - self._gp_configuration_backup = None - - if os.path.exists(self._status_filename): - self._read_status_file() - logger.debug("__init__ for GpUpgradeMirrorStatus complete. dbid = %s" % str(segment_dbid)) - - - #------------------------------------------------------------------------------- - def _read_status_file(self): - """Reads in an existing gpupgrademirror status file""" - self.logger.debug("Trying to read in a pre-existing gpupgrademirror status file %s" % (self._status_filename)) - try: - self._fp = open(self._status_filename, 'a+') - self._fp.seek(0) - - for line in self._fp: - (status, status_info) = line.rstrip().split(':') - ''' Determine if the previous run was phase 1 and was in safe or un-safe mode ''' - ''' re-set the mode based on the result''' - if status == 'PHASE1_SAFE_MODE': - options.mode = 'S' - if self.overall == False: - self._status_values = self._safeModeStatusValues - elif status == 'PHASE1_UNSAFE_MODE': - options.mode = 'U' - if self.overall == False: - self._status_values = self._unsafeModeStatusValues - elif status == 'START_PHASE2': - options.phase2 = True - self._status.append(status) - self._status_info.append(status_info) - except Exception, e: - logger.warning('Unable to read segment status file: ' + str(e)) - logger.warning('gpupgradmirror exiting') - """ KAS note to self. There is a reason for only putting out a warning here. Something - to do with how upgrade wants me to handle the situation. Need to revisit this and - figure out if this is still correct - """ - exit(0) - - if not self._status_values.has_key(self._status[-1]): - raise InvalidStatusError('Invalid status file. Unknown status %s' % self._status) - - #------------------------------------------------------------------------------- - def status_file_exists(self): - """ Return true if the status file already exists. """ - if os.path.exists(self._status_filename): - return True - else: - return False - - #------------------------------------------------------------------------------- - def create_status_file(self): - """Creates a new gpupgrademirror status file""" - try: - if options.phase2 == True: - startStatus = 'PHASE1_DONE' - elif options.mode == 'S': - startStatus = 'PHASE1_SAFE_MODE' - else: - startStatus = 'PHASE1_UNSAFE_MODE' - - self._fp = open(self._status_filename, 'w') - self._fp.write(startStatus + ':None\n') - self._fp.flush() - self._status.append(startStatus) - self._status_info.append('None') - except IOError, e: - self.logger.error('Unable to create status file %s.' % self._status_filename) - raise e - - #------------------------------------------------------------------------------- - def set_status(self, status, status_info=None): - """ - Sets the current status. gpupgrademirror status must be set in - proper order. Any out of order status result in an - InvalidStatusError exception - """ - self.logger.debug("Segment %s: Transitioning from %s to %s" % (str(self.segment_dbid), self._status[-1], status)) - - if not self._fp: - raise InvalidStatusError('The status file is invalid and cannot be written to') - if not self._status_values.has_key(status): - raise InvalidStatusError('%s is an invalid gpupgrademirror status' % status) - # Only allow state transitions forward or backward 1 - ''' KAS note to self. Should probably not skip checking if in rollback ''' - if self._status and options.rollback == False: - if options.continue_upgrade == False and self._status_values[status] != self._status_values[self._status[-1]] + 1 and self._status_values[status] != self._status_values[self._status[-1]]: - raise InvalidStatusError('Invalid status transition from %s to %s' % (self._status[-1], status)) - elif options.continue_upgrade == True and \ - self._status_values[status] != self._status_values[self._status[-1]] and \ - self._status_values[status] != self._status_values[self._status[-1]] + 1: - raise InvalidStatusError('Invalid status transition from %s to %s' % (self._status[-1], status)) - self._fp.write('%s:%s\n' % (status, status_info)) - self._fp.flush() - self._status.append(status) - self._status_info.append(status_info) - - - #------------------------------------------------------------------------------- - def get_current_status(self): - """Gets the current status that has been written to the gpupgrademirror - status file""" - if (len(self._status) > 0 and len(self._status_info) > 0): - return (self._status[-1], self._status_info[-1]) - else: - return (None, None) - - - #------------------------------------------------------------------------------- - def get_current_status_step_number(self): - currentStatus = self.get_current_status() - if currentStatus[0] == None: - return -1 - return self._status_values[currentStatus[0]] - - - #------------------------------------------------------------------------------- - def compare_status(self, status2): - ''' return -1 status1 < status2 - 0 status1 = status2 - 1 status1 > status2 - ''' - status1 = self.get_current_status()[0] - if status1 == None: - if status2 == None: - return 0 - return -1 - if status2 == None: - return 1 - - num1 = self._status_values[status1] - num2 = self._status_values[status2] - if num1 < num2: - return -1 - elif num1 == num2: - return 0 - else: - return 1 - - - #------------------------------------------------------------------------------- - def get_status_history(self): - """Gets the full status history""" - return zip(self._status, self._status_info) - - - #------------------------------------------------------------------------------- - def remove_status_file(self): - """Closes and removes the gpupgrademirror status file""" - logger.debug("Remove status file: %s" % self._status_filename) - if self._fp: - self._fp.close() - self._fp = None - if self._fp_standby: - self._fp_standby.close() - self._fp_standby = None - if os.path.exists(self._status_filename): - os.unlink(self._status_filename) - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class Phase1(threading.Thread): - ''' - An object of this class represents a single node in a cluster for Phase 1. - It will upgrade all mirror segments on that node. - ''' - - def __init__(self, dburl, logger, info_data_directory, node_name, mirror_info_list, setup_completed_event, continue_event): - threading.Thread.__init__(self) - - self.tempOldMirrorDirSuffix = 'gpupgrademirrors.originalmirror' - self.tempNewMirrorDirSuffix = 'gpupgrademirrors.copyfromprimary' - self.fixupFileList = [ 'postgresql.conf' - , 'pg_hba.conf' - , 'pg_ident.conf' - ] - self.dburl = dburl - self.logger = logger - self.info_data_directory = info_data_directory - self.nodeName = node_name - self.mirrorSegmentList = mirror_info_list - self.nodeNameNIC = mirror_info_list[0].mirror_host - self.statusList = [] - self.setupCompletedEvent = setup_completed_event - self.continueEvent = continue_event - self.startedRun = False - self.completedRun = False - - #------------------------------------------------------------------------------- - def setup(self): - self.logger.debug("Phase1 setup for mirror: " + str(self.nodeName)) - - """ Setup each segment on the mirror. """ - for seg in self.mirrorSegmentList: - self.logger.debug("setup for segment: " + str(seg.mirror_dbid)) - - """ Create a temp directory in each primary segment for our use""" - seg.primaryTempDir = seg.primary_data_directory + "/" + GPUPGRADEMIRROR_TEMP_DIR - cmdComplete = False - while cmdComplete == False: - rmCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make temp dir: %s' % seg.primaryTempDir - , directory = seg.primaryTempDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - , mode = "0700" - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - """ Set the number of workers based on net speed. """ - if netMBSpeed < 200: - seg.workersPerNic = 2 - elif netMBSpeed < 400: - seg.workersPerNic = 4 - else: - seg.workersPerNic = 8 - - ''' Create a status object for each mirror segment ''' - status = GpUpgradeMirrorStatus( logger = self.logger - , info_data_directory = self.info_data_directory - , segment_dbid = seg.mirror_dbid - , phase = 1 - ) - self.statusList.append(status) - - ''' Make sure their is enough space for the largest segment ''' - if options.rollback == False and status.compare_status('START_PRIMARY_COPY') < 0: - self.checkAvailableSpace(seg) - - seg.rollForward = False - if options.rollback == True: - ''' setup for rollback ''' - if status.get_current_status() == (None, None): - self.logger.warning('There is no upgrade mirror in progress for this segment: ' + str(seg.mirror_dbid)) - """ Create a status file so that other logic works correctly """ - status.create_status_file() - elif status.compare_status('START_REMOVE_OLD_MIRROR') >= 0 \ - and status.compare_status('PHASE1_DONE') < 0: - ''' We are past the point of recovering back. Must roll forward at this point ''' - self.logger.debug("Past the point of no return. Must roll forward on mirror: %s mirror seg dbid: %s" % (self.nodeName, str(seg.mirror_dbid))) - seg.rollForward = True - elif status.compare_status('START_SEG_SETUP') >= 0: - genLink = SegHardLinks( mirror_info = seg - , logger = self.logger - ) - genLink.removeLinks() - if options.continue_upgrade == True or seg.rollForward == True: - ''' Setup to continue upgrade ''' - if status.get_current_status() == (None, None): - status.create_status_file() - if status.compare_status('END_SEG_SETUP') < 0: - genLink = SegHardLinks( mirror_info = seg - , logger = self.logger - ) - genLink.removeLinks() - status.set_status('START_SEG_SETUP') - genLink.createLinks() - status.set_status('END_SEG_SETUP') - elif options.rollback == False: - if (status.get_current_status() != (None, None)): - self.logger.error("Upgrade mirrors already started.") - self.logger.error("You must either rollback or continue mirror upgrade") - raise ValidationError("Unable to continue") - else: - """ We are doing a normal phase 1 setup """ - status.create_status_file() - genLink = SegHardLinks( mirror_info = seg - , logger = self.logger - ) - status.set_status('START_SEG_SETUP') - genLink.createLinks() - status.set_status('END_SEG_SETUP') - - #------------------------------------------------------------------------------- - def renameOldMirrorToTemp(self, seg): - self.logger.debug('in renameOldMirror %s:%s seg id = %s' % (self.nodeName, seg.mirror_data_directory, seg.mirror_dbid)) - - tempDataDir = seg.mirror_data_directory + self.tempOldMirrorDirSuffix - - if options.continue_upgrade == True: - cmdComplete = False - while cmdComplete == False: - cmd = RemoveFiles( name = 'gpupgrademirror remove any old directory' - , remoteHost = self.nodeNameNIC - , directory = tempDataDir - , ctxt = REMOTE - ) - cmdComplete = runAndCheckCommandComplete(cmd) - cmd.validate() - - if options.mode == 'S': - cmdComplete = False - while cmdComplete == False: - rmCmd = GPumMoveDirectory( name = 'gpupgrademirror move old mirror to temp location: %s:%s' % (seg.mirror_data_directory, tempDataDir) - , srcDirectory = seg.mirror_data_directory - , dstDirectory = tempDataDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - else: - ''' Un-safe mode ''' - cmdComplete = False - while cmdComplete == False: - rmCmd = MakeDirectoryWithMode( name = 'gpupgrademirror make temp directory for old mirror special files: %s' % tempDataDir - , directory = tempDataDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - , mode = 700 - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - for file in self.fixupFileList: - fullPathFile = seg.mirror_data_directory + '/' + file - fullPathTarget = tempDataDir + '/' + file - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoteCopyPreserve( name = 'gpupgrademirror fixup file list fixup copy : %s:%s' % (fullPathFile, fullPathTarget) - , srcDirectory = fullPathFile - , dstHost = self.nodeNameNIC - , dstDirectory = fullPathTarget - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - - #------------------------------------------------------------------------------- - def renameTempMirrorToOldMirror(self, seg, dirSuffix): - self.logger.debug("in renameTempMirrorToOldMirror for mirror data directory %s:%s" % (self.nodeName, seg.mirror_data_directory)) - - tempDataDir = seg.mirror_data_directory + dirSuffix - cmdComplete = False - while cmdComplete == False: - rmCmd = GPumMoveDirectory( name = 'gpupgrademirror move temp mirror to mirror location: %s:%s' % (tempDataDir, seg.mirror_data_directory) - , srcDirectory = tempDataDir - , dstDirectory = seg.mirror_data_directory - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - #------------------------------------------------------------------------------- - def copyPrimaryToMirror(self, seg): - try: - self.logger.debug("in copyPrimaryToMirror for primary %s:%s" % (seg.primary_host, seg.primary_data_directory)) - - mirrorTempDataDir = seg.mirror_data_directory + self.tempNewMirrorDirSuffix - cmdComplete = False - """ Make the gpupgrademirrors.copyfromprimary directory """ - while cmdComplete == False: - mkDirCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make temp directory for old mirror special files: %s' % mirrorTempDataDir - , directory = mirrorTempDataDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - , mode = '0700' - ) - cmdComplete = runAndCheckCommandComplete(mkDirCmd) - mkDirCmd.validate() - - mirrorTempBaseDataDir = mirrorTempDataDir + "/base" - cmdComplete = False - """ Make the gpupgrademirrors.copyfromprimary/base directory """ - while cmdComplete == False: - mkDirCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make temp directory for old mirror special files: %s' % mirrorTempBaseDataDir - , directory = mirrorTempBaseDataDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - , mode = '0700' - ) - cmdComplete = runAndCheckCommandComplete(mkDirCmd) - mkDirCmd.validate() - - genLink = SegHardLinks(seg, self.logger) - dirList = genLink.getHardDirList() - - for dir in dirList: - targetTempDir = mirrorTempDataDir + "/base/" + SegHardLinks.getUnqualifiedLinkDir(dir) - if dir.endswith(".0"): - self.logger.debug("handle special directory *.0 for %s:%s" % (self.nodeName, targetTempDir)) - """ Special directory """ - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove old mirror special directory: %s:%s' % (self.nodeName, dir) - , directory = targetTempDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - continue - - cmdComplete = False - while cmdComplete == False: - mkDirCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make temp directory for old mirror special files: %s' % targetTempDir - , directory = targetTempDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - , mode = '0700' - ) - cmdComplete = runAndCheckCommandComplete(mkDirCmd) - mkDirCmd.validate() - - nicList = p1MirrorInfo.getInterfaceList(self.nodeName) - self.logger.debug("NICs for mirror %s are %s" % (self.nodeName, str(nicList))) - - totalWorkers = len(nicList) * seg.workersPerNic - availableNicQueue = Queue(totalWorkers) - - """ Need to prime the queue """ - for wpn in range(seg.workersPerNic): - for nic in nicList: - availableNicQueue.put(nic) - - wPool = WorkerPool(numWorkers = totalWorkers) - - """ copy everyting in the base directory """ - numCopiesStarted = 0 - numCopiesCompleted = 0 - for dir in dirList: - sourceDir = dir - specialSourceDir = sourceDir.endswith(".0") - targetDir = mirrorTempDataDir + "/base/" + SegHardLinks.getUnqualifiedLinkDir(sourceDir) - nicToUse = availableNicQueue.get() - if specialSourceDir == True: - self.logger.debug("handle special case directoryn %s:%s" % (seg.primary_host, str(dir))) - """ Special case of directory with empty files """ - rmCopy = RemoteCopyPreserve( name = 'gpupgrademirror remotecopy special .0 file to mirror: %s:%s' % (self.nodeName, seg.mirror_data_directory) - , srcDirectory = sourceDir - , dstHost = nicToUse - , dstDirectory = targetDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - rmCopy.run(validateAfter = True) - availableNicQueue.put(nicToUse) - continue - else: - nameStr = 'gpupgrademirror pysync primary to mirror: %s:%s' % (self.nodeName, seg.mirror_data_directory) - nameStr = nameStr + " SourceHost=%s NICname=%s " %(seg.primary_host, nicToUse) - pysyncCmd = PySyncPlus( name = nameStr - , srcDir = sourceDir - , dstHost = nicToUse - , dstDir = targetDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - wPool.addCommand(pysyncCmd) - numCopiesStarted = numCopiesStarted + 1 - if availableNicQueue.empty() == True: - while True: - completedCmdList = wPool.getCompletedItems() - if len(completedCmdList) > 0: - numCopiesCompleted = numCopiesCompleted + len(completedCmdList) - break; - for compCmd in completedCmdList: - if sshBusy(compCmd) == True: - time.sleep(1) - if isinstance(compCmd, PySyncPlus) == True: - redoCmd = PySyncPlus( name = compCmd.namePlus - , srcDir = compCmd.srcDirPlus - , dstHost = compCmd.dstHostPlus - , dstDir = compCmd.dstDirPlus - , ctxt = compCmd.ctxtPlus - , remoteHost = compCmd.remoteHostPlus - , options = compCmd.optionsPlus - ) - else: - redoCmd = RemoteCopyPreserve( name = 'gpupgrademirror remotecopy special .0 file to mirror: %s:%s' % (compCmd.dstHost, compCmd.srcDirectory) - , srcDirectory = compCmd.srcDirectory - , dstHost = compCmd.dstHost - , dstDirectory = compCmd.dstDirectory - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - wPool.addCommand(redoCmd) - else: - compCmd.validate() - availableNicQueue.put(compCmd.destinationHost) - - """ Collect all the uncompleted copies. """ - while numCopiesStarted > numCopiesCompleted: - completedCmdList = wPool.getCompletedItems() - if len(completedCmdList) > 0: - numCopiesCompleted = numCopiesCompleted + len(completedCmdList) - for compCmd in completedCmdList: - """ Go through the last commands and see if we need to retry """ - retryPossible = sshBusy(compCmd) - if retryPossible == True: - cmdComplete = False - else: - cmdComplete = True - while cmdComplete == False: - if isinstance(compCmd, PySyncPlus) == True: - compCmd = PySyncPlus( name = compCmd.namePlus - , srcDir = compCmd.srcDirPlus - , dstHost = compCmd.dstHostPlus - , dstDir = compCmd.dstDirPlus - , ctxt = compCmd.ctxtPlus - , remoteHost = compCmd.remoteHostPlus - , options = compCmd.optionsPlus - ) - else: - compCmd = RemoteCopyPreserve( name = 'gpupgrademirror remotecopy special .0 file to mirror: %s:%s' % (compCmd.dstHost, compCmd.srcDirectory) - , srcDirectory = compCmd.srcDirectory - , dstHost = compCmd.dstHost - , dstDirectory = compCmd.dstDirectory - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(compCmd) - compCmd.validate() - else: - time.sleep(1) - - wPool.haltWork() - wPool.joinWorkers() - genLink.removeLinks() - - """ - Now we need to copy everything but the base directory. We assume these files are small in - comparison to the base directory, so we will use scp instead of pysync. Also, there may - be some zero-lenght files, which pysync can not handle. - """ - cmdComplete = False - while cmdComplete == False: - fdlCmd = FileDirectoryList( name = "gpupgrademirror find all non-base dirs and files for %s:%s " % (seg.primary_host, seg.primary_data_directory) - , filePattern = seg.primary_data_directory - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(fdlCmd) - fdlCmd.validate() - nonBaseList = fdlCmd.get_result_list() - nonBaseList.remove("base") - nonBaseList.remove(GPUPGRADEMIRROR_TEMP_DIR) - - for nonBase in nonBaseList: - cmdComplete = False - mirrorNonBase = mirrorTempDataDir + "/" + nonBase - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove non base file or dir before creating %s:%s' % (seg.mirror_host, mirrorNonBase) - , directory = mirrorNonBase - , remoteHost = seg.mirror_host - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - cmdComplete = False - primaryNonBase = seg.primary_data_directory + "/" + nonBase - while cmdComplete == False: - self.logger.debug('remotecopy everything but base: %s:%s' % (self.nodeName, primaryNonBase)) - rcCmd = RemoteCopyPreserve( name = 'gpupgrademirror remotecopy everything but base: %s:%s' % (self.nodeName, primaryNonBase) - , srcDirectory = primaryNonBase - , dstHost = seg.mirror_host - , dstDirectory = mirrorTempDataDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(rcCmd) - rcCmd.validate() - - """ Put the base directory back together the way it was originally (e.g. 1.1 and 1.2 go into 1) """ - dbDirs = DirectoryList( name = 'gpupgrademirror get database dirs for primary seg dbid ' + str(seg.primary_dbid) - , dirLocation = seg.primary_data_directory + "/base" - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - dbDirs.run(validateAfter = True) - dbDirList = dbDirs.get_result_list() - for db in dbDirList: - unqualDir = SegHardLinks.getUnqualifiedLinkDir(db) - qualDir = mirrorTempDataDir + "/base/" + unqualDir - cmdComplete = False - while cmdComplete == False: - rmCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make consolidation directory: %s' % qualDir - , directory = qualDir - , ctxt = REMOTE - , remoteHost = seg.mirror_host - , mode = "0700" - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - for dir in dirList: - unqualDir = SegHardLinks.getUnqualifiedLinkDir(dir) - srcDirFiles = mirrorTempDataDir + "/base/" + unqualDir - trgDir = mirrorTempDataDir + "/base/" + SegHardLinks.getUnqualifiedRealDir(dir) - cmdComplete = False - while cmdComplete == False: - mvCmd = GPumMoveDirectoryContents( name = 'gpupgrademirror consolidate a directory: %s' % srcDirFiles - , srcDirectory = srcDirFiles - , dstDirectory = trgDir - , ctxt = REMOTE - , remoteHost = seg.mirror_host - ) - cmdComplete = runAndCheckCommandComplete(mvCmd) - mvCmd.validate() - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveDirectory( name = 'gpupgrademirror remove unconsolidated dir: %s' % srcDirFiles - , directory = srcDirFiles - , ctxt = REMOTE - , remoteHost = seg.mirror_host - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - except Exception, e: - self.logger.error("An exception occured in copyPrimaryToMirror: " + str(e)) - raise Exception("Unable to continue upgrade.") - #------------------------------------------------------------------------------ - def newMirrorFileFixup(self, seg): - self.logger.debug('in newMirrorFileFixup for %s:%s' % (self.nodeName, seg.mirror_data_directory)) - - if options.mode == 'S': - tempSourceDataDir = seg.mirror_data_directory - else: - tempSourceDataDir = seg.mirror_data_directory + self.tempOldMirrorDirSuffix - - tempTargetDataDir = seg.mirror_data_directory + self.tempNewMirrorDirSuffix - - for file in self.fixupFileList: - fullPathFile = tempSourceDataDir + '/' + file - fullPathTarget = tempTargetDataDir + '/' + file - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoteCopyPreserve( name = 'gpupgrademirror fixup copy : %s to %s' % (fullPathFile, fullPathTarget) - , srcDirectory = fullPathFile - , dstHost = self.nodeNameNIC - , dstDirectory = fullPathTarget - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - #------------------------------------------------------------------------------- - def removeOldMirror(self, seg): - self.logger.debug('in removeOldMirror %s:%s' % (self.nodeName, seg.mirror_data_directory)) - - if options.mode == 'S' and (options.rollback == False or seg.rollForward == True): - ''' If in a safe mode upgrade, point to the temp location of the old mirror ''' - tempDataDir = seg.mirror_data_directory + self.tempOldMirrorDirSuffix - else: - ''' if in un-safe mode or in rollback, remove all files in the real mirror location''' - tempDataDir = seg.mirror_data_directory - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove old mirror data directory: %s:%s' % (self.nodeName, tempDataDir) - , directory = tempDataDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - - #------------------------------------------------------------------------------- - def removeOldMirrorSpecialFiles(self, seg): - self.logger.debug('in removeOldMirrorSpecialFiles %s:%s' % (self.nodeName, seg.mirror_data_directory)) - - tempDataDir = seg.mirror_data_directory + self.tempOldMirrorDirSuffix - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove old mirror special files: %s:%s' % (self.nodeName, tempDataDir) - , directory = tempDataDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - #------------------------------------------------------------------------------- - def removeCopyOfPrimary(self, seg): - ''' This method is used during recovery ''' - self.logger.debug('in removeCopyOfPrimary %s:%s' % (self.nodeName, seg.mirror_data_directory)) - - tempDataDir = seg.mirror_data_directory + self.tempNewMirrorDirSuffix - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove the copied mirror data directory: %s:%s' % (self.nodeName, tempDataDir) - , directory = tempDataDir - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - #------------------------------------------------------------------------------- - def checkAvailableSpace(self, seg): - self.logger.debug('in checkAvailableSpace %s:%s' % (seg.primary_host, seg.primary_data_directory)) - - ''' Get size of primary ''' - cmdComplete = False - while cmdComplete == False: - dirSizeCmd = DiskUsage( name = "gpupgrademirror check for disk usage on primary: " + str(seg.primary_data_directory) - , directory = seg.primary_data_directory - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(dirSizeCmd) - dirSizeCmd.validate() - dirSize = dirSizeCmd.get_bytes_used() - - ''' Get available space on mirror ''' - cmdComplete = False - while cmdComplete == False: - mirrorFreeSizeCmd = GPumDiskFree( name = 'gpupgrademirror disk space available on mirror: ' + str(seg.mirror_data_directory) - , directory = seg.mirror_data_directory - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(mirrorFreeSizeCmd) - mirrorFreeSizeCmd.validate() - mirrorFreeSize = mirrorFreeSizeCmd.get_bytes_free() - if options.mode == 'U': - ''' Add current mirror size to available mirror space ''' - cmdComplete = False - while cmdComplete == False: - currentMirrorSizeCmd = DiskUsage( name = "gpupgrademirror check for disk usage on mirror: " + str(seg.mirror_data_directory) - , directory = seg.mirror_data_directory - , ctxt = REMOTE - , remoteHost = self.nodeNameNIC - ) - cmdComplete = runAndCheckCommandComplete(currentMirrorSizeCmd) - currentMirrorSizeCmd.validate() - currentMirrorSize = currentMirrorSizeCmd.get_bytes_used() - mirrorFreeSize = mirrorFreeSize + currentMirrorSize - - ''' Compare available space to needed space (add some extra space just to make sure) ''' - requiredSpace = int(dirSize) + int(TEN_GIG) - if mirrorFreeSize < requiredSpace: - self.logger.error("Size of mirror directory is too small %s:%s." % (self.nodeName, seg.mirror_data_directory)) - self.logger.error("The mirror directory must have at least %s bytes free. " % str(requiredSpace)) - self.logger.error("The mirror has %s bytes free." % str(mirrorFreeSize)) - raise ValidationError('Mirror data directory is too small') - - #------------------------------------------------------------------------------- - def run(self): - try: - self.startedRun = True - self.setup() - """ - self.setupCompletedEvent.set() - self.continueEvent.wait() - """ - self.logger.debug("continuing run of phase1 for mirror: " + str(self.nodeName)) - statusIndex = 0 - for seg in self.mirrorSegmentList: - status = self.statusList[statusIndex] - - if options.rollback == True and seg.rollForward == False: - ''' Put the mirror's back if necessary, and remove temp files ''' - if options.mode == 'S': - self.removeCopyOfPrimary(seg) - if status.compare_status('END_MOVE_OLD_MIRROR') >= 0 \ - and status.compare_status('PHASE1_DONE') < 0: - ''' We have already moved the old mirror to a temp location,''' - ''' Go ahead and remove anything that might be in the segment's datadir''' - self.removeOldMirror(seg) - ''' Move the original mirror back to it's original location ''' - self.renameTempMirrorToOldMirror(seg, self.tempOldMirrorDirSuffix) - status.set_status('PHASE1_SAFE_MODE') - else: - self.removeOldMirrorSpecialFiles(seg) - status.set_status('PHASE1_UNSAFE_MODE') - - elif options.mode == 'S': - if status.compare_status('START_PRIMARY_COPY') <= 0: - ''' Copy the entire primary segment over to the mirror ''' - status.set_status('START_PRIMARY_COPY') - self.copyPrimaryToMirror(seg) - status.set_status('END_PRIMARY_COPY') - - if status.compare_status('START_NEW_MIRROR_FILE_FIXUP') <= 0: - ''' Fixup some of the new mirror files with the old mirror files ''' - status.set_status('START_NEW_MIRROR_FILE_FIXUP') - self.newMirrorFileFixup(seg) - status.set_status('END_NEW_MIRROR_FILE_FIXUP') - - if status.compare_status('START_MOVE_OLD_MIRROR') <= 0: - ''' Rename the old mirror to a temporary name ''' - status.set_status('START_MOVE_OLD_MIRROR') - self.renameOldMirrorToTemp(seg) - status.set_status('END_MOVE_OLD_MIRROR') - - if status.compare_status('START_MOVE_NEW_MIRROR') <= 0: - ''' Rename the new mirror the the name of the old mirror ''' - status.set_status('START_MOVE_NEW_MIRROR') - self.renameTempMirrorToOldMirror(seg, dirSuffix = self.tempNewMirrorDirSuffix) - status.set_status('END_MOVE_NEW_MIRROR') - - if status.compare_status('START_REMOVE_OLD_MIRROR') <= 0: - ''' Remove the old mirror ''' - status.set_status('START_REMOVE_OLD_MIRROR') - self.removeOldMirror(seg) - status.set_status('END_REMOVE_OLD_MIRROR') - - if status.compare_status('PHASE1_DONE') < 0: - status.set_status('PHASE1_DONE') - if seg.rollForward == True: - status.set_status('PHASE1_SAFE_MODE') - - else: - ''' Un-safe mode ''' - if status.compare_status('START_MOVE_OLD_MIRROR_SPECIAL_FILES') <= 0: - ''' Rename the old mirror special files to a temporary name ''' - status.set_status('START_MOVE_OLD_MIRROR_SPECIAL_FILES') - self.renameOldMirrorToTemp(seg) - status.set_status('END_MOVE_OLD_MIRROR_SPECIAL_FILES') - - if status.compare_status('START_REMOVE_OLD_MIRROR') <= 0: - ''' Remove the old mirror ''' - status.set_status('START_REMOVE_OLD_MIRROR') - self.removeOldMirror(seg) - status.set_status('END_REMOVE_OLD_MIRROR') - - if status.compare_status('START_PRIMARY_COPY') <= 0: - ''' Copy the entire primary segment over to the mirror ''' - status.set_status('START_PRIMARY_COPY') - self.copyPrimaryToMirror(seg) - status.set_status('END_PRIMARY_COPY') - - if status.compare_status('START_NEW_MIRROR_FILE_FIXUP') <= 0: - ''' Fixup some of the new mirror files with the old mirror files ''' - status.set_status('START_NEW_MIRROR_FILE_FIXUP') - self.newMirrorFileFixup(seg) - status.set_status('END_NEW_MIRROR_FILE_FIXUP') - - if status.compare_status('START_MOVE_NEW_MIRROR') <= 0: - ''' Rename the new mirror the the name of the old mirror ''' - status.set_status('START_MOVE_NEW_MIRROR') - self.renameTempMirrorToOldMirror(seg, dirSuffix = self.tempNewMirrorDirSuffix) - status.set_status('END_MOVE_NEW_MIRROR') - - if status.compare_status('START_REMOVE_OLD_MIRROR_SPECIAL_FILES'): - ''' Remove the special files we saved away''' - status.set_status('START_REMOVE_OLD_MIRROR_SPECIAL_FILES') - self.removeOldMirrorSpecialFiles(seg) - status.set_status('END_REMOVE_OLD_MIRROR_SPECIAL_FILES') - - if status.compare_status('PHASE1_DONE') < 0: - status.set_status('PHASE1_DONE') - if seg.rollForward == True: - status.set_status('PHASE1_UNSAFE_MODE') - statusIndex = statusIndex + 1 - self.shutdown() - self.completedRun = True - except Exception, e: - self.logger.error('ERROR in processing mirror: %s Exception: %s' % (self.nodeName, str(e))) - self.logger.error('gpupgradmirror exiting') - traceback.print_exc() - sys.exit("The gpupgrademirror log information can be found in " + str(get_logfile()) + "\n") - - #------------------------------------------------------------------------------- - def shutdown(self): - for seg in self.mirrorSegmentList: - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove temp dir: %s:%s' % (seg.primary_host, seg.primaryTempDir) - , directory = seg.primaryTempDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - - -#------------------------------------------------------------------------------- -#------------------------------------------------------------------------------- -class Phase2(threading.Thread): - - def __init__(self, dburl, logger, info_data_directory, node_name): - threading.Thread.__init__(self) - - self.dburl = dburl - self.logger = logger - self.info_data_directory = info_data_directory - self.nodeName = node_name - self.mirrorSegmentList = [] - self.statusList = [] - self.fixupFileList = [] - self.startedRun = False - - - #------------------------------------------------------------------------------- - def setup(self, mirrorInfoList, fileList, specialDirectoryList): - self.logger.debug("in Phase2.setup: " + str(self.nodeName)) - self.mirrorSegmentList = mirrorInfoList - - for seg in self.mirrorSegmentList: - - ''' Get status file from Phase 1 for each mirror segment ''' - status = GpUpgradeMirrorStatus( logger = self.logger - , info_data_directory = self.info_data_directory - , segment_dbid = seg.mirror_dbid - , phase = 2 - ) - self.statusList.append(status) - - """ gpmigrator has removed or moved the mirror data directory, so we will attempt to make it.""" - cmdComplete = False - while cmdComplete == False: - rmCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make the mirror data directory: %s' % seg.mirror_data_directory - , directory = seg.mirror_data_directory - , ctxt = REMOTE - , remoteHost = seg.mirror_host - , mode = "0700" - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - """ Also may need to add base directory, so we will attemplt to make it.""" - cmdComplete = False - while cmdComplete == False: - rmCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make the mirror data directory: %s' % seg.mirror_data_directory - , directory = seg.mirror_data_directory + "/base" - , ctxt = REMOTE - , remoteHost = seg.mirror_host - , mode = "0700" - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - """ Create a temp gpmigrator directory in each primary if it isn't already there.""" - """ In theory, this only happens in unit testing of gpupgrademirror, otherwise it should always be there.""" - seg.gpmigratorTempDir = seg.primary_data_directory + "/" + GPMIGRATOR_TEMP_DIR - cmdComplete = False - while cmdComplete == False: - rmCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make temp dir: %s' % seg.gpmigratorTempDir - , directory = seg.gpmigratorTempDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - , mode = "0700" - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - """ Create a temp directory in each primary segment for our use""" - seg.primaryTempDir = seg.primary_data_directory + "/" + GPMIGRATOR_TEMP_DIR + "/" + GPUPGRADEMIRROR_TEMP_DIR - cmdComplete = False - while cmdComplete == False: - rmCmd = CreateDirIfNecessaryWithMode( name = 'gpupgrademirror make temp dir: %s' % seg.primaryTempDir - , directory = seg.primaryTempDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - , mode = "0700" - ) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - - if options.continue_upgrade == True: - ''' Setup to continue upgrade ''' - else: - if (status.get_current_status() != (None, None)): - self.logger.error("Upgrade mirrors phase 2 already started.") - self.logger.error("You must continue mirror upgrade") - raise ValidationError("Unable to run phase 2") - else: - status.create_status_file() - self.logger.debug("completed Phase2.setup for mirror seg dbid: " + str(seg.mirror_dbid)) - - self.fixupFileList = fileList - self.specialDirectoryList = specialDirectoryList - - - #------------------------------------------------------------------------------ - def newMirrorSpecialFileFixup(self, seg): - self.logger.debug('in newMirrorSpecialFileFixup for seg dbid %s location: %s:%s' % (str(seg.mirror_dbid), seg.mirror_host, seg.mirror_data_directory)) - - """ Delete all files on the delete list """ - for file in self.specialDirectoryList: - fullPathTarget = seg.mirror_data_directory + '/' + file - cmdComplete = False - while cmdComplete == False: - cmd = RemoveFiles( name = 'gpupgrademirror remove special old directory' - , remoteHost = self.nodeName - , directory = fullPathTarget - , ctxt = REMOTE - ) - cmdComplete = runAndCheckCommandComplete(cmd) - - """ Check for files with extensions (i.e. .1, .2, ...) """ - """ First, get a list of all files in the base directory """ - fixupFileExtsList = [] - baseDir = seg.primary_data_directory + "/base" - primaryTempFile = seg.primaryTempDir + '/tempfilesindir' + str(seg.mirror_dbid) - cmdComplete = False - while cmdComplete == False: - fdCmd = FilesInDir( name = "gpupgrademirror files in dir %s:%s" % (seg.primary_host, baseDir) - , filePattern = baseDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - , remoteTempFile = primaryTempFile - ) - cmdComplete = runAndCheckCommandComplete(fdCmd) - fdCmd.validate() - allBaseList = fdCmd.get_result_list(localTempFile = options.info_data_directory + "/tempallbase" + str(seg.mirror_dbid)) - - allBaseList.sort() - - """ Searh the base directory list for all fixup files that have dots (i.e. 123.1, 123.2) """ - dotFiles = [] - for file in self.fixupFileList: - fullPathFile = seg.primary_data_directory + '/' + file - tempList = findRelFileDotNodes(fullPathFile, allBaseList) - for suffix in tempList: - dotFiles.append(file + "." + suffix) - - allFixupFileList = self.fixupFileList + dotFiles - - sph = SpecialFileHandling( sourceList = allFixupFileList - , seg = seg - ) - sph.createLinks() - - cmdComplete = False - while cmdComplete == False: - fdlCmd = FileDirectoryList( name = "gpupgrademirror find all dirs and files for %s:%s " % (seg.primary_host, sph.fullPathLinkDir) - , filePattern = sph.fullPathLinkDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(fdlCmd) - fdlCmd.validate() - linkDirList = fdlCmd.get_result_list() - - copyList = [] - for element in self.specialDirectoryList: - elementList = element.split("/") - if len(elementList) > 1: - elementPrefix = "/" + "/".join(elementList[:-1]) - else: - elementPrefix = "" - copyList.append([seg.primary_data_directory + "/" + element, seg.mirror_data_directory + elementPrefix ]) - for element in linkDirList: - copyList.append([sph.fullPathLinkDir + "/" + element, seg.mirror_data_directory]) - - for copyElement in copyList: - cmdComplete = False - while cmdComplete == False: - rcCmd = RemoteCopyPreserve( name = 'gpupgrademirror phase 2 fixup copy : %s:%s to %s:%s' % (seg.primary_host, copyElement[0], seg.mirror_host, copyElement[1]) - , srcDirectory = copyElement[0] - , dstHost = seg.mirror_host - , dstDirectory = copyElement[1] - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - cmdComplete = runAndCheckCommandComplete(rcCmd) - rcCmd.validate() - - return - - #------------------------------------------------------------------------------- - def run(self): - self.startedRun = True - try: - i = 0 - for seg in self.mirrorSegmentList: - self.logger.debug("started run for Phase 2 for seg mirror: %s" % str(seg.mirror_dbid)) - status = self.statusList[i] - currentStatus = status.get_current_status() - - if status.compare_status('START_PHASE2') <= 0: - status.set_status('START_PHASE2') - ''' Copy catalog tables to mirror ''' - self.newMirrorSpecialFileFixup(seg) - if status.compare_status('PHASE2_DONE') <= 0: - status.set_status('PHASE2_DONE') - i = i + 1 - self.shutdown() - except Exception, e: - self.logger.error('ERROR in processing mirror: %s Exception: %s' % (self.nodeName, str(e))) - self.logger.error('gpupgradmirror exiting') - traceback.print_exc() - sys.exit("The gpupgrademirror log information can be found in " + str(get_logfile()) + "\n") - - #------------------------------------------------------------------------------- - def shutdown(self): - for seg in self.mirrorSegmentList: - cmdComplete = False - while cmdComplete == False: - rmCmd = RemoveFiles( name = 'gpupgrademirror remove temp dir: %s:%s' % (seg.primary_host, seg.primaryTempDir) - , directory = seg.primaryTempDir - , ctxt = REMOTE - , remoteHost = seg.primary_host - ) - ###print "rmCmd = " - ###print str(rmCmd) - cmdComplete = runAndCheckCommandComplete(rmCmd) - rmCmd.validate() - -# -#------------------------------------------------------------------------------- -#--------------------------------- Main ---------------------------------------- -#------------------------------------------------------------------------------- -""" - - This the the main body of code for gpupgrademirror. gpupgradmirror has two phases - (phase 1 and phase 2) and two modes (safe and unsafe). - -""" - - -dburl = None -conn = None -remove_pid = True -p1MirrorInfo = None -phase2gparray = None -MirrorNodeList = [] -MirrorInfoList = [] -phase2MirrorList = [] -minionList = [] - -coverage = GpCoverage() -coverage.start() - -try: - # setup signal handlers so we can clean up correctly - signal.signal(signal.SIGTERM, sig_handler) - signal.signal(signal.SIGHUP, sig_handler) - - """ If we are a minion, we need to do a lookup of our seg id to properly setup the log file name. """ - minionSegdbid = "" - argPrevious = "" - for arg in sys.argv: - if argPrevious == "-i": - segdbidList = arg.split(":") - minionSegdbid = segdbidList[0] - break - else: - argPrevious = arg - - logger = get_default_logger() - applicationNameWithExt = EXECNAME + str(minionSegdbid) - setup_tool_logging( appName = applicationNameWithExt - , hostname = getLocalHostname() - , userName = getUserName() - ) - - options, args = parseargs() - - if len(options.ids) > 0: - options.ids = options.ids.split(':') - - if options.verbose: - enable_verbose_logging() - - if is_gpupgrademirror_running(options.info_data_directory): - logger.error('gpupgrademirror is already running. Only one instance') - logger.error('of gpupgrademirror is allowed at a time.') - remove_pid = False - sys.exit("The gpupgrademirror log information can be found in " + str(get_logfile()) + "\n") - else: - create_pid_file(options.info_data_directory) - - if options.phase2 == True: - phase = 2 - else: - phase = 1 - overallStatus = GpUpgradeMirrorStatus( logger = logger - , info_data_directory = options.info_data_directory - , segment_dbid = 0 - , phase = phase - , overall = True - ) - if options.continue_upgrade == True and \ - len(options.ids) == 0 and \ - overallStatus.get_current_status() == (None, None): - """ If there is no status, then we haven't gotten anywhere. Reset options.continue_upgrade to False""" - logger.warning("Although continue was specified, the upgrade are not far enough elong to continue. gpupgrademirror will attempt to restart this phase of upgrade") - options.continue_upgrade = False - - """ - We will start the database in master only mode unless we are in phase 1 and - we are rolling back or continuing. If we are in phase 1 and are rolling back - or continuing, we will get the information we need from a flat file we - created on the first phase 1 attempt. - """ - dbUrl = dbconn.DbURL(dbname = 'template1') - phase1gpstartStatus = None - if options.rollback == True: - """ Never start the database in rollback mode. """ - pass - elif options.phase2 == False and overallStatus.compare_status("END_PHASE1_SETUP") >= 0: - """ If we are past getting all the information we need, then don't start the database. """ - pass - elif len(options.ids) == 0: - if options.phase2 == True: - GpStart.local('gpupgrademirror start database', masterOnly = True) - logging.debug('started database master only with GpStart') - else: - """ We are running in a 4.0 environment, but need to start like we are in a 3.3.x environment. """ - env = SetupEnv(options.gphome) - Startup(env = env, utility = True) - logging.debug('started database master only with Startup') - phase1gpstartStatus = "STARTUP_MASTER" - - if options.rollback == True and options.phase2 == False: - ''' setup for rollback ''' - if overallStatus.get_current_status() == (None, None): - logging.warning('There is no upgrade mirror in progress') - rmCmd = RemoveFiles( name = 'gpupgrademirror remove info directory: %s' % (options.info_data_directory) - , directory = options.info_data_directory - , ctxt = LOCAL - , remoteHost = None - ) - rmCmd.run(validateAfter = False) - exit(0) - elif overallStatus.compare_status("END_PHASE1_SETUP") < 0: - """ - We have not gotten past the setup phase, - so remove the gpugprademirr info directory and exit. - """ - rmCmd = RemoveFiles( name = 'gpupgrademirror remove info directory: %s' % (options.info_data_directory) - , directory = options.info_data_directory - , ctxt = LOCAL - , remoteHost = None - ) - rmCmd.run(validateAfter = False) - exit(0) - elif options.continue_upgrade == True: - pass - elif options.phase2 == False: - if (overallStatus.get_current_status() != (None, None)): - logging.error("Upgrade mirrors already started.") - logging.error("You must either rollback or continue mirror upgrade") - raise ValidationError("Unable to continue") - else: - """ We are in normal phase 1 mode """ - overallStatus.create_status_file() - elif options.phase2 == True: - pass - - if options.phase2 == True: - """ - Phase 2 of upgrade mirrors - - There are a number of special files that we need to copy from the primary in phase 2. - Most of these files are catalog files. - """ - - databaseList = [] - - if options.rollback == True: - raise ValidationError('Rollback is not possible in phase 2 mirror upgrade.') - - conn = dbconn.connect( dburl = dbUrl - , utility = True - ) - - ''' Get a list of databases. ''' - databaseCursor = dbconn.execSQL(conn, DatabaseRow.query()) - for row in databaseCursor: - dbrow = DatabaseRow(row) - databaseList.append(dbrow) - conn.close() - - ''' Setup gparray object. ''' - try: - phase2gparray = GpArray.initFromCatalog(dbUrl, utility = True) - except Exception, e: - logger.warning('Unable to obtain gparray information: ' + str(e)) - logger.warning('gpupgradmirror exiting') - exit(1) - - ''' Setup mirror info object. ''' - try: - p2MirrorInfo = Phase1and2MirrorInfo(dbUrl) - except Exception, e: - logger.warning('Unable to obtain mirror information: ' + str(e)) - logger.warning('gpupgradmirror exiting') - exit(1) - - ''' Start all the primary segments ''' - StartupPrimaries(phase2gparray) - - ''' Get list of all segments ''' - allSegMirrorInfo = p2MirrorInfo.getAllMirrorInfoList() - - ''' Go to each segment''' - for mirrorInfo in allSegMirrorInfo: - - ''' Connect to each segment's database, and make a list of files we will use ''' - fileList = [] - specialDirList = [] - ''' always copy the global directory ''' - specialDirList.append('global') - ''' always copy the pg_xlog directory ''' - specialDirList.append('pg_xlog') - for db in databaseList: - if str(db.databaseName) == str("template0"): - ''' Special case where we copy the entire database ''' - specialDirList.append('base/' + str(db.databaseDirectory)) - continue - - connectURL = dbconn.DbURL(dbname = db.databaseName, hostname = mirrorInfo.primary_host, port = mirrorInfo.primary_host_port) - connectForDB = dbconn.connect( dburl = connectURL - , utility = True - ) - ''' List of non-toast catalog tables ''' - DBcursor = dbconn.execSQL(connectForDB, NonCatalogToastTablesRow.query()) - for fileRow in DBcursor: - nonToastRow = NonCatalogToastTablesRow(fileRow) - fileList.append('base/' + str(db.databaseDirectory) + '/' + str(nonToastRow.filename)) - - ''' List of Information Schema catalog tables ''' - DBcursor = dbconn.execSQL(connectForDB, InformationSchemaTablesRow.query()) - for fileRow in DBcursor: - informationSchemaRow = InformationSchemaTablesRow(fileRow) - fileList.append('base/' + str(db.databaseDirectory) + '/' + str(informationSchemaRow.filename)) - - ''' List of toast catalog tables and their toast files ''' - DBcursor = dbconn.execSQL(connectForDB, CatalogToastTablesRow.query()) - for fileRow in DBcursor: - toastTableRow = CatalogToastTablesRow(fileRow) - fileList.append('base/' + str(db.databaseDirectory) + '/' + str(toastTableRow.filename)) - fileList.append('base/' + str(db.databaseDirectory) + '/' + str(toastTableRow.toastfilename)) - fileList.append('base/' + str(db.databaseDirectory) + '/' + str(toastTableRow.toastfileindex)) - connectForDB.close() - """ END for db in databaseList """ - - """ Create a phase2 object for each mirror segment """ - tempNode = Phase2(dbUrl, logger, options.info_data_directory, mirrorInfo.nic_address) - - """ Setup the mirror node """ - tempNode.setup([mirrorInfo], fileList, specialDirList) - - phase2MirrorList.append(tempNode) - """ END for mirrorInfo in allSegMirrorInfo: """ - - """ Stop all the primaries. """ - ShutdownPrimaries(phase2gparray) - - logging.debug('stopping database master') - GpStop.local('gpupgrademirror stop databse', fast=True, masterOnly = True) - - """ - There is a theoretical issue with running too many threads at once, - so we will limit the number of threads running at once to 64. - """ - maxThreads = 64 - startIndex = 0 - endIndex = startIndex + maxThreads - - while startIndex < len(phase2MirrorList): - phase2MirrorSubList = phase2MirrorList[startIndex:endIndex] - """ Start the threads """ - for node in phase2MirrorSubList: - node.start() - - """ Wait no more than 60 seconds for all the threads to start """ - numTries = 12 - for thisTry in range(numTries): - started = True - for node in phase2MirrorSubList: - if node.startedRun == False: - sleep(5) - break - if started == True: - break - if started == False: - raise Exception("Unable to initialize threads for Phase 2 mirror upgrade") - - done = False - completedNodeList = [] - notCompletedNodeList = list(phase2MirrorSubList) - while done == False: - justCompletedNodeList = [] - for node in notCompletedNodeList: - node.join(10) - logging.debug("checking to see if thread for mirror node is alive: " + str(node.nodeName)) - if node.isAlive() == False: - """ The node has completed """ - for segStatus in node.statusList: - if segStatus.get_current_status()[0] != 'PHASE2_DONE': - done = True - break - if done == True: - break - justCompletedNodeList.append(node) - completedNodeList.extend(justCompletedNodeList) - for completedNode in justCompletedNodeList: - notCompletedNodeList.remove(completedNode) - if len(completedNodeList) == len(phase2MirrorSubList): - done = True - startIndex = endIndex - endIndex = endIndex + maxThreads - - """ Make sure all phase 2 threads complated successfully """ - p2Completed = True - for node in phase2MirrorList: - for segStatus in node.statusList: - if segStatus.get_current_status()[0] != 'PHASE2_DONE': - logging.error('A segment has reported an error. Segment dbid: %s' % segStatus.segment_dbid) - p2Completed = False - break - if p2Completed == False: - break - - if p2Completed == False: - logging.error('Unable to complete upgrade mirrors for phase 2') - raise ValidationError('Unable to complete upgrade') - else: - Phase1and2MirrorInfo.removeMirrorInfoFile() - overallStatus.remove_status_file() - for node in phase2MirrorList: - for segStatus in node.statusList: - segStatus.remove_status_file() - - rmCmd = RemoveFiles( name = 'gpupgrademirror remove info directory: %s' % (options.info_data_directory) - , directory = options.info_data_directory - , ctxt = LOCAL - , remoteHost = None - ) - rmCmd.run(validateAfter = False) - else: - """ - Phase 1 - """ - - if overallStatus.compare_status("START_PHASE1_SETUP") < 0: - overallStatus.set_status("START_PHASE1_SETUP") - - try: - p1MirrorInfo = Phase1and2MirrorInfo(dbUrl) - except Exception, e: - logger.warning('Unable to obtain mirror information: ' + str(e)) - logger.warning('gpupgradmirror exiting') - traceback.print_exc() - exit(2) - - allSegMirrorInfo = p1MirrorInfo.getAllMirrorInfoList() - - if overallStatus.compare_status("END_PHASE1_SETUP") < 0: - overallStatus.set_status("END_PHASE1_SETUP") - - """ - At this point, all setup that requires database access is done. We can continue from here - without connecting to the database. - """ - if phase1gpstartStatus != None and phase1gpstartStatus == "STARTUP_MASTER": - env = SetupEnv(options.gphome) - logging.debug('stopping database master only') - Shutdown(env = env, utility = True) - phase1gpstartStatus = "STOPPED" - - """ Test the first segment for speed with its mirror. """ - aSeg = allSegMirrorInfo[0] - cmdComplete = False - - netMBSpeed = 0 - if options.rollback == False: - if len(options.speed_network) > 0: - netMBSpeed = int(options.speed_network) - else: - while cmdComplete == False: - netMBCmd = NetworkSpeed( name = "gpupgrademirror get nextwork speed for: " + str(aSeg.primary_host) - , host1 = aSeg.primary_host - , host2 = aSeg.mirror_host - , tempDir1 = aSeg.primary_data_directory - , tempDir2 = aSeg.mirror_data_directory - , ctxt = REMOTE - , remoteHost = aSeg.primary_host - ) - logger.debug("Testing network speed. This will take about 20 seconds....") - cmdComplete = runAndCheckCommandComplete(netMBCmd) - netMBSpeed = netMBCmd.getSpeed() - logger.debug("Net speed for primary %s to mirror %s is estimated to be %s Meg per Sec." % (aSeg.primary_host, aSeg.mirror_host, str(netMBSpeed))) - - if len(options.ids) > 0: - """ I am a minion, so skip the minion setup. """ - if overallStatus.compare_status("MINION_START_SETUP") < 0: - overallStatus.set_status("MINION_START_SETUP") - if overallStatus.compare_status("MINION_SETUP_DONE") < 0: - overallStatus.set_status("MINION_SETUP_DONE") - - if len(options.ids) == 0: - """ - We are the king of the gpupgrademirror processes, so we do not do any upgrading ourself. - Instead, we will create a bunch of gpupgrademirror processes on a set of our - nodes, and slave them to this king process. - """ - - allNodes = p1MirrorInfo.getMirrorNodeList() - - """ - We are limited to how many ssh connections we can make on a given node (estimate 64). - Also, there may be issues with Python and a large number of threads. - The number of ssh's at any given moment on a node should be about 8 (2 per NIC, 4 NICs). - We should be able to support about 8 mirror nodes per gpupgrademirror instance. - """ - logger.debug("Start setup for minion gpupgrademirror processes") - nodesPerMinion = 8 - index = 0 - while index < len(allNodes): - numNodes = 0 - segdbidList = [] - minionNode = allNodes[index] - while numNodes < nodesPerMinion and index < len(allNodes): - node = allNodes[index] - nodeSegs = p1MirrorInfo.getMirrorInfo(node) - for segInfo in nodeSegs: - segdbidList.append(str(segInfo.mirror_dbid)) - index = index + 1 - numNodes = numNodes + 1 - if options.rollback == True: - minionOptions = " -r " - else: - minionOptions = " -c " - if options.verbose == True: - minionOptions = minionOptions + " -v " - minionOptions = minionOptions + " -s " + str(netMBSpeed) + " " - minionSegdbid = segdbidList[0] - firstMirrorDataDir = p1MirrorInfo.getMirrorInfoFromMirrordbid(mirrordbid = minionSegdbid).mirror_data_directory - minion = GPUpgradeMirrorMinion( name = "gpupgrademirror create a minion" - , options = minionOptions - , segs = ":".join(segdbidList) - , minionSeg = minionSegdbid - , segmentDir = firstMirrorDataDir - , ctxt = REMOTE - , remoteHost = str(minionNode) - ) - minionList.append(minion) - - minionSegdbidList = [] - for minion in minionList: - minionSegdbidList.append(minion.minionSeg) - minionSegs = "-".join(minionSegdbidList) - if overallStatus.compare_status("MINION_START_SETUP") <= 0: - overallStatus.set_status(status = "MINION_START_SETUP", status_info = minionSegs) - """ Copy all the needed files to the minion hosts """ - for minion in minionList: - logger.debug("Creating and copying files for minion: %s:%s" % (minion.remoteHost, minion.minionDir)) - minion.createInfoDirectory() - minion.copyStatusFileToHost() - minion.copyInfoFileToHost() - minion.copyGpHostCacheFileToHost() - - if overallStatus.compare_status("MINION_SETUP_DONE") <= 0: - overallStatus.set_status("MINION_SETUP_DONE") - - """ All the minions are set up and ready to go. """ - minionWP = WorkerPool(numWorkers = len(minionList)) - for minion in minionList: - minionWP.addCommand(cmd = minion) - numDone = 0 - minionError = False - while numDone < len(minionList): - resultList = minionWP.getCompletedItems() - numDone = numDone + len(resultList) - if len(resultList) > 0: - """ Check the results for the minions. """ - for minion in resultList: - results = minion.get_results() - resultStr = results.printResult() - if results.rc != 0: - logger.error("gpupgrademirror failed: sub command on %s:%s. Dbid = %s" % (minion.remoteHost, minion.minionDir, str(minion.minionSeg))) - minionWP.haltWork() - minionWP.joinWorkers() - minionError = True - if minionError == False: - minionWP.haltWork() - minionWP.joinWorkers() - minionWP = None - - logFile = get_logfile() - for minion in minionList: - minion.copyLogFile(parentLogFile = logFile) - - if minionError == True: - raise Exception("There was a problem with one of the gpupgrademirror sub processes.") - else: - overallStatus.set_status('PHASE1_DONE') - rmCmd = RemoveFiles( name = 'gpupgrademirror remove info directory: %s' % (options.info_data_directory) - , directory = options.info_data_directory - , ctxt = LOCAL - , remoteHost = None - ) - rmCmd.run(validateAfter = False) - exit(0) - - - """ Create a list of nodes and their segment information """ - ''' Get a list of mirror nodes ''' - mirrorNodeCursor = p1MirrorInfo.getMirrorNodeList() - numMirrorNodes = len(mirrorNodeCursor) - - shiftIndex = 0 - for mirrorName in mirrorNodeCursor: - ''' Get info on mirror segments for this host''' - mirrorInfoList = p1MirrorInfo.getMirrorInfo(mirrorName) - - """ Randomize the seg order to minimize the change of disk read/write at the same time.""" - for rotateV in range(shiftIndex): - tempItem = mirrorInfoList.pop(0) - mirrorInfoList.append(tempItem) - - ''' Create a gpupgrademirror object for each mirror node''' - tempNode = Phase1( dburl = dbUrl - , logger = logger - , info_data_directory = options.info_data_directory - , node_name = mirrorName - , mirror_info_list = mirrorInfoList - , setup_completed_event = threading.Event() - , continue_event = threading.Event() - ) - MirrorNodeList.append(tempNode) - shiftIndex = shiftIndex + 1 - - if phase1gpstartStatus != None and phase1gpstartStatus == "STARTUP_MASTER": - logging.debug('stopping database master only') - env = SetupEnv(options.gphome) - Shutdown(env = env, utility = True) - phase1gpstartStatus = "STOPPED" - - """ - Start the threads - """ - for node in MirrorNodeList: - node.start() - - - """ Wait no more than 60 seconds for all the threads to start """ - numTries = 12 - for thisTry in range(numTries): - started = True - for node in MirrorNodeList: - if node.startedRun == False: - sleep(5) - break - if started == True: - break - if started == False: - raise Exception("Unable to initialize threads for Phase 1 mirror upgrade") - - """ - KAS note to self. Need to add code here to detect and notify all the threads if - there is an issue with one of the threads, and all others must stop. - """ - - done = False - completedNodeList = [] - notCompletedNodeList = list(MirrorNodeList) - while done == False: - justCompletedNodeList = [] - for node in notCompletedNodeList: - node.join(10) - logging.debug("checking to see if thread for mirror node is alive: " + str(node.nodeName)) - if node.isAlive() == False: - """ The node has completed """ - for segStatus in node.statusList: - if segStatus.get_current_status()[0] != 'PHASE1_DONE': - done = True - break - if done == True: - break - justCompletedNodeList.append(node) - completedNodeList.extend(justCompletedNodeList) - for completedNode in justCompletedNodeList: - notCompletedNodeList.remove(completedNode) - if len(completedNodeList) == len(MirrorNodeList): - done = True - - if options.rollback == False: - p1Completed = True - for node in MirrorNodeList: - for segStatus in node.statusList: - if segStatus.get_current_status()[0] != 'PHASE1_DONE': - p1Completed = False - break - if p1Completed == False: - break - - if p1Completed == False: - logging.error('Unable to complete upgrade mirrors for phase 1') - raise ValidationError('Unable to complete upgrade') - else: - overallStatus.set_status('PHASE1_DONE') - rmCmd = RemoveFiles( name = 'gpupgrademirror remove info directory: %s' % (options.info_data_directory) - , directory = options.info_data_directory - , ctxt = LOCAL - , remoteHost = None - ) - rmCmd.run(validateAfter = False) - else: - ''' We are in rollback mode ''' - p1RollbackCompleted = True - for node in MirrorNodeList: - for segStatus in node.statusList: - currentStatus = segStatus.get_current_status()[0] - if currentStatus != None: - if (options.mode == 'S' and currentStatus != 'PHASE1_SAFE_MODE') \ - or (options.mode == 'U' and currentStatus != 'PHASE1_UNSAFE_MODE'): - p1RollbackCompleted = False - break - if p1RollbackCompleted == False: - break - - if p1RollbackCompleted == False: - logging.error('Unable to complete rollback of upgrade mirrors for phase 1') - raise ValidationError('Unable to complete rollback') - else: - RemoveFiles.local("gpupgrademirror remove %s" % options.info_data_directory - , options.info_data_directory - ) - - sys.exit(0) - -except Exception,e: - logger.error("gpupgrademirror failed: %s \n\nExiting..." % e ) - traceback.print_exc() - sys.exit("The gpupgrademirror log information can be found in " + str(get_logfile()) + "\n") - -except KeyboardInterrupt: - # Disable SIGINT while we shutdown. - signal.signal(signal.SIGINT,signal.SIG_IGN) - - # Re-enabled SIGINT - signal.signal(signal.SIGINT,signal.default_int_handler) - - sys.exit('\nUser Interrupted' + "The gpupgrademirror log information can be found in " + str(get_logfile()) + "\n") - -except Exception, e: - print "FATAL Exception: " + str(e) - traceback.print_exc() - sys.exit("The gpupgrademirror log information can be found in " + str(get_logfile()) + "\n") - - -finally: - try: - logger.info("==========================================================================") - logger.info("The gpupgrademirror log information can be found in " + str(get_logfile())) - if len(minionList) > 0: - logger.info("Under certain failure situations, not all log information can be collected.") - logger.info("There may be additional log information under the gpAdminLogs directory on the following hosts:") - logger.info("") - for minion in minionList: - logger.info(" " + minion.remoteHost) - logger.info("") - logger.info("==========================================================================") - - if remove_pid: - remove_pid_file(options.info_data_directory) - - """ Normally, the cluster should already be stopped, but if there was an issue, then the cluster might not be down. """ - """ Try to stop gpdb if anything is still running. """ - logging.debug('Attempting to stop any remaining processes on the cluster if necessary.') - if options.phase2 == True: - try: - """ Try shutting down the master. """ - GpStop.local('gpupgrademirror stop databse', fast=True, masterOnly = True) - except Exception: - pass - try: - """ Try shutting down the primary segments. """ - ShutdownPrimaries(phase2gparray) - except Exception: - pass - else: - try: - """ We are in phase 1, try shutting down the master. """ - env = SetupEnv(options.gphome) - Shutdown(env = env, utility = True) - except Exception: - pass - try: - if minionWP != None: - minionWP.haltWork() - minionWP.joinWorkers() - except Exception: - pass - except NameError: - pass - logger.info("gpupgrademirror exit") - - coverage.stop() - coverage.generate_report() - diff --git a/gpdb-doc/dita/GPUtilityGuide.ditamap b/gpdb-doc/dita/GPUtilityGuide.ditamap index cc6a9bb2e04a2541ffd4e568084f90de0aa7c868..aaa039e7492d8e7154a22fbc9c1ba1613b140176 100644 --- a/gpdb-doc/dita/GPUtilityGuide.ditamap +++ b/gpdb-doc/dita/GPUtilityGuide.ditamap @@ -88,8 +88,6 @@ - - diff --git a/gpdb-doc/dita/utility_guide/admin_utilities/gpbitmapreindex.xml b/gpdb-doc/dita/utility_guide/admin_utilities/gpbitmapreindex.xml index 1158166b95bd9ddc4f640dc98e5a48f2ca05cb86..e5f3104a82dbe224e81d4cec326552bb48961cfb 100644 --- a/gpdb-doc/dita/utility_guide/admin_utilities/gpbitmapreindex.xml +++ b/gpdb-doc/dita/utility_guide/admin_utilities/gpbitmapreindex.xml @@ -4,9 +4,8 @@ gpbitmapreindex

Rebuilds bitmap indexes after a 3.3.x to 4.0.x upgrade.

Synopsisgpbitmapreindex -m { r | d | {l [-o output_sql_file]} } [-h master_host] [-p master_port] [-n number_of_processes] [-v] -gpmigrator --version -gpmigrator --help | -?
Description

The on-disk format of bitmap indexes has changed from release 3.3.x to 4.0.x. Users who upgrade +

Description

The on-disk format of bitmap indexes has changed from release 3.3.x to 4.0.x. Users who upgrade must rebuild all bitmap indexes after upgrading to 4.0. The gpbitmapreindex utility facilitates the upgrade of bitmap indexes by either running the REINDEX command to reindex them, or running the DROP @@ -24,4 +23,4 @@ can be used to recreate the bitmap indexes.-p PGPORT or defaults to 5432.-v | --verboseShow verbose output.--versionDisplays the version of this utility. -? | --helpDisplays the online help.

Examples

Reindex all bitmap indexes:

gpbitmapreindex -m r

Output a file of SQL commands that can be used to recreate all bitmap indexes:

gpbitmapreindex -m list --outfile /home/gpadmin/bmp_ix.sql

Drop all bitmap indexes and run in verbose mode:

gpbitmapreindex -m d -v
See Also

Greenplum Database Reference Guide: REINDEX, DROP - INDEX, CREATE INDEX

 \ No newline at end of file + INDEX, CREATE INDEX

diff --git a/gpdb-doc/dita/utility_guide/admin_utilities/gpmigrator.xml b/gpdb-doc/dita/utility_guide/admin_utilities/gpmigrator.xml deleted file mode 100644 index 4796e1d4c8707cc45ff18cca660d3242002ba881..0000000000000000000000000000000000000000 --- a/gpdb-doc/dita/utility_guide/admin_utilities/gpmigrator.xml +++ /dev/null @@ -1,144 +0,0 @@ - - - - gpmigrator - -

Upgrades an existing Greenplum Database 4.2.x system without mirrors to 4.3.x.

-

Use - to upgrade a 4.2.x system that has mirrors.

- Using gpmigrator on a system with mirrors causes an - error. -
- Synopsis - gpmigrator old_GPHOME_path new_GPHOME_path -           [-d master_data_directory] -           [-l logfile_directory] [-q] [--debug] -           [--check-only] [--skip-check] [-R] - -gpmigrator --version | -v - -gpmigrator --help | -h -
-
- Prerequisites -

The following tasks should be performed prior to executing an upgrade:

-
    -
  • Make sure you are logged in to the master host as the Greenplum Database - superuser (gpadmin).
    • -
  • Install the Greenplum Database 4.3 binaries on all Greenplum hosts.
    • -
  • Copy or preserve any additional folders or files (such as backup folders) - that you have added in the Greenplum data directories or $GPHOME - directory. Only files or folders strictly related to Greenplum Database operations are - preserved by the migration utility.
    • -
  • (Optional) Run VACUUM on all databases, and remove - old server log files from pg_log in your master and segment data - directories. This is not required, but will reduce the size of Greenplum Database files to - be backed up and migrated.
    • -
  • Check for and recover any failed segments in your current Greenplum - Database system (gpstate, gprecoverseg).
    • -
  • (Optional, but highly recommended) Backup your current databases - (gpcrondump). If you find any issues when testing your upgraded system, - you can restore this backup.
    • -
  • Remove the standby master from your system configuration - (gpinitstandby -r).
    • -
  • Do a clean shutdown of your current system (gpstop).
    • -
  • Update your environment to source the 4.3 installation.
    • -
  • Inform all database users of the upgrade and lockout time frame. Once the - upgrade is in process, users will not be allowed on the system until the upgrade is - complete.
    • -
-
-
- Description -

The gpmigrator utility upgrades an existing Greenplum Database 4.2.x.x - system without mirrors to 4.3. This utility updates the system catalog and internal version - number, but not the actual software binaries. During the migration process, all client - connections to Greenplum Database will be locked out.

-
-
- Options - - - - old_GPHOME_path - - Required. The absolute path to the current version of Greenplum Database software you - want to migrate away from. - - - - new_GPHOME_path - - Required. The absolute path to the new version of Greenplum Database software you want - to migrate to. - - - -d master_data_directory - Optional. The current master host data directory. If not specified, the value set for - $MASTER_DATA_DIRECTORY will be used. - - - -l logfile_directory - The directory to write the log file. Defaults to ~/gpAdminLogs. - - - -q (quiet mode) - Run in quiet mode. Command output is not displayed on the screen, but is still written - to the log file. - - - -R (revert) - In the event of an error during upgrade, reverts all changes made by - gpmigrator. - - - --check-only - Runs pre-migrate checks to verify that your database is healthy. - Checks include:
  • Check catalog health
  • Check that the - Greenplum Database binaries on each segment match those on the master
  • Check - for a minimum amount of free disk space

Performing a pre-migration check - of your database should done during a database maintenance period. If the utility - detects catalog errors, the utility stops the database.

 - - - --skip-check - Skip the catalog check during the normal upgrade process. This can save some time, if - a catalog check was performed separately during the upgrade process. Use this option only after you have - checked for catalog issues with the --check-only option and have - resolved any catalog issues. - - - --help | -h - Displays the online help. - - - --debug - Sets logging level to debug. - - - --version | -v - Displays the version of this utility. - - -
-
- Examples -

Upgrade to version 4.3.x from version 4.2.x (make sure you are using the 4.3 version of - gpmigrator). This example upgrades to version 4.3.0.0 from version - 4.2.6.3:

- /usr/local/greenplum-db-4.3.0.0/bin/gpmigrator \ -  /usr/local/greenplum-db-4.2.6.3 \ -  /usr/local/greenplum-db-4.3.0.0 -
-
- See Also -

, - , - , - , -

-
- - diff --git a/gpdb-doc/dita/utility_guide/admin_utilities/gpmigrator_mirror.xml b/gpdb-doc/dita/utility_guide/admin_utilities/gpmigrator_mirror.xml deleted file mode 100644 index 5e915159083e2bda02c5329e14dd307bfc109163..0000000000000000000000000000000000000000 --- a/gpdb-doc/dita/utility_guide/admin_utilities/gpmigrator_mirror.xml +++ /dev/null @@ -1,139 +0,0 @@ - - - - gpmigrator_mirror - -

Upgrades an existing Greenplum Database 4.2.x system with mirrors to 4.3.x.

-

Use to - upgrade a 4.2.x system that does not have mirrors.

- Using gpmigrator_mirror on a system without mirrors causes an - error. -
- Synopsis - gpmigrator_mirror old_GPHOME_path new_GPHOME_path -                  [-d master_data_directory] -                  [-l logfile_directory] [-q] [--debug] -                  [--check-only] [--skip-check] [--debug] - -gpmigrator_mirror --version | -v - -gpmigrator_mirror --help | -h -
-
- Prerequisites -

The following tasks should be performed prior to executing an upgrade:

-
    -
  • Make sure you are logged in to the master host as the Greenplum Database - superuser (gpadmin).
    • -
  • Install the Greenplum Database 4.3 binaries on all Greenplum hosts.
    • -
  • Copy or preserve any additional folders or files (such as backup folders) - that you have added in the Greenplum data directories or $GPHOME - directory. Only files or folders strictly related to Greenplum Database operations are - preserved by the migration utility.
    • -
  • (Optional) Run VACUUM on all databases, and remove - old server log files from pg_log in your master and segment data - directories. This is not required, but will reduce the size of Greenplum Database files to - be backed up and migrated.
    • -
  • Check for and recover any failed segments in your current Greenplum - Database system (gpstate, gprecoverseg).
    • -
  • (Optional, but highly recommended) Backup your current databases - (gpcrondump). If you find any issues when testing your upgraded system, - you can restore this backup.
    • -
  • Remove the standby master from your system configuration - (gpinitstandby -r).
    • -
  • Do a clean shutdown of your current system (gpstop).
    • -
  • Update your environment to source the 4.3 installation.
    • -
  • Inform all database users of the upgrade and lockout time frame. Once the - upgrade is in process, users will not be allowed on the system until the upgrade is - complete.
    • -
-
-
- Description -

The gpmigrator_mirror utility upgrades an existing Greenplum Database - 4.2.x.x system with mirrors to 4.3. This utility updates the system catalog and internal - version number, but not the actual software binaries. During the migration process, all - client connections to Greenplum Database will be locked out.

-
-
- Options - - - - old_GPHOME_path - - Required. The absolute path to the current version of Greenplum Database software you - want to migrate away from. - - - - new_GPHOME_path - - Required. The absolute path to the new version of Greenplum Database software you want - to migrate to. - - - -d master_data_directory - Optional. The current master host data directory. If not specified, the value set for - $MASTER_DATA_DIRECTORY will be used. - - - -l logfile_directory - The directory to write the log file. Defaults to ~/gpAdminLogs. - - - -q (quiet mode) - Run in quiet mode. Command output is not displayed on the screen, but is still written - to the log file. - - - --check-only - Runs pre-migrate checks to verify that your database is healthy. - Checks include:

Check catalog health

Check that the Greenplum Database - binaries on each segment match those on the master

Check for a minium amount of - free disk space

Performing a pre-migration check of your database should done - during a database maintenance period. If the utility detects catalog errors, the - utility stops the database.

 - - - --skip-check - Skip the catalog check during the normal upgrade process. This can save some time, if - a catalog check was performed separately during the upgrade process.Use this option only after you have - checked for catalog issues with the --check-only option and have - resolved any catalog issues. - - - --help | -h - Displays the online help. - - - --debug - Sets logging level to debug. - - - --version | -v - Displays the version of this utility. - - -
-
- Examples -

Upgrade to version 4.3.x from version 4.2.x with mirrors (make sure you are using the 4.3 - version of gpmigrator_mirror). This example upgrades to 4.3.0.0 from - 4.2.6.3:

- /usr/local/greenplum-db-4.3.0.0/bin/gpmigrator_mirror \ -  /usr/local/greenplum-db-4.2.6.3 \ -  /usr/local/greenplum-db-4.3.0.0 -
-
- See Also -

, - , - , - , -

-
- - diff --git a/gpdb-doc/dita/utility_guide/admin_utilities/util_ref.xml b/gpdb-doc/dita/utility_guide/admin_utilities/util_ref.xml index fa0db943d4cb3fdc57818a403c946eceb9f53733..13899d78f9652ea4e4b2928e754d57df331abd77 100644 --- a/gpdb-doc/dita/utility_guide/admin_utilities/util_ref.xml +++ b/gpdb-doc/dita/utility_guide/admin_utilities/util_ref.xml @@ -83,12 +83,6 @@

-

- -

-

- -

diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/partition/manual/partition_upgrade_tpch.sql b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/partition/manual/partition_upgrade_tpch.sql deleted file mode 100644 index d32a5178d17094519be8c69bf31eeb7b2c06f1be..0000000000000000000000000000000000000000 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/basic/partition/manual/partition_upgrade_tpch.sql +++ /dev/null @@ -1,146 +0,0 @@ -CREATE TABLE NATION ( N_NATIONKEY INTEGER NOT NULL, - N_NAME CHAR(25) NOT NULL, - N_REGIONKEY INTEGER NOT NULL, - N_COMMENT VARCHAR(152)); - -CREATE TABLE REGION ( R_REGIONKEY INTEGER NOT NULL, - R_NAME CHAR(25) NOT NULL, - R_COMMENT VARCHAR(152)); - -CREATE TABLE PART ( P_PARTKEY INTEGER NOT NULL, - P_NAME VARCHAR(55) NOT NULL, - P_MFGR CHAR(25) NOT NULL, - P_BRAND CHAR(10) NOT NULL, - P_TYPE VARCHAR(25) NOT NULL, - P_SIZE INTEGER NOT NULL, - P_CONTAINER CHAR(10) NOT NULL, - P_RETAILPRICE DECIMAL(15,2) NOT NULL, - P_COMMENT VARCHAR(23) NOT NULL ); - -CREATE TABLE SUPPLIER ( S_SUPPKEY INTEGER NOT NULL, - S_NAME CHAR(25) NOT NULL, - S_ADDRESS VARCHAR(40) NOT NULL, - S_NATIONKEY INTEGER NOT NULL, - S_PHONE CHAR(15) NOT NULL, - S_ACCTBAL DECIMAL(15,2) NOT NULL, - S_COMMENT VARCHAR(101) NOT NULL); -CREATE TABLE PARTSUPP ( PS_PARTKEY INTEGER NOT NULL, - PS_SUPPKEY INTEGER NOT NULL, - PS_AVAILQTY INTEGER NOT NULL, - PS_SUPPLYCOST DECIMAL(15,2) NOT NULL, - PS_COMMENT VARCHAR(199) NOT NULL ); - -CREATE TABLE CUSTOMER ( C_CUSTKEY INTEGER NOT NULL, - C_NAME VARCHAR(25) NOT NULL, - C_ADDRESS VARCHAR(40) NOT NULL, - C_NATIONKEY INTEGER NOT NULL, - C_PHONE CHAR(15) NOT NULL, - C_ACCTBAL DECIMAL(15,2) NOT NULL, - C_MKTSEGMENT CHAR(10) NOT NULL, - C_COMMENT VARCHAR(117) NOT NULL); - -CREATE TABLE ORDERS ( O_ORDERKEY INTEGER NOT NULL, - O_CUSTKEY INTEGER NOT NULL, - O_ORDERSTATUS CHAR(1) NOT NULL, - O_TOTALPRICE DECIMAL(15,2) NOT NULL, - O_ORDERDATE DATE NOT NULL, - O_ORDERPRIORITY CHAR(15) NOT NULL, - O_CLERK CHAR(15) NOT NULL, - O_SHIPPRIORITY INTEGER NOT NULL, - O_COMMENT VARCHAR(79) NOT NULL); - -CREATE TABLE LINEITEM ( L_ORDERKEY INTEGER NOT NULL, - L_PARTKEY INTEGER NOT NULL, - L_SUPPKEY INTEGER NOT NULL, - L_LINENUMBER INTEGER NOT NULL, - L_QUANTITY DECIMAL(15,2) NOT NULL, - L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL, - L_DISCOUNT DECIMAL(15,2) NOT NULL, - L_TAX DECIMAL(15,2) NOT NULL, - L_RETURNFLAG CHAR(1) NOT NULL, - L_LINESTATUS CHAR(1) NOT NULL, - L_SHIPDATE DATE NOT NULL, - L_COMMITDATE DATE NOT NULL, - L_RECEIPTDATE DATE NOT NULL, - L_SHIPINSTRUCT CHAR(25) NOT NULL, - L_SHIPMODE CHAR(10) NOT NULL, - L_COMMENT VARCHAR(44) NOT NULL); - -CREATE TABLE partlist -( pid INTEGER UNIQUE, - plist TEXT NOT NULL, - pval TEXT -); - - -COPY CUSTOMER FROM '/home/jsoedomo/perforce/cdbfast/main/partition/_data/customer.tbl' delimiter '|'; -COPY NATION FROM '/home/jsoedomo/perforce/cdbfast/main/partition/_data/nation.tbl' delimiter '|'; -COPY LINEITEM FROM '/home/jsoedomo/perforce/cdbfast/main/partition/_data/lineitem.tbl' delimiter '|'; -COPY PART FROM '/home/jsoedomo/perforce/cdbfast/main/partition/_data/part.tbl' delimiter '|'; -COPY PARTSUPP FROM '/home/jsoedomo/perforce/cdbfast/main/partition/_data/partsupp.tbl' delimiter '|'; -COPY ORDERS FROM '/home/jsoedomo/perforce/cdbfast/main/partition/_data/orders.tbl' delimiter '|'; -COPY REGION FROM '/home/jsoedomo/perforce/cdbfast/main/partition/_data/region.tbl' delimiter '|'; -COPY SUPPLIER FROM '/home/jsoedomo/perforce/cdbfast/main/partition/_data/supplier.tbl' delimiter '|'; - -INSERT INTO partlist VALUES (1,'A','AAAAA'); -INSERT INTO partlist VALUES (2,'A','AAAAA'); -INSERT INTO partlist VALUES (3,'A','AAAAA'); -INSERT INTO partlist VALUES (4,'A','AAAAA'); -INSERT INTO partlist VALUES (5,'B','AAAAA'); -INSERT INTO partlist VALUES (6,'B','AAAAA'); -INSERT INTO partlist VALUES (7,'B','AAAAA'); -INSERT INTO partlist VALUES (8,'B','AAAAA'); -INSERT INTO partlist VALUES (9,'C','AAAAA'); -INSERT INTO partlist VALUES (10,'C','AAAAA'); -INSERT INTO partlist VALUES (11,'C','AAAAA'); -INSERT INTO partlist VALUES (12,'C','AAAAA'); -INSERT INTO partlist VALUES (13,'C','AAAAA'); -INSERT INTO partlist VALUES (14,'C','AAAAA'); -INSERT INTO partlist VALUES (15,'C','AAAAA'); - -select count(*) from customer; -select count(*) from nation; -select count(*) from lineitem; -select count(*) from part; -select count(*) from partsupp; -select count(*) from orders; -select count(*) from region; -select count(*) from supplier; -select count(*) from partlist; - --- Make sure that MASTER_DATA_DIRECTORY is set first, requirement for using gpcreatepart -\! gpcreatepart -a -t public.orders -f o_orderdate -p d -c y -s '1990-01-01' -e '2000-12-31' -n 10 -x gptest -\! gpcreatepart -a -t public.lineitem -f l_shipdate -p d -c y -s '1990-01-01' -e '2000-12-31' -n 10 -x gptest -\! gpcreatepart -a -t public.customer -f c_nationkey -p r -s 0 -e 25 -g 5 -x gptest -\! gpcreatepart -a -t public.partlist -f plist -p l -o A,B,C -x gptest - --- There is an existing INDEX for the table, what happen after upgrade for new enhanced partition? -CREATE INDEX lineitem_shipdate ON lineitem(l_shipdate); -CREATE INDEX lineitem_p1 ON lineitem_c_d_1_y_1_16536_year(l_shipdate); -CREATE INDEX lineitem_p2 ON lineitem_c_d_1_y_2_16536_year(l_shipdate); -CREATE INDEX lineitem_p3 ON lineitem_c_d_1_y_3_16536_year(l_shipdate); -CREATE INDEX lineitem_p4 ON lineitem_c_d_1_y_4_16536_year(l_shipdate); -CREATE INDEX lineitem_p5 ON lineitem_c_d_1_y_5_16536_year(l_shipdate); -CREATE INDEX lineitem_p6 ON lineitem_c_d_1_y_6_16536_year(l_shipdate); -CREATE INDEX lineitem_p7 ON lineitem_c_d_1_y_7_16536_year(l_shipdate); -CREATE INDEX lineitem_p8 ON lineitem_c_d_1_y_8_16536_year(l_shipdate); -CREATE INDEX lineitem_p9 ON lineitem_c_d_1_y_9_16536_year(l_shipdate); -CREATE INDEX lineitem_p10 ON lineitem_c_d_1_y_10_16536_year(l_shipdate); - -ANALYZE; - -\d - --- Run Upgrade for 3.2 with a script? or gpmigrator? --- Should use the new enhance partition --- \d --- select count(*) from pg_partition; -select count(*) from customer; -select count(*) from nation; -select count(*) from lineitem; -select count(*) from part; -select count(*) from partsupp; -select count(*) from orders; -select count(*) from region; -select count(*) from supplier; -select count(*) from partlist;