diff --git a/_cmd.py b/_cmd.py index 221d34fa19d554f4a5f2f56071ee03ace9f7c96b..8d24210c16a7f421caf53e3ec0b312c019d0be2d 100644 --- a/_cmd.py +++ b/_cmd.py @@ -27,11 +27,12 @@ import time import logging from logging import handlers from uuid import uuid1 as uuid -from optparse import OptionParser,OptionGroup +from optparse import OptionParser, OptionGroup, BadOptionError, Option from core import ObdHome from _stdio import IO from log import Logger +from _errno import DOC_LINK_MSG, LockError from tool import DirectoryUtil, FileUtil @@ -43,6 +44,49 @@ BUILD_TIME = '' DEBUG = True if '' else False +class AllowUndefinedOptionParser(OptionParser): + + def __init__(self, + usage=None, + option_list=None, + option_class=Option, + version=None, + conflict_handler="error", + description=None, + formatter=None, + add_help_option=True, + prog=None, + epilog=None, + allow_undefine=True): + OptionParser.__init__( + self, usage, option_list, option_class, version, conflict_handler, + description, formatter, add_help_option, prog, epilog + ) + self.allow_undefine = allow_undefine + + def warn(self, msg, file=None): + print ('warn: %s' % msg) + + def _process_long_opt(self, rargs, values): + try: + OptionParser._process_long_opt(self, rargs, values) + except BadOptionError as e: + if self.allow_undefine: + return self.warn(e) + else: + raise e + + def _process_short_opts(self, rargs, values): + try: + OptionParser._process_short_opts(self, rargs, values) + except BadOptionError as e: + if self.allow_undefine: + return self.warn(e) + return + else: + raise e + + class BaseCommand(object): def __init__(self, name, summary): @@ -53,7 +97,8 @@ class BaseCommand(object): self.opts = {} self.prev_cmd = '' self.is_init = False - self.parser = OptionParser(add_help_option=False) + self.hidden = False + self.parser = AllowUndefinedOptionParser(add_help_option=False) self.parser.add_option('-h', '--help', action='callback', callback=self._show_help, help='Show help and exit.') self.parser.add_option('-v', '--verbose', action='callback', callback=self._set_verbose, help='Activate verbose output.') @@ -91,6 +136,7 @@ class ObdCommand(BaseCommand): OBD_PATH = os.path.join(os.environ.get('OBD_HOME', os.getenv('HOME')), '.obd') OBD_INSTALL_PRE = os.environ.get('OBD_INSTALL_PRE', '/') + OBD_DEV_MODE_FILE = '.dev_mode' def init_home(self): version_path = os.path.join(self.OBD_PATH, 'version') @@ -99,23 +145,30 @@ class ObdCommand(BaseCommand): version_fobj.seek(0) version = version_fobj.read() if VERSION != version: - obd_plugin_path = os.path.join(self.OBD_PATH, 'plugins') - if DirectoryUtil.mkdir(self.OBD_PATH): - root_plugin_path = os.path.join(self.OBD_INSTALL_PRE, 'usr/obd/plugins') - if os.path.exists(root_plugin_path): - DirectoryUtil.copy(root_plugin_path, obd_plugin_path, ROOT_IO) - obd_mirror_path = os.path.join(self.OBD_PATH, 'mirror') - obd_remote_mirror_path = os.path.join(self.OBD_PATH, 'mirror/remote') - if DirectoryUtil.mkdir(obd_mirror_path): - root_remote_mirror = os.path.join(self.OBD_INSTALL_PRE, 'usr/obd/mirror/remote') - if os.path.exists(root_remote_mirror): - DirectoryUtil.copy(root_remote_mirror, obd_remote_mirror_path, ROOT_IO) + for part in ['plugins', 'config_parser', 'mirror/remote']: + obd_part_dir = os.path.join(self.OBD_PATH, part) + if DirectoryUtil.mkdir(self.OBD_PATH): + root_part_path = os.path.join(self.OBD_INSTALL_PRE, 'usr/obd/', part) + if os.path.exists(root_part_path): + DirectoryUtil.copy(root_part_path, obd_part_dir, ROOT_IO) version_fobj.seek(0) version_fobj.truncate() version_fobj.write(VERSION) version_fobj.flush() version_fobj.close() + @property + def dev_mode_path(self): + return os.path.join(self.OBD_PATH, self.OBD_DEV_MODE_FILE) + + @property + def dev_mode(self): + return os.path.exists(self.dev_mode_path) + + def parse_command(self): + self.parser.allow_undefine = self.dev_mode + return super(ObdCommand, self).parse_command() + def do_command(self): self.parse_command() self.init_home() @@ -127,7 +180,7 @@ class ObdCommand(BaseCommand): log_path = os.path.join(log_dir, 'obd') logger = Logger('obd') handler = handlers.TimedRotatingFileHandler(log_path, when='midnight', interval=1, backupCount=30) - handler.setFormatter(logging.Formatter("[%%(asctime)s] [%s] [%%(levelname)s] %%(message)s" % trace_id, "%Y-%m-%d %H:%M:%S")) + handler.setFormatter(logging.Formatter("[%%(asctime)s.%%(msecs)03d] [%s] [%%(levelname)s] %%(message)s" % trace_id, "%Y-%m-%d %H:%M:%S")) logger.addHandler(handler) ROOT_IO.trace_logger = logger obd = ObdHome(self.OBD_PATH, ROOT_IO) @@ -135,14 +188,17 @@ class ObdCommand(BaseCommand): ROOT_IO.verbose('cmd: %s' % self.cmds) ROOT_IO.verbose('opts: %s' % self.opts) ret = self._do_command(obd) + if not ret: + ROOT_IO.print(DOC_LINK_MSG) except NotImplementedError: ROOT_IO.exception('command \'%s\' is not implemented' % self.prev_cmd) - except IOError: + except LockError: ROOT_IO.exception('Another app is currently holding the obd lock.') except SystemExit: pass except: - ROOT_IO.exception('Running Error.') + e = sys.exc_info()[1] + ROOT_IO.exception('Running Error: %s' % e) if DEBUG: ROOT_IO.print('Trace ID: %s' % trace_id) return ret @@ -163,7 +219,8 @@ class MajorCommand(BaseCommand): commands = [x for x in self.commands.values() if not (hasattr(x, 'hidden') and x.hidden)] commands.sort(key=lambda x: x.name) for command in commands: - usage.append("%-14s %s\n" % (command.name, command.summary)) + if command.hidden is False: + usage.append("%-14s %s\n" % (command.name, command.summary)) self.parser.set_usage('\n'.join(usage)) return super(MajorCommand, self)._mk_usage() @@ -188,6 +245,63 @@ class MajorCommand(BaseCommand): self.commands[command.name] = command + + +class HiddenObdCommand(ObdCommand): + + def __init__(self, name, summary): + super(HiddenObdCommand, self).__init__(name, summary) + self.hidden = self.dev_mode is False + + +class HiddenMajorCommand(MajorCommand, HiddenObdCommand): + + pass + + +class DevCommand(HiddenObdCommand): + + def do_command(self): + if self.hidden: + ROOT_IO.error('`%s` is a developer command. Please start the developer mode first.\nUse `obd devmode enable` to start the developer mode' % self.prev_cmd) + return False + return super(DevCommand, self).do_command() + + +class DevModeEnableCommand(HiddenObdCommand): + + def __init__(self): + super(DevModeEnableCommand, self).__init__('enable', 'Enable Dev Mode') + + def _do_command(self, obd): + from tool import FileUtil + if FileUtil.open(self.dev_mode_path, _type='w', stdio=obd.stdio): + obd.stdio.print("Dev Mode: ON") + return True + return False + + +class DevModeDisableCommand(HiddenObdCommand): + + def __init__(self): + super(DevModeDisableCommand, self).__init__('disable', 'Disable Dev Mode') + + def _do_command(self, obd): + from tool import FileUtil + if FileUtil.rm(self.dev_mode_path, stdio=obd.stdio): + obd.stdio.print("Dev Mode: OFF") + return True + return False + + +class DevModeMajorCommand(HiddenMajorCommand): + + def __init__(self): + super(DevModeMajorCommand, self).__init__('devmode', 'Developer mode switch') + self.register_command(DevModeEnableCommand()) + self.register_command(DevModeDisableCommand()) + + class MirrorCloneCommand(ObdCommand): def __init__(self): @@ -246,9 +360,12 @@ class MirrorListCommand(ObdCommand): self.show_pkg(name, pkgs) return True else: - repos = obd.mirror_manager.get_mirrors() + repos = obd.mirror_manager.get_mirrors(is_enabled=None) for repo in repos: if repo.section_name == name: + if not repo.enabled: + ROOT_IO.error('Mirror repository %s is disabled.' % name) + return False pkgs = repo.get_all_pkg_info() self.show_pkg(name, pkgs) return True @@ -292,7 +409,7 @@ class MirrorEnableCommand(ObdCommand): def _do_command(self, obd): name = self.cmds[0] - obd.mirror_manager.set_remote_mirror_enabled(name, True) + return obd.mirror_manager.set_remote_mirror_enabled(name, True) class MirrorDisableCommand(ObdCommand): @@ -302,7 +419,7 @@ class MirrorDisableCommand(ObdCommand): def _do_command(self, obd): name = self.cmds[0] - obd.mirror_manager.set_remote_mirror_enabled(name, False) + return obd.mirror_manager.set_remote_mirror_enabled(name, False) class MirrorMajorCommand(MajorCommand): @@ -352,6 +469,35 @@ class ClusterMirrorCommand(ObdCommand): return self +class ClusterConfigStyleChange(ClusterMirrorCommand): + + def __init__(self): + super(ClusterConfigStyleChange, self).__init__('chst', 'Change Deployment Configuration Style') + self.parser.add_option('-c', '--components', type='string', help="List the components. Multiple components are separated with commas.") + self.parser.add_option('--style', type='string', help="Preferred Style") + + def _do_command(self, obd): + if self.cmds: + return obd.change_deploy_config_style(self.cmds[0], self.opts) + else: + return self._show_help() + + + +class ClusterCheckForOCPChange(ClusterMirrorCommand): + + def __init__(self): + super(ClusterCheckForOCPChange, self).__init__('check4ocp', 'Check Whether OCP Can Take Over Configurations in Use') + self.parser.add_option('-c', '--components', type='string', help="List the components. Multiple components are separated with commas.") + self.parser.add_option('-V', '--version', type='string', help="OCP Version", default='3.1.1') + + def _do_command(self, obd): + if self.cmds: + return obd.check_for_ocp(self.cmds[0], self.opts) + else: + return self._show_help() + + class ClusterAutoDeployCommand(ClusterMirrorCommand): def __init__(self): @@ -413,7 +559,7 @@ class ClusterStopCommand(ClusterMirrorCommand): def __init__(self): super(ClusterStopCommand, self).__init__('stop', 'Stop a started cluster.') self.parser.add_option('-s', '--servers', type='string', help="List the started servers. Multiple servers are separated with commas.") - self.parser.add_option('-c', '--components', type='string', help="List the started components. Multiple components are separated with commas.") + self.parser.add_option('-c', '--components', type='string', help="List the stoped components. Multiple components are separated with commas.") def _do_command(self, obd): if self.cmds: @@ -452,7 +598,7 @@ class ClusterRestartCommand(ClusterMirrorCommand): def __init__(self): super(ClusterRestartCommand, self).__init__('restart', 'Restart a started cluster.') self.parser.add_option('-s', '--servers', type='string', help="List the started servers. Multiple servers are separated with commas.") - self.parser.add_option('-c', '--components', type='string', help="List the started components. Multiple components are separated with commas.") + self.parser.add_option('-c', '--components', type='string', help="List the restarted components. Multiple components are separated with commas.") self.parser.add_option('--with-parameter', '--wp', action='store_true', help='Restart with parameters.') def _do_command(self, obd): @@ -588,6 +734,8 @@ class ClusterMajorCommand(MajorCommand): def __init__(self): super(ClusterMajorCommand, self).__init__('cluster', 'Deploy and manage a cluster.') + self.register_command(ClusterCheckForOCPChange()) + self.register_command(ClusterConfigStyleChange()) self.register_command(ClusterAutoDeployCommand()) self.register_command(ClusterDeployCommand()) self.register_command(ClusterStartCommand()) @@ -690,7 +838,7 @@ class TPCHCommand(TestMirrorCommand): self.parser.add_option('--tenant', type='string', help='Tenant for a test. [test]', default='test') self.parser.add_option('--database', type='string', help='Database for a test. [test]', default='test') self.parser.add_option('--obclient-bin', type='string', help='OBClient bin path. [obclient]', default='obclient') - self.parser.add_option('--dbgen-bin', type='string', help='dbgen bin path. [/usr/local/tpc-h-tools/bin/dbgen]', default='/usr/local/tpc-h-tools/bin/dbgen') + self.parser.add_option('--dbgen-bin', type='string', help='dbgen bin path. [/usr/tpc-h-tools/tpc-h-tools/bin/dbgen]', default='/usr/tpc-h-tools/tpc-h-tools/bin/dbgen') self.parser.add_option('-s', '--scale-factor', type='int', help='Set Scale Factor (SF) to . [1] ', default=1) self.parser.add_option('--tmp-dir', type='string', help='The temporary directory for executing TPC-H. [./tmp]', default='./tmp') self.parser.add_option('--ddl-path', type='string', help='Directory for DDL files.') @@ -698,7 +846,7 @@ class TPCHCommand(TestMirrorCommand): self.parser.add_option('--sql-path', type='string', help='Directory for SQL files.') self.parser.add_option('--remote-tbl-dir', type='string', help='Directory for the tbl on target observers. Make sure that you have read and write access to the directory when you start observer.') self.parser.add_option('--disable-transfer', '--dt', action='store_true', help='Disable the transfer. When enabled, OBD will use the tbl files under remote-tbl-dir instead of transferring local tbl files to remote remote-tbl-dir.') - self.parser.add_option('--dss-config', type='string', help='Directory for dists.dss. [/usr/local/tpc-h-tools]', default='/usr/local/tpc-h-tools/') + self.parser.add_option('--dss-config', type='string', help='Directory for dists.dss. [/usr/tpc-h-tools/tpc-h-tools]', default='/usr/tpc-h-tools/tpc-h-tools/') self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1}. [1]', default=1) self.parser.add_option('--test-only', action='store_true', help='Only testing SQLs are executed. No initialization is executed.') @@ -709,6 +857,37 @@ class TPCHCommand(TestMirrorCommand): return self._show_help() +class TPCCCommand(TestMirrorCommand): + + def __init__(self): + super(TPCCCommand, self).__init__('tpcc', 'Run a TPC-C test for a deployment.') + self.parser.add_option('--component', type='string', help='Components for a test.') + self.parser.add_option('--test-server', type='string', help='The server for a test. By default, the first root server in the component is the test server.') + self.parser.add_option('--user', type='string', help='Username for a test. [root]', default='root') + self.parser.add_option('--password', type='string', help='Password for a test.') + self.parser.add_option('--tenant', type='string', help='Tenant for a test. [test]', default='test') + self.parser.add_option('--database', type='string', help='Database for a test. [test]', default='test') + self.parser.add_option('--obclient-bin', type='string', help='OBClient bin path. [obclient]', default='obclient') + self.parser.add_option('--java-bin', type='string', help='Java bin path. [java]', default='java') + self.parser.add_option('--tmp-dir', type='string', help='The temporary directory for executing TPC-C. [./tmp]', default='./tmp') + self.parser.add_option('--bmsql-dir', type='string', help='The directory of BenchmarkSQL.') + self.parser.add_option('--bmsql-jar', type='string', help='BenchmarkSQL jar path.') + self.parser.add_option('--bmsql-libs', type='string', help='BenchmarkSQL libs path.') + self.parser.add_option('--bmsql-sql-dir', type='string', help='The directory of BenchmarkSQL sql scripts.') + self.parser.add_option('--warehouses', type='int', help='The number of warehouses.') + self.parser.add_option('--load-workers', type='int', help='The number of workers to load data.') + self.parser.add_option('--terminals', type='int', help='The number of terminals.') + self.parser.add_option('--run-mins', type='int', help='To run for specified minutes.', default=10) + self.parser.add_option('--test-only', action='store_true', help='Only testing SQLs are executed. No initialization is executed.') + self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1/2}. [1] 0 - No optimization. 1 - Optimize some of the parameters which do not need to restart servers. 2 - Optimize all the parameters and maybe RESTART SERVERS for better performance.', default=1) + + def _do_command(self, obd): + if self.cmds: + return obd.tpcc(self.cmds[0], self.opts) + else: + return self._show_help() + + class TestMajorCommand(MajorCommand): def __init__(self): @@ -716,6 +895,7 @@ class TestMajorCommand(MajorCommand): self.register_command(MySQLTestCommand()) self.register_command(SysBenchCommand()) self.register_command(TPCHCommand()) + # self.register_command(TPCCCommand()) class BenchMajorCommand(MajorCommand): @@ -743,6 +923,7 @@ class MainCommand(MajorCommand): def __init__(self): super(MainCommand, self).__init__('obd', '') + self.register_command(DevModeMajorCommand()) self.register_command(MirrorMajorCommand()) self.register_command(ClusterMajorCommand()) self.register_command(RepositoryMajorCommand()) diff --git a/_deploy.py b/_deploy.py index 5fa80b5a0a015866b99eb482df0e57d72becb89a..549b2c6d4611dcdf7e693efc8dfc1983e06d8a48 100644 --- a/_deploy.py +++ b/_deploy.py @@ -26,14 +26,20 @@ import pickle import getpass from copy import deepcopy from enum import Enum -from collections import OrderedDict -from tool import ConfigUtil, FileUtil, YamlLoader +from ruamel.yaml.comments import CommentedMap + +from tool import ConfigUtil, FileUtil, YamlLoader, OrderedDict from _manager import Manager from _repository import Repository yaml = YamlLoader() +DEFAULT_CONFIG_PARSER_MANAGER = None + + +class ParserError(Exception): + pass class UserConfig(object): @@ -90,9 +96,212 @@ class ServerConfigFlyweightFactory(object): return ServerConfigFlyweightFactory._CACHE[_key] +class InnerConfigItem(str): + pass + + +class InnerConfig(object): + + def __init__(self, path, yaml_loader): + self.path = path + self.yaml_loader = yaml_loader + self.config = {} + self._load() + + def _load(self): + self.config = {} + try: + with FileUtil.open(self.path, 'rb') as f: + config = self.yaml_loader.load(f) + for component_name in config: + self.config[component_name] = {} + c_config = config[component_name] + for server in c_config: + i_config = {} + s_config = c_config[server] + for key in s_config: + i_config[InnerConfigItem(key)] = s_config[key] + self.config[component_name][server] = i_config + except: + pass + + def _dump(self): + try: + stdio = self.yaml_loader.stdio if self.yaml_loader else None + with FileUtil.open(self.path, 'w', stdio=stdio) as f: + self.yaml_loader.dump(self.config, f) + return True + except: + pass + return False + + def dump(self): + return self._dump() + + def get_component_config(self, component_name): + return self.config.get(component_name, {}) + + def get_server_config(self, component_name, server): + return self.get_component(component_name).get(server, {}) + + def update_component_config(self, component_name, config): + self.config[component_name] = {} + for server in config: + c_config = {} + data = config[server] + for key in data: + if not isinstance(key, InnerConfigItem): + key = InnerConfigItem(key) + c_config[key] = data[key] + self.config[component_name][server] = c_config + + +class ConfigParser(object): + + STYLE = '' + INNER_CONFIG_MAP = {} + PREFIX = '$_' + + @classmethod + def _is_inner_item(cls, key): + return isinstance(key, InnerConfigItem) and key.startswith(cls.PREFIX) + + @classmethod + def extract_inner_config(cls, cluster_config, config): + return {} + + @classmethod + def _to_cluster_config(cls, component_name, config): + raise NotImplementedError + + @classmethod + def to_cluster_config(cls, component_name, config): + cluster_config = cls._to_cluster_config(component_name, config) + cluster_config.parser = cls + return cluster_config + + @classmethod + def _from_cluster_config(cls, conf, cluster_config): + raise NotImplementedError + + @classmethod + def from_cluster_config(cls, cluster_config): + if not cls.STYLE: + raise NotImplementedError('undefined Style ConfigParser') + + conf = CommentedMap() + conf['style'] = cls.STYLE + if cluster_config.origin_package_hash: + conf['package_hash'] = cluster_config.origin_package_hash + if cluster_config.origin_version: + conf['version'] = cluster_config.origin_version + if cluster_config.origin_tag: + conf['tag'] = cluster_config.origin_tag + if cluster_config.depends: + conf['depends'] = cluster_config.depends + conf = cls._from_cluster_config(conf, cluster_config) + inner_config = cls.extract_inner_config(cluster_config, conf) + return { + 'inner_config': inner_config, + 'config': conf + } + + @classmethod + def get_server_src_conf(cls, cluster_config, component_config, server): + if server.name not in component_config: + component_config[server.name] = {} + return component_config[server.name] + + @classmethod + def get_global_src_conf(cls, cluster_config, component_config): + if 'global' not in component_config: + component_config['global'] = {} + return component_config['global'] + + +class DefaultConfigParser(ConfigParser): + + STYLE = 'default' + + @classmethod + def _to_cluster_config(cls, component_name, conf): + if 'servers' in conf and isinstance(conf['servers'], list): + servers = [] + for server in conf['servers']: + if isinstance(server, dict): + ip = ConfigUtil.get_value_from_dict(server, 'ip', transform_func=str) + name = ConfigUtil.get_value_from_dict(server, 'name', transform_func=str) + else: + ip = server + name = None + if not re.match('^\d{1,3}(\\.\d{1,3}){3}$', ip): + continue + server = ServerConfigFlyweightFactory.get_instance(ip, name) + if server not in servers: + servers.append(server) + else: + servers = [] + cluster_config = ClusterConfig( + servers, + component_name, + ConfigUtil.get_value_from_dict(conf, 'version', None, str), + ConfigUtil.get_value_from_dict(conf, 'tag', None, str), + ConfigUtil.get_value_from_dict(conf, 'package_hash', None, str) + ) + if 'global' in conf: + cluster_config.set_global_conf(conf['global']) + for server in servers: + if server.name in conf: + cluster_config.add_server_conf(server, conf[server.name]) + return cluster_config + + @classmethod + def extract_inner_config(cls, cluster_config, config): + inner_config = cluster_config.get_inner_config() + for server in cluster_config.servers: + if server.name not in inner_config: + inner_config[server.name] = {} + + global_config = config.get('global', {}) + keys = list(global_config.keys()) + for key in keys: + if cls._is_inner_item(key): + for server in cluster_config.servers: + inner_config[server.name][key] = global_config[key] + del global_config[key] + + for server in cluster_config.servers: + if server.name not in config: + continue + server_config = config[server.name] + keys = list(server_config.keys()) + for key in keys: + if cls._is_inner_item(key): + inner_config[server.name][key] = server_config[key] + del server_config[key] + return inner_config + + @classmethod + def _from_cluster_config(cls, conf, cluster_config): + conf['servers'] = [] + for server in cluster_config.servers: + if server.name == server.ip: + conf['servers'].append(server.name) + else: + conf['servers'].append({'name': server.name, 'ip': server.ip}) + + if cluster_config.get_original_global_conf(): + conf['global'] = cluster_config.get_original_global_conf() + for server in cluster_config.servers: + server_config = cluster_config.get_original_server_conf(server) + if server_config: + conf[server.name] = server_config + return conf + + class ClusterConfig(object): - def __init__(self, servers, name, version, tag, package_hash): + def __init__(self, servers, name, version, tag, package_hash, parser=None): self.version = version self.origin_version = version self.tag = tag @@ -106,6 +315,8 @@ class ClusterConfig(object): self._server_conf = {} self._cache_server = {} self._original_global_conf = {} + self._inner_config = {} + servers = list(servers) self.servers = servers self._original_servers = servers # 保证顺序 for server in servers: @@ -113,18 +324,28 @@ class ClusterConfig(object): self._cache_server[server] = None self._deploy_config = None self._depends = {} + self.parser = parser def __eq__(self, other): if not isinstance(other, self.__class__): return False return self._global_conf == other._global_conf and self._server_conf == other._server_conf + def __deepcopy__(self, memo): + cluster_config = self.__class__(deepcopy(self.servers), self.name, self.version, self.tag, self.package_hash, self.parser) + copy_attrs = ['origin_tag', 'origin_version', 'origin_package_hash', 'parser'] + deepcopy_attrs = ['_temp_conf', '_default_conf', '_global_conf', '_server_conf', '_cache_server', '_original_global_conf', '_depends', '_original_servers', '_inner_config'] + for attr in copy_attrs: + setattr(cluster_config, attr, getattr(self, attr)) + for attr in deepcopy_attrs: + setattr(cluster_config, attr, deepcopy(getattr(self, attr))) + return cluster_config + def set_deploy_config(self, _deploy_config): if self._deploy_config is None: self._deploy_config = _deploy_config return True return False - @property def original_servers(self): return self._original_servers @@ -133,26 +354,41 @@ class ClusterConfig(object): def depends(self): return self._depends.keys() + def _clear_cache_server(self): + for server in self._cache_server: + self._cache_server[server] = None + + def get_inner_config(self): + return self._inner_config + + def apply_inner_config(self, config): + self._inner_config = config + self._clear_cache_server() + def add_depend(self, name, cluster_conf): + if self.name == name: + raise Exception('Can not set %s as %s\'s dependency' % (name, name)) + if self.name in cluster_conf.depends: + raise Exception('Circular Dependency: %s and %s' % (self.name, name)) self._depends[name] = cluster_conf def del_depend(self, name, component_name): if component_name in self._depends: del self._depends[component_name] - def get_depled_servers(self, name): + def get_depend_servers(self, name): if name not in self._depends: return None cluster_config = self._depends[name] return deepcopy(cluster_config.original_servers) - def get_depled_config(self, name, server=None): + def get_depend_config(self, name, server=None): if name not in self._depends: return None cluster_config = self._depends[name] config = cluster_config.get_server_conf_with_default(server) if server else cluster_config.get_global_conf() return deepcopy(config) - + def update_server_conf(self, server, key, value, save=True): if self._deploy_config is None: return False @@ -230,6 +466,12 @@ class ClusterConfig(object): self._default_conf[key] = self._temp_conf[key].default self.set_global_conf(self._global_conf) # 更新全局配置 + def get_temp_conf_item(self, key): + if self._temp_conf: + return self._temp_conf.get(key) + else: + return None + def check_param(self): error = [] if self._temp_conf: @@ -253,8 +495,7 @@ class ClusterConfig(object): self._original_global_conf = deepcopy(conf) self._global_conf = deepcopy(self._default_conf) self._global_conf.update(self._original_global_conf) - for server in self._cache_server: - self._cache_server[server] = None + self._clear_cache_server() def add_server_conf(self, server, conf): if server not in self.servers: @@ -271,11 +512,15 @@ class ClusterConfig(object): if server not in self._server_conf: return None if self._cache_server[server] is None: - conf = deepcopy(self._global_conf) + conf = deepcopy(self._inner_config.get(server.name, {})) + conf.update(self._global_conf) conf.update(self._server_conf[server]) self._cache_server[server] = conf return self._cache_server[server] + def get_original_global_conf(self): + return self._original_global_conf + def get_original_server_conf(self, server): return self._server_conf.get(server) @@ -319,20 +564,40 @@ class DeployInfo(object): class DeployConfig(object): - def __init__(self, yaml_path, yaml_loader=yaml): + def __init__(self, yaml_path, yaml_loader=yaml, inner_config=None, config_parser_manager=None): self._user = None self.unuse_lib_repository = False self.auto_create_tenant = False + self._inner_config = inner_config self.components = OrderedDict() self._src_data = None self.yaml_path = yaml_path self.yaml_loader = yaml_loader + self.config_parser_manager = config_parser_manager if config_parser_manager else DEFAULT_CONFIG_PARSER_MANAGER + if self.config_parser_manager is None: + raise ParserError('ConfigParaserManager Not Set') self._load() @property def user(self): return self._user + @property + def inner_config(self): + return self._inner_config + + @inner_config.setter + def inner_config(self, inner_config): + if inner_config: + def get_inner_config(component_name): + return inner_config.get_component_config(component_name) + else: + def get_inner_config(component_name): + return {} + for component_name in self.components: + self.components[component_name].apply_inner_config(get_inner_config(component_name)) + self._inner_config = inner_config + def set_unuse_lib_repository(self, status): if self.unuse_lib_repository != status: self.unuse_lib_repository = status @@ -371,44 +636,59 @@ class DeployConfig(object): return False def _load(self): - try: - with open(self.yaml_path, 'rb') as f: - depends = {} - self._src_data = self.yaml_loader.load(f) - for key in self._src_data: - if key == 'user': - self.set_user_conf(UserConfig( - ConfigUtil.get_value_from_dict(self._src_data[key], 'username'), - ConfigUtil.get_value_from_dict(self._src_data[key], 'password'), - ConfigUtil.get_value_from_dict(self._src_data[key], 'key_file'), - ConfigUtil.get_value_from_dict(self._src_data[key], 'port', 0, int), - ConfigUtil.get_value_from_dict(self._src_data[key], 'timeout', 0, int), - )) - elif key == 'unuse_lib_repository': - self.unuse_lib_repository = self._src_data['unuse_lib_repository'] - elif key == 'auto_create_tenant': - self.auto_create_tenant = self._src_data['auto_create_tenant'] - elif issubclass(type(self._src_data[key]), dict): - self._add_component(key, self._src_data[key]) - depends[key] = self._src_data[key].get('depends', []) - for comp in depends: - conf = self.components[comp] - for name in depends[comp]: - if name == comp: - continue - if name in self.components: - conf.add_depend(name, self.components[name]) - except: - pass + with open(self.yaml_path, 'rb') as f: + depends = {} + self._src_data = self.yaml_loader.load(f) + for key in self._src_data: + if key == 'user': + self.set_user_conf(UserConfig( + ConfigUtil.get_value_from_dict(self._src_data[key], 'username'), + ConfigUtil.get_value_from_dict(self._src_data[key], 'password'), + ConfigUtil.get_value_from_dict(self._src_data[key], 'key_file'), + ConfigUtil.get_value_from_dict(self._src_data[key], 'port', 0, int), + ConfigUtil.get_value_from_dict(self._src_data[key], 'timeout', 0, int), + )) + elif key == 'unuse_lib_repository': + self.unuse_lib_repository = self._src_data['unuse_lib_repository'] + elif key == 'auto_create_tenant': + self.auto_create_tenant = self._src_data['auto_create_tenant'] + elif issubclass(type(self._src_data[key]), dict): + self._add_component(key, self._src_data[key]) + depends[key] = self._src_data[key].get('depends', []) + for comp in depends: + conf = self.components[comp] + for name in depends[comp]: + if name == comp: + continue + if name in self.components: + conf.add_depend(name, self.components[name]) if not self.user: self.set_user_conf(UserConfig()) + def _separate_config(self): + if self.inner_config: + for component_name in self.components: + cluster_config = self.components[component_name] + src_data = self._src_data[component_name] + parser = cluster_config.parser + if parser: + inner_config = parser.extract_inner_config(cluster_config, src_data) + self.inner_config.update_component_config(component_name, inner_config) + + def _dump_inner_config(self): + if self.inner_config: + self._separate_config() + self.inner_config.dump() + def _dump(self): try: with open(self.yaml_path, 'w') as f: + self._dump_inner_config() self.yaml_loader.dump(self._src_data, f) return True except: + import logging + logging.exception('') pass return False @@ -452,68 +732,81 @@ class DeployConfig(object): cluster_config = self.components[component_name] if server not in cluster_config.servers: return False - component_config = self._src_data[component_name] - if server.name not in component_config: - component_config[server.name] = {key: value} - else: - component_config[server.name][key] = value + component_config = cluster_config.parser.get_server_src_conf(cluster_config, self._src_data[component_name], server) + component_config[key] = value return self.dump() if save else True def update_component_global_conf(self, component_name, key, value, save=True): if component_name not in self.components: return False - component_config = self._src_data[component_name] - if 'global' not in component_config: - component_config['global'] = {key: value} - else: - component_config['global'][key] = value + cluster_config = self.components[component_name] + component_config = cluster_config.parser.get_global_src_conf(cluster_config, self._src_data[component_name]) + component_config[key] = value return self.dump() if save else True - def _add_component(self, component_name, conf): - if 'servers' in conf and isinstance(conf['servers'], list): - servers = [] - for server in conf['servers']: - if isinstance(server, dict): - ip = ConfigUtil.get_value_from_dict(server, 'ip', transform_func=str) - name = ConfigUtil.get_value_from_dict(server, 'name', transform_func=str) - else: - ip = server - name = None - if not re.match('^\d{1,3}(\\.\d{1,3}){3}$', ip): - continue - server = ServerConfigFlyweightFactory.get_instance(ip, name) - if server not in servers: - servers.append(server) + def _set_component(self, cluster_config): + cluster_config.set_deploy_config(self) + if self.inner_config: + inner_config = self.inner_config.get_component_config(cluster_config.name) + if inner_config: + cluster_config.apply_inner_config(inner_config) + self.components[cluster_config.name] = cluster_config + + def update_component(self, cluster_config): + component_name = cluster_config.name + if cluster_config.parser: + parser = cluster_config.parser else: - servers = [] - cluster_conf = ClusterConfig( - servers, - component_name, - ConfigUtil.get_value_from_dict(conf, 'version', None, str), - ConfigUtil.get_value_from_dict(conf, 'tag', None, str), - ConfigUtil.get_value_from_dict(conf, 'package_hash', None, str) - ) - if 'global' in conf: - cluster_conf.set_global_conf(conf['global']) - for server in servers: - if server.name in conf: - cluster_conf.add_server_conf(server, conf[server.name]) - cluster_conf.set_deploy_config(self) - self.components[component_name] = cluster_conf + conf = self._src_data.get('component_name', {}) + style = conf.get('style', 'default') + parser = self.config_parser_manager.get_parser(component_name, style) + + conf = parser.from_cluster_config(cluster_config) + if component_name in self.components: + self.components[component_name].set_deploy_config(None) + cluster_config = deepcopy(cluster_config) + cluster_config.apply_inner_config(conf['inner_config']) + if self.inner_config: + self.inner_config.update_component_config(component_name, conf['inner_config']) + self._src_data[component_name] = conf['config'] + self._set_component(cluster_config) + return True + + def _add_component(self, component_name, conf): + parser = self.config_parser_manager.get_parser(component_name, conf.get('style')) + cluster_config = parser.to_cluster_config(component_name, conf) + self._set_component(cluster_config) + + def change_component_config_style(self, component_name, style): + if component_name not in self.components: + return False + + parser = self.config_parser_manager.get_parser(component_name, style) + cluster_config = self.components[component_name] + if cluster_config.parser != parser: + new_config = parser.from_cluster_config(cluster_config) + self._add_component(component_name, new_config) + self.components[component_name].apply_inner_config(new_config['inner_config']) + if self.inner_config: + self.inner_config.update_component_config(component_name, new_config['inner_config']) + self._src_data[component_name] = new_config['config'] + return True class Deploy(object): DEPLOY_STATUS_FILE = '.data' DEPLOY_YAML_NAME = 'config.yaml' + INNER_CONFIG_NAME = 'inner_config.yaml' UPRADE_META_NAME = '.upgrade' - def __init__(self, config_dir, stdio=None): + def __init__(self, config_dir, config_parser_manager=None, stdio=None): self.config_dir = config_dir self.name = os.path.split(config_dir)[1] self._info = None self._config = None self.stdio = stdio + self.config_parser_manager = config_parser_manager self._uprade_meta = None def use_model(self, name, repository, dump=True): @@ -535,6 +828,10 @@ class Deploy(object): def get_upgrade_meta_path(path): return os.path.join(path, Deploy.UPRADE_META_NAME) + @staticmethod + def get_inner_config_path(path): + return os.path.join(path, Deploy.INNER_CONFIG_NAME) + @staticmethod def get_temp_deploy_yaml_path(path): return os.path.join(path, 'tmp_%s' % Deploy.DEPLOY_YAML_NAME) @@ -556,24 +853,33 @@ class Deploy(object): self._info = DeployInfo(self.name, DeployStatus.STATUS_CONFIGURED) return self._info + def _load_deploy_config(self, path): + yaml_loader = YamlLoader(stdio=self.stdio) + deploy_config = DeployConfig(path, yaml_loader=yaml_loader, config_parser_manager=self.config_parser_manager) + deploy_info = self.deploy_info + for component_name in deploy_info.components: + if component_name not in deploy_config.components: + continue + config = deploy_info.components[component_name] + cluster_config = deploy_config.components[component_name] + if 'version' in config and config['version']: + cluster_config.version = config['version'] + if 'hash' in config and config['hash']: + cluster_config.package_hash = config['hash'] + + deploy_config.inner_config = InnerConfig(self.get_inner_config_path(self.config_dir), yaml_loader=yaml_loader) + return deploy_config + + @property + def temp_deploy_config(self): + path = self.get_temp_deploy_yaml_path(self.config_dir) + return self._load_deploy_config(path) + @property def deploy_config(self): if self._config is None: - try: - path = self.get_deploy_yaml_path(self.config_dir) - self._config = DeployConfig(path, YamlLoader(stdio=self.stdio)) - deploy_info = self.deploy_info - for component_name in deploy_info.components: - if component_name not in self._config.components: - continue - config = deploy_info.components[component_name] - cluster_config = self._config.components[component_name] - if 'version' in config and config['version']: - cluster_config.version = config['version'] - if 'hash' in config and config['hash']: - cluster_config.package_hash = config['hash'] - except: - pass + path = self.get_deploy_yaml_path(self.config_dir) + self._config = self._load_deploy_config(path) return self._config def _get_uprade_meta(self): @@ -597,6 +903,7 @@ class Deploy(object): return uprade_meta.get('component') if uprade_meta else None def apply_temp_deploy_config(self): + self.stdio and getattr(self.stdio, 'verbose', print)('%s apply temp config' % self.name) src_yaml_path = self.get_temp_deploy_yaml_path(self.config_dir) target_src_path = self.get_deploy_yaml_path(self.config_dir) try: @@ -677,7 +984,7 @@ class Deploy(object): self._uprade_meta = uprade_meta return False - def _update_componet_repository(self, component, repository): + def _update_component_repository(self, component, repository): if not self.deploy_config.update_component_package_hash(component, repository.hash, repository.version): return False self.use_model(component, repository) @@ -688,7 +995,7 @@ class Deploy(object): self._uprade_meta = None self._dump_upgrade_meta_data() if dest_repository: - self._update_componet_repository(dest_repository.name, dest_repository) + self._update_component_repository(dest_repository.name, dest_repository) return True return False @@ -706,6 +1013,79 @@ class Deploy(object): return self._update_deploy_config_status(status) return False + def effect_tip(self): + cmd_map = { + DeployConfigStatus.NEED_RELOAD: 'obd cluster reload %s' % self.name, + DeployConfigStatus.NEED_RESTART: 'obd cluster restart %s --wp' % self.name, + DeployConfigStatus.NEED_REDEPLOY: 'obd cluster redeploy %s' % self.name, + } + if self.deploy_info.config_status in cmd_map: + return '\nUse `%s` to make changes take effect.' % cmd_map[self.deploy_info.config_status] + return '' + + +class ConfigParserManager(Manager): + + RELATIVE_PATH = 'config_parser/' + + def __init__(self, home_path, stdio=None): + super(ConfigParserManager, self).__init__(home_path, stdio) + self.global_parsers = { + 'default': DefaultConfigParser, + } + self.component_parsers = {} + + def _formate_paraser_name(self, style): + style = style.title() + return '%sConfigParser' % style.replace('-', '').replace(' ', '').replace('_', '') + + def _load_paraser(self, lib_path, style): + from tool import DynamicLoading + module_name = '%s_config_parser' % style + file_name = '%s.py' % module_name + path = os.path.join(lib_path, file_name) + if os.path.isfile(path): + DynamicLoading.add_lib_path(lib_path) + self.stdio and getattr(self.stdio, 'verbose', 'print')('load config parser: %s' % path) + module = DynamicLoading.import_module(module_name, self.stdio) + clz_name = self._formate_paraser_name(style) + try: + return getattr(module, clz_name) + except: + self.stdio and getattr(self.stdio, 'exception', 'print')('') + return None + finally: + DynamicLoading.remove_lib_path(lib_path) + else: + self.stdio and getattr(self.stdio, 'verbose', 'print')('No config parser: %s' % path) + return None + + def _get_component_parser(self, component, style): + if component not in self.component_parsers: + self.component_parsers[component] = {} + component_parsers = self.component_parsers[component] + if style not in component_parsers: + lib_path = os.path.join(self.path, component) + component_parsers[style] = self._load_paraser(lib_path, style) + return component_parsers[style] + + def _get_global_parser(self, style): + if style not in self.global_parsers: + self.global_parsers[style] = self._load_paraser(self.path, style) + return self.global_parsers[style] + + def get_parser(self, component='', style=''): + if not style or style == 'default': + return self.global_parsers['default'] + if component: + parser = self._get_component_parser(component, style) + if parser: + return parser + parser = self._get_global_parser(style) + if not parser: + raise ParserError('Unsupported configuration style: %s' % style) + return parser + class DeployManager(Manager): @@ -714,6 +1094,7 @@ class DeployManager(Manager): def __init__(self, home_path, lock_manager=None, stdio=None): super(DeployManager, self).__init__(home_path, stdio) self.lock_manager = lock_manager + self.config_parser_manager = ConfigParserManager(home_path, stdio) def _lock(self, name, read_only=False): if self.lock_manager: @@ -729,14 +1110,14 @@ class DeployManager(Manager): path = os.path.join(self.path, name) if os.path.isdir(path): self._lock(name, read_only) - configs.append(Deploy(path, self.stdio)) + configs.append(Deploy(path, config_parser_manager=self.config_parser_manager, stdio=self.stdio)) return configs def get_deploy_config(self, name, read_only=False): self._lock(name, read_only) path = os.path.join(self.path, name) if os.path.isdir(path): - return Deploy(path, self.stdio) + return Deploy(path, config_parser_manager=self.config_parser_manager, stdio=self.stdio) return None def create_deploy_config(self, name, src_yaml_path): @@ -745,7 +1126,7 @@ class DeployManager(Manager): target_src_path = Deploy.get_deploy_yaml_path(config_dir) self._mkdir(config_dir) if FileUtil.copy(src_yaml_path, target_src_path, self.stdio): - return Deploy(config_dir, self.stdio) + return Deploy(config_dir, config_parser_manager=self.config_parser_manager, stdio=self.stdio) else: self._rm(config_dir) return None diff --git a/_errno.py b/_errno.py new file mode 100644 index 0000000000000000000000000000000000000000..f4dc987766c1f5e650fab2e3f478b25a33031fae --- /dev/null +++ b/_errno.py @@ -0,0 +1,72 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +from enum import Enum + + +class LockError(Exception): + pass + + +class OBDErrorCode(object): + + def __init__(self, code, msg): + self.code = code + self.msg = msg + self._str_ = ('OBD-%04d: ' % code) + msg + + def format(self, *args, **kwargs): + return self._str_.format(*args, **kwargs) + + def __str__(self): + return self._str_ + + +class InitDirFailedErrorMessage(object): + + PATH_ONLY = ': {path}.' + NOT_EMPTY = ': {path} is not empty.' + CREATE_FAILED = ': create {path} failed.' + PERMISSION_DENIED = ': {path} permission denied .' + + +DOC_LINK_MSG = 'See https://open.oceanbase.com/docs/obd-cn/V1.2.0/10000000000017237.' + +EC_CONFIG_CONFLICT_PORT = OBDErrorCode(1000, 'Configuration conflict {server1}:{port} port is used for {server2}\'s {key}') +EC_CONFLICT_PORT = OBDErrorCode(1001, '{server}:{port} port is already used') +EC_FAIL_TO_INIT_PATH = OBDErrorCode(1002, 'Fail to init {server} {key}{msg}') +EC_CLEAN_PATH_FAILED = OBDErrorCode(1003, 'Fail to clean {server}:{path}') +EC_CONFIG_CONFLICT_DIR = OBDErrorCode(1004, 'Configuration conflict {server1}: {path} is used for {server2}\'s {key}') +EC_SOME_SERVER_STOPED = OBDErrorCode(1005, 'Some of the servers in the cluster have been stopped') + +EC_OBSERVER_NOT_ENOUGH_MEMORY = OBDErrorCode(2000, '({ip}) not enough memory. (Free: {free}, Need: {need})') +EC_OBSERVER_CAN_NOT_MIGRATE_IN = OBDErrorCode(2001, 'server can not migrate in') +EC_OBSERVER_FAIL_TO_START = OBDErrorCode(2002, 'Failed to start {server} observer') +EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG = OBDErrorCode(2003, '({ip}) {path} not enough disk space for clog. Use redo_dir to set other disk for clog, or reduce the value of datafile_size') +EC_OBSERVER_INVALID_MODFILY_GLOBAL_KEY = OBDErrorCode(2004, 'Invalid: {key} is not a single server configuration item') + +EC_MYSQLTEST_PARSE_CMD_FAILED = OBDErrorCode(3000, 'parse cmd failed: {path}') +EC_MYSQLTEST_FAILE_NOT_FOUND = OBDErrorCode(3001, '{file} not found in {path}') + +EC_OBAGENT_RELOAD_FAILED = OBDErrorCode(4000, 'Fail to reload {server}') +EC_OBAGENT_SEND_CONFIG_FAILED = OBDErrorCode(4001, 'Fail to send config file to {server}') + diff --git a/_lock.py b/_lock.py index 7c52b900c787aa8c0d8eea86620ac2323b7d898c..b799b4094f27e7fd4252d3159356528a5b17f537 100644 --- a/_lock.py +++ b/_lock.py @@ -26,6 +26,7 @@ from enum import Enum from tool import FileUtil from _manager import Manager +from _errno import LockError class LockType(Enum): @@ -59,11 +60,17 @@ class MixLock(object): def _ex_lock(self): if self.lock_obj: - FileUtil.exclusive_lock_obj(self.lock_obj, stdio=self.stdio) + try: + FileUtil.exclusive_lock_obj(self.lock_obj, stdio=self.stdio) + except Exception as e: + raise LockError(e) def _sh_lock(self): if self.lock_obj: - FileUtil.share_lock_obj(self.lock_obj, stdio=self.stdio) + try: + FileUtil.share_lock_obj(self.lock_obj, stdio=self.stdio) + except Exception as e: + raise LockError(e) def sh_lock(self): if not self.locked: @@ -80,7 +87,7 @@ class MixLock(object): if self._sh_cnt: self.lock_escalation(LockManager.TRY_TIMES) else: - raise e + raise LockError(e) self._ex_cnt += 1 self.stdio and getattr(self.stdio, 'verbose', print)('exclusive lock `%s`, count %s' % (self.path, self._ex_cnt)) return True @@ -92,7 +99,7 @@ class MixLock(object): self.stdio and getattr(self.stdio, 'stop_loading', print)('succeed') except Exception as e: self.stdio and getattr(self.stdio, 'stop_loading', print)('fail') - raise e + raise LockError(e) def _lock_escalation(self, try_times): stdio = self.stdio @@ -107,13 +114,13 @@ class MixLock(object): break except KeyboardInterrupt: self.stdio = stdio - raise IOError('fail to get lock') + raise Exception('fail to get lock') except Exception as e: if try_times: time.sleep(LockManager.TRY_INTERVAL) else: self.stdio = stdio - raise e + raise LockError(e) self.stdio = stdio def _sh_unlock(self): @@ -209,12 +216,19 @@ class LockManager(Manager): def __del__(self): for lock in self.locks[::-1]: lock.unlock() + self.locks = None def _get_mix_lock(self, path): if path not in self.LOCKS: self.LOCKS[path] = MixLock(path, stdio=self.stdio) return self.LOCKS[path] + @classmethod + def shutdown(cls): + for path in cls.LOCKS: + cls.LOCKS[path] = None + cls.LOCKS = None + def _lock(self, path, clz): mix_lock = self._get_mix_lock(path) lock = clz(mix_lock) @@ -248,3 +262,7 @@ class LockManager(Manager): def deploy_sh_lock(self, deploy_name): return self._sh_lock(self._deploy_lock_fp(deploy_name)) + + +import atexit +atexit.register(LockManager.shutdown) \ No newline at end of file diff --git a/_mirror.py b/_mirror.py index 5e5bddc2dd6465a50edb7cdb9748de90de3e9d73..8295f76602b12a23ef149ffea9b0dac673e01038 100644 --- a/_mirror.py +++ b/_mirror.py @@ -534,7 +534,7 @@ class RemoteMirrorRepository(MirrorRepository): if release and info.release != release: raise Exception ('break') return [0 ,] - + c = [len(name) / len(info.name), info] return c @@ -763,7 +763,7 @@ class LocalMirrorRepository(MirrorRepository): return [0 ,] if release and info.release != release: return [0 ,] - + c = [len(name) / len(info.name), info] return c @@ -1008,8 +1008,8 @@ class MirrorRepositoryManager(Manager): enabled_str = '1' if enabled else '0' self.stdio and getattr(self.stdio, 'start_loading')('%s %s' % (op, section_name)) if section_name == 'local': - self.stdio and getattr(self.stdio, 'error', print)('%s is local mirror repository.' % (section_name)) - return + self.stdio and getattr(self.stdio, 'error', print)('Local mirror repository CANNOT BE %sD.' % op.upper()) + return False if section_name == 'remote': self._scan_repo_configs() for repo_config in self._cache_path_repo_config.values(): @@ -1024,21 +1024,22 @@ class MirrorRepositoryManager(Manager): mirror_section.meta_data['enabled'] = enabled_str mirror_section.meta_data['repo_age'] = repo_age self.stdio and getattr(self.stdio, 'stop_loading')('succeed') + return True else: mirror_section = self._get_section(section_name) if not mirror_section: self.stdio and getattr(self.stdio, 'error', print)('%s not found.' % (section_name)) self.stdio and getattr(self.stdio, 'stop_loading')('fail') - return + return False if mirror_section.is_enabled == enabled: - self.stdio and getattr(self.stdio, 'print', print)('%s is already %sd' % (section_name, op)) + self.stdio and getattr(self.stdio, 'print', print)('%s is already %sd' % (section_name, op.lower())) self.stdio and getattr(self.stdio, 'stop_loading')('succeed') - return + return True repo_config = self._get_repo_config_by_section(section_name) if not repo_config: self.stdio and getattr(self.stdio, 'error', print)('%s not found.' % (section_name)) self.stdio and getattr(self.stdio, 'stop_loading')('fail') - return + return False repo_config.parser.set(section_name, 'enabled', enabled_str) with FileUtil.open(repo_config.path, 'w', stdio=self.stdio) as confpp_obj: @@ -1049,4 +1050,5 @@ class MirrorRepositoryManager(Manager): repo_config.repo_age = repo_age mirror_section.meta_data['enabled'] = enabled_str mirror_section.meta_data['repo_age'] = repo_age - self.stdio and getattr(self.stdio, 'stop_loading')('succeed') \ No newline at end of file + self.stdio and getattr(self.stdio, 'stop_loading')('succeed') + return True diff --git a/_rpm.py b/_rpm.py index 3795dfb700384c3dd918489a20fed2995edb0f7d..17cd2e13d4ce7d159f782ccca1d3dbbf35f722c6 100644 --- a/_rpm.py +++ b/_rpm.py @@ -36,6 +36,7 @@ if sys.version_info.major == 2: from backports import lzma setattr(sys.modules['rpmfile'], 'lzma', getattr(sys.modules[__name__], 'lzma')) + class Version(str): def __init__(self, bytes_or_buffer, encoding=None, errors=None): diff --git a/_stdio.py b/_stdio.py index fb5c2e67bb8a1e1759d1493da0a1fb775f4b850e..3f022d0c05f4af5b3817c2bbd0972fc88421c1e6 100644 --- a/_stdio.py +++ b/_stdio.py @@ -21,6 +21,7 @@ from __future__ import absolute_import, division, print_function import os +import signal import sys import traceback @@ -28,7 +29,7 @@ from enum import Enum from halo import Halo, cursor from colorama import Fore from prettytable import PrettyTable -from progressbar import Bar, ETA, FileTransferSpeed, Percentage, ProgressBar +from progressbar import AdaptiveETA, Bar, SimpleProgress, ETA, FileTransferSpeed, Percentage, ProgressBar if sys.version_info.major == 3: @@ -145,11 +146,19 @@ class IOHalo(Halo): class IOProgressBar(ProgressBar): - def __init__(self, maxval=None, text='', term_width=None, poll=1, left_justify=True, stream=None): - widgets=['%s: ' % text, Percentage(), ' ', - Bar(marker='#', left='[', right=']'), - ' ', ETA(), ' ', FileTransferSpeed()] - super(IOProgressBar, self).__init__(maxval=maxval, widgets=widgets, term_width=term_width, poll=poll, left_justify=left_justify, fd=stream) + @staticmethod + def _get_widgets(widget_type, text): + if widget_type == 'download': + return ['%s: ' % text, Percentage(), ' ', Bar(marker='#', left='[', right=']'), ' ', ETA(), ' ', FileTransferSpeed()] + elif widget_type == 'timer': + return ['%s: ' % text, Percentage(), ' ', Bar(marker='#', left='[', right=']'), ' ', AdaptiveETA()] + elif widget_type == 'simple_progress': + return ['%s: (' % text, SimpleProgress(sep='/'), ') ', Bar(marker='#', left='[', right=']')] + else: + return ['%s: ' % text, Percentage(), ' ', Bar(marker='#', left='[', right=']')] + + def __init__(self, maxval=None, text='', term_width=None, poll=1, left_justify=True, stream=None, widget_type='download'): + super(IOProgressBar, self).__init__(maxval=maxval, widgets=self._get_widgets(widget_type, text), term_width=term_width, poll=poll, left_justify=left_justify, fd=stream) def start(self): self._hide_cursor() @@ -159,9 +168,20 @@ class IOProgressBar(ProgressBar): return super(IOProgressBar, self).update(value=value) def finish(self): + if self.finished: + return self._show_cursor() return super(IOProgressBar, self).finish() + def interrupt(self): + if self.finished: + return + self._show_cursor() + self.finished = True + self.fd.write('\n') + if self.signal_set: + signal.signal(signal.SIGWINCH, signal.SIG_DFL) + def _need_update(self): return (self.currval == self.maxval or self.currval == 0 or getattr(self.fd, 'isatty', lambda : False)()) \ and super(IOProgressBar, self)._need_update() @@ -356,10 +376,17 @@ class IO(object): else: return self._stop_sync_obj(IOHalo, 'stop') - def start_progressbar(self, text, maxval): + def update_loading_text(self, text): + if not isinstance(self.sync_obj, IOHalo): + return False + self.log(MsgLevel.VERBOSE, text) + self.sync_obj.text = text + return self.sync_obj + + def start_progressbar(self, text, maxval, widget_type='download'): if self.sync_obj: return False - self.sync_obj = self._start_sync_obj(IOProgressBar, lambda x: x.finish_progressbar(), text=text, maxval=maxval) + self.sync_obj = self._start_sync_obj(IOProgressBar, lambda x: x.finish_progressbar(), text=text, maxval=maxval, widget_type=widget_type) if self.sync_obj: self.log(MsgLevel.INFO, text) return self.sync_obj.start() @@ -373,6 +400,11 @@ class IO(object): if not isinstance(self.sync_obj, IOProgressBar): return False return self._stop_sync_obj(IOProgressBar, 'finish') + + def interrupt_progressbar(self): + if not isinstance(self.sync_obj, IOProgressBar): + return False + return self._stop_sync_obj(IOProgressBar, 'interrupt') def sub_io(self, pid=None, msg_lv=None): if not pid: @@ -381,7 +413,7 @@ class IO(object): msg_lv = self.msg_lv key = "%s-%s" % (pid, msg_lv) if key not in self.sub_ios: - self.sub_ios[key] = IO( + self.sub_ios[key] = self.__class__( self.level + 1, msg_lv=msg_lv, trace_logger=self.trace_logger, diff --git a/config_parser/oceanbase-ce b/config_parser/oceanbase-ce new file mode 120000 index 0000000000000000000000000000000000000000..a304487557241f9f414dd00690cf145465ed8b45 --- /dev/null +++ b/config_parser/oceanbase-ce @@ -0,0 +1 @@ +oceanbase \ No newline at end of file diff --git a/config_parser/oceanbase/cluster_config_parser.py b/config_parser/oceanbase/cluster_config_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..4d24a22b378790c9e34278da6e99a9b6edcb44e5 --- /dev/null +++ b/config_parser/oceanbase/cluster_config_parser.py @@ -0,0 +1,240 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +import re +from copy import deepcopy +from collections import OrderedDict + +from tool import ConfigUtil +from _deploy import ( + InnerConfigItem, + ServerConfigFlyweightFactory, + ClusterConfig, + ConfigParser, + CommentedMap +) + + +class ClusterConfigParser(ConfigParser): + + STYLE = 'cluster' + INNER_CONFIG_MAP = { + '$_zone_idc': 'idc' + } + + @classmethod + def get_server_src_conf(cls, cluster_config, component_config, server): + server_config = cluster_config.get_server_conf(server) + zones = component_config['zones'] + zone_name = server_config.get('zone', list(zones.keys())[0]) + zone = zones[zone_name] + if 'config' not in zone: + zone['config'] = {} + zone_config = zone['config'] + if server.name not in zone_config: + zone_config[server.name] = {} + return zone_config[server.name] + + @classmethod + def _to_cluster_config(cls, component_name, conf): + servers = OrderedDict() + zones = conf.get('zones', {}) + for zone_name in zones: + zone = zones[zone_name] + zone_servers = zone.get('servers', []) + zone_configs = zone.get('config', {}) + zone_global_config = zone_configs.get('global', {}) + for server in zone_servers: + if isinstance(server, dict): + ip = ConfigUtil.get_value_from_dict(server, 'ip', transform_func=str) + name = ConfigUtil.get_value_from_dict(server, 'name', transform_func=str) + else: + ip = server + name = None + if not re.match('^\d{1,3}(\\.\d{1,3}){3}$', ip): + continue + server = ServerConfigFlyweightFactory.get_instance(ip, name) + if server not in servers: + server_config = deepcopy(zone_global_config) + if server.name in zone_configs: + server_config.update(zone_configs[server.name]) + if 'idc' in zone: + key = '$_zone_idc' + if key in server_config: + del server_config[key] + server_config[InnerConfigItem(key)] = str(zone['idc']) + server_config['zone'] = zone_name + servers[server] = server_config + + cluster_conf = ClusterConfig( + servers.keys(), + component_name, + ConfigUtil.get_value_from_dict(conf, 'version', None, str), + ConfigUtil.get_value_from_dict(conf, 'tag', None, str), + ConfigUtil.get_value_from_dict(conf, 'package_hash', None, str) + ) + global_config = {} + if 'id' in conf: + global_config['cluster_id'] = int(conf['id']) + if 'name' in conf: + global_config['appname'] = str(conf['name']) + if 'config' in conf: + global_config.update(conf['config']) + cluster_conf.set_global_conf(global_config) + + for server in servers: + cluster_conf.add_server_conf(server, servers[server]) + return cluster_conf + + @classmethod + def extract_inner_config(cls, cluster_config, config): + inner_config = cluster_config.get_inner_config() + for server in inner_config: + server_config = inner_config[server] + keys = list(server_config.keys()) + for key in keys: + if key in cls.INNER_CONFIG_MAP: + del server_config[key] + + for server in cluster_config.servers: + if server.name not in inner_config: + inner_config[server.name] = {} + server_config = cluster_config.get_server_conf(server) + keys = list(server_config.keys()) + for key in keys: + if cls._is_inner_item(key) and key not in cls.INNER_CONFIG_MAP: + inner_config[server.name] = server_config[key] + del server_config[key] + return inner_config + + @classmethod + def _from_cluster_config(cls, conf, cluster_config): + global_config_items = {} + zones_config = {} + for server in cluster_config.servers: + server_config = cluster_config.get_server_conf(server) + server_config_with_default = cluster_config.get_server_conf_with_default(server) + zone_name = server_config_with_default.get('zone', 'zone1') + if zone_name not in zones_config: + zones_config[zone_name] = { + 'servers': OrderedDict(), + 'config': OrderedDict(), + } + zone_servers = zones_config[zone_name]['servers'] + zone_config_items = zones_config[zone_name]['config'] + zone_servers[server] = server_config + for key in server_config: + if key in zone_config_items: + if zone_config_items[key]['value'] == server_config[key]: + zone_config_items[key]['count'] += 1 + else: + zone_config_items[key] = { + 'value': server_config[key], + 'count': 1 + } + + zones = CommentedMap() + server_num = len(cluster_config.servers) + for zone_name in zones_config: + zones[zone_name] = CommentedMap() + zone_global_config = {} + zone_servers = zones_config[zone_name]['servers'] + zone_config_items = zones_config[zone_name]['config'] + zone_server_num = len(zone_servers) + zone_global_config_items = {} + for key in zone_config_items: + item = zone_config_items[key] + clear_item = isinstance(key, InnerConfigItem) + if item['count'] == zone_server_num: + zone_global_config_items[key] = item['value'] + if key in global_config_items: + if global_config_items[key]['value'] == zone_global_config_items[key]: + global_config_items[key]['count'] += 1 + else: + global_config_items[key] = { + 'value': zone_global_config_items[key], + 'count': 1 + } + clear_item = True + + if clear_item: + for server in zone_servers: + del zone_servers[server][key] + + for key in zone_global_config_items: + if cls._is_inner_item(key): + if key in cls.INNER_CONFIG_MAP: + zones[zone_name][cls.INNER_CONFIG_MAP[key]] = zone_global_config_items[key] + else: + zone_global_config[key] = zone_global_config_items[key] + + zones[zone_name]['servers'] = [] + zone_config = {} + + if 'zone' in zone_global_config: + del zone_global_config['zone'] + if zone_global_config: + zone_config['global'] = zone_global_config + + for server in zone_servers: + if server.name == server.ip: + zones[zone_name]['servers'].append(server.name) + else: + zones[zone_name]['servers'].append({'name': server.name, 'ip': server.ip}) + if zone_servers[server]: + zone_config[server.name] = zone_servers[server] + if zone_config: + zones[zone_name]['config'] = zone_config + + global_config = CommentedMap() + zone_num = len(zones) + del global_config_items['zone'] + for key in global_config_items: + item = global_config_items[key] + if item['count'] == zone_num: + global_config[key] = item['value'] + for zone_name in zones: + del zones[zone_name]['config']['global'][key] + + for zone_name in zones: + if 'config' in zones[zone_name]: + configs = zones[zone_name]['config'] + keys = list(configs.keys()) + for key in keys: + if not configs[key]: + del configs[key] + if not configs: + del zones[zone_name]['config'] + + + if 'cluster_id' in global_config: + conf['id'] = global_config['cluster_id'] + del global_config['cluster_id'] + if 'appname' in global_config: + conf['name'] = global_config['appname'] + del global_config['appname'] + + if global_config: + conf['config'] = global_config + conf['zones'] = zones + + return conf \ No newline at end of file diff --git a/core.py b/core.py index 2c335ab0eb91c74fdca23ae1f80a7df31beb6b1f..58454c2b422eaab596043b699e589daf855a650f 100644 --- a/core.py +++ b/core.py @@ -39,7 +39,12 @@ from _rpm import Version from _mirror import MirrorRepositoryManager, PackageInfo from _plugin import PluginManager, PluginType, InstallPlugin from _repository import RepositoryManager, LocalPackage, Repository -from _deploy import DeployManager, DeployStatus, DeployConfig, DeployConfigStatus +from _deploy import ( + DeployManager, DeployStatus, + DeployConfig, DeployConfigStatus, + ParserError, Deploy +) +from _errno import EC_SOME_SERVER_STOPED from _lock import LockManager @@ -179,7 +184,12 @@ class ObdHome(object): return None return plugins - def search_py_script_plugin(self, repositories, script_name, no_found_exit=True): + def search_py_script_plugin(self, repositories, script_name, no_found_act='exit'): + if no_found_act == 'exit': + no_found_exit = True + else: + no_found_exit = False + msg_lv = 'warn' if no_found_act == 'warn' else 'verbose' plugins = {} self._call_stdio('verbose', 'Searching %s plugin for components ...', script_name) for repository in repositories: @@ -193,7 +203,7 @@ class ObdHome(object): self._call_stdio('critical', 'No such %s plugin for %s-%s' % (script_name, repository.name, repository.version)) break else: - self._call_stdio('warn', 'No such %s plugin for %s-%s' % (script_name, repository.name, repository.version)) + self._call_stdio(msg_lv, 'No such %s plugin for %s-%s' % (script_name, repository.name, repository.version)) return plugins def search_images(self, component_name, version, release=None, disable=[], usable=[], release_first=False): @@ -337,15 +347,6 @@ class ObdHome(object): if deploy_config.components[component_name].servers != deploy.deploy_config.components[component_name].servers: return True return False - def effect_tip(config_status): - cmd_map = { - DeployConfigStatus.NEED_RELOAD: 'obd cluster reload %s' % name, - DeployConfigStatus.NEED_RESTART: 'obd cluster restart %s --wp' % name, - DeployConfigStatus.NEED_REDEPLOY: 'obd cluster redeploy %s' % name, - } - if config_status in cmd_map: - return '\nUse `%s` to make changes take effect.' % cmd_map[config_status] - return '' self._call_stdio('verbose', 'Get Deploy by name') deploy = self.deploy_manager.get_deploy_config(name) param_plugins = {} @@ -358,7 +359,7 @@ class ObdHome(object): if deploy.deploy_info.config_status == DeployConfigStatus.UNCHNAGE: path = deploy.deploy_config.yaml_path else: - path = deploy.get_temp_deploy_yaml_path(deploy.config_dir) + path = Deploy.get_temp_deploy_yaml_path(deploy.config_dir) self._call_stdio('verbose', 'Load %s' % path) with open(path, 'r') as f: initial_config = f.read() @@ -373,6 +374,7 @@ class ObdHome(object): msg = 'Create deploy "%s" configuration' % name if is_deployed: repositories = self.load_local_repositories(deploy.deploy_info) + self._call_stdio('start_loading', 'Search param plugin and load') for repository in repositories: self._call_stdio('verbose', 'Search param plugin for %s' % repository) plugin = self.plugin_manager.get_best_plugin(PluginType.PARAM, repository.name, repository.version) @@ -381,6 +383,8 @@ class ObdHome(object): cluster_config = deploy.deploy_config.components[repository.name] cluster_config.update_temp_conf(plugin.params) param_plugins[repository.name] = plugin + self._call_stdio('stop_loading', 'succeed') + EDITOR = os.environ.get('EDITOR','vi') self._call_stdio('verbose', 'Get environment variable EDITOR=%s' % EDITOR) self._call_stdio('verbose', 'Create tmp yaml file') @@ -388,12 +392,19 @@ class ObdHome(object): tf.write(initial_config.encode()) tf.flush() self.lock_manager.set_try_times(-1) + config_status = DeployConfigStatus.UNCHNAGE while True: tf.seek(0) self._call_stdio('verbose', '%s %s' % (EDITOR, tf.name)) subprocess_call([EDITOR, tf.name]) self._call_stdio('verbose', 'Load %s' % tf.name) - deploy_config = DeployConfig(tf.name, YamlLoader(self.stdio)) + try: + deploy_config = DeployConfig(tf.name, yaml_loader=YamlLoader(self.stdio), config_parser_manager=self.deploy_manager.config_parser_manager) + except Exception as e: + if confirm(e): + continue + break + self._call_stdio('verbose', 'Configure component change check') if not deploy_config.components: if self._call_stdio('confirm', 'Empty configuration. Continue editing?'): @@ -401,34 +412,25 @@ class ObdHome(object): return False self._call_stdio('verbose', 'Information check for the configuration component.') if not deploy: - config_status = DeployConfigStatus.NEED_REDEPLOY + config_status = DeployConfigStatus.UNCHNAGE elif is_deployed: - if deploy_config.components.keys() != deploy.deploy_config.components.keys(): - if confirm('Modifying the component list of a deployed cluster is not permitted.'): + if deploy_config.components.keys() != deploy.deploy_config.components.keys() or is_server_list_change(deploy_config): + if not self._call_stdio('confirm', 'Modifications to the deployment architecture take effect after you redeploy the architecture. Are you sure that you want to start a redeployment? '): continue - return False - if is_server_list_change(deploy_config): - if confirm('Modifying the server list of a deployed cluster is not permitted.'): - continue - return False - - success = True - for component_name in deploy_config.components: - old_cluster_config = deploy.deploy_config.components[component_name] - new_cluster_config = deploy_config.components[component_name] - if new_cluster_config.version != old_cluster_config.origin_version \ - or new_cluster_config.package_hash != old_cluster_config.origin_package_hash \ - or new_cluster_config.tag != old_cluster_config.origin_tag: - success = False - break - if not success: - if confirm('Modifying the version and hash of the component is not permitted.'): - continue - return False + config_status = DeployConfigStatus.NEED_REDEPLOY + else: + for component_name in deploy_config.components: + old_cluster_config = deploy.deploy_config.components[component_name] + new_cluster_config = deploy_config.components[component_name] + if new_cluster_config.version != old_cluster_config.origin_version \ + or new_cluster_config.package_hash != old_cluster_config.origin_package_hash \ + or new_cluster_config.tag != old_cluster_config.origin_tag: + config_status = DeployConfigStatus.NEED_REDEPLOY + break # Loading the parameter plugins that are available to the application self._call_stdio('start_loading', 'Search param plugin and load') - if not is_deployed: + if not is_deployed or config_status == DeployConfigStatus.NEED_REDEPLOY: param_plugins = {} pkgs, repositories, errors = self.search_components_from_mirrors(deploy_config, update_if_need=False) for repository in repositories: @@ -459,12 +461,13 @@ class ObdHome(object): self._call_stdio('verbose', 'configure change check') if initial_config and initial_config == tf.read().decode(errors='replace'): config_status = deploy.deploy_info.config_status if deploy else DeployConfigStatus.UNCHNAGE - self._call_stdio('print', 'Deploy "%s" config %s%s' % (name, config_status.value, effect_tip(config_status))) + self._call_stdio('print', 'Deploy "%s" config %s%s' % (name, config_status.value, deploy.effect_tip() if deploy else '')) return True - config_status = DeployConfigStatus.UNCHNAGE - if is_deployed: + if is_deployed and config_status != DeployConfigStatus.NEED_REDEPLOY: if is_started: + if deploy.deploy_config.user.username != deploy_config.user.username: + config_status = DeployConfigStatus.NEED_RESTART errors = [] for component_name in param_plugins: old_cluster_config = deploy.deploy_config.components[component_name] @@ -481,9 +484,13 @@ class ObdHome(object): self._call_stdio('exceptione', '') errors.append('[%s] %s: %s' % (component_name, server, str(e))) if errors: - if confirm('\n'.join(errors)): + self._call_stdio('print', '\n'.join(errors)) + if self._call_stdio('confirm', 'Modifications take effect after a redeployment. Are you sure that you want to start a redeployment?'): + config_status = DeployConfigStatus.NEED_REDEPLOY + elif self._call_stdio('confirm', 'Continue to edit?'): continue - return False + else: + return False for component_name in deploy_config.components: if config_status == DeployConfigStatus.NEED_REDEPLOY: @@ -509,7 +516,7 @@ class ObdHome(object): if config_status == DeployConfigStatus.UNCHNAGE: ret = self.deploy_manager.create_deploy_config(name, tf.name).update_deploy_config_status(config_status) else: - target_src_path = deploy.get_temp_deploy_yaml_path(deploy.config_dir) + target_src_path = Deploy.get_temp_deploy_yaml_path(deploy.config_dir) old_config_status = deploy.deploy_info.config_status try: if deploy.update_deploy_config_status(config_status): @@ -519,7 +526,7 @@ class ObdHome(object): if deploy.deploy_info.status == DeployStatus.STATUS_RUNNING or ( config_status == DeployConfigStatus.NEED_REDEPLOY and is_deployed ): - msg += effect_tip(config_status) + msg += deploy.effect_tip() except Exception as e: deploy.update_deploy_config_status(old_config_status) self._call_stdio('exception', 'Copy %s to %s failed, error: \n%s' % (tf.name, target_src_path, e)) @@ -587,7 +594,7 @@ class ObdHome(object): with tempfile.NamedTemporaryFile(suffix=".yaml", mode='w') as tf: yaml_loader = YamlLoader(self.stdio) yaml_loader.dump(data, tf) - deploy_config = DeployConfig(tf.name, yaml_loader) + deploy_config = DeployConfig(tf.name, yaml_loader=yaml_loader, config_parser_manager=self.deploy_manager.config_parser_manager) # Look for the best suitable mirrors for the components self._call_stdio('verbose', 'Search best suitable repository libs') pkgs, lib_repositories, errors = self.search_components_from_mirrors(deploy_config, only_info=False) @@ -619,7 +626,7 @@ class ObdHome(object): for server in servers: self._call_stdio('verbose', '%s %s repository integrity check' % (server, repository)) client = ssh_clients[server] - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_data_path = repository.data_file_path.replace(self.home_path, remote_home_path) remote_repository_data = client.execute_command('cat %s' % remote_repository_data_path).stdout self._call_stdio('verbose', '%s %s install check' % (server, repository)) @@ -651,7 +658,7 @@ class ObdHome(object): self._call_stdio('verbose', '%s %s repository lib check' % (server, repository)) client = ssh_clients[server] need_libs = set() - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_path = repository.repository_dir.replace(self.home_path, remote_home_path) remote_repository_data_path = repository.data_file_path.replace(self.home_path, remote_home_path) client.add_env('LD_LIBRARY_PATH', '%s/lib:' % remote_repository_path, True) @@ -681,7 +688,7 @@ class ObdHome(object): for server in cluster_config.servers: client = ssh_clients[server] if server not in servers_obd_home: - servers_obd_home[server] = client.execute_command('echo $HOME/.obd').stdout.strip() + servers_obd_home[server] = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_home_path = servers_obd_home[server] remote_lib_repository_data_path = lib_repository.repository_dir.replace(self.home_path, remote_home_path) # lib installation @@ -746,6 +753,27 @@ class ObdHome(object): install_plugins = self.get_install_plugin_and_install(repositories, pkgs) return repositories, install_plugins + def sort_repositories_by_depends(self, deploy_config, repositories): + sort_repositories = [] + wait_repositories = repositories + imported_depends = [] + available_depends = [repository.name for repository in repositories] + while wait_repositories: + repositories = wait_repositories + wait_repositories = [] + for repository in repositories: + cluster_config = deploy_config.components[repository.name] + for component_name in cluster_config.depends: + if component_name not in available_depends: + continue + if component_name not in imported_depends: + wait_repositories.append(repository) + break + else: + sort_repositories.append(repository) + imported_depends.append(repository.name) + return sort_repositories + def genconfig(self, name, opt=Values()): self._call_stdio('verbose', 'Get Deploy by name') deploy = self.deploy_manager.get_deploy_config(name) @@ -817,6 +845,144 @@ class ObdHome(object): self.deploy_manager.remove_deploy_config(name) return False + def check_for_ocp(self, name, options=Values()): + self._call_stdio('verbose', 'Get Deploy by name') + deploy = self.deploy_manager.get_deploy_config(name) + if not deploy: + self._call_stdio('error', 'No such deploy: %s.' % name) + return False + + deploy_info = deploy.deploy_info + self._call_stdio('verbose', 'Deploy status judge') + if deploy_info.status != DeployStatus.STATUS_RUNNING: + self._call_stdio('error', 'Deploy "%s" not RUNNING' % (name)) + return False + + version = getattr(options, 'version', '') + if not version: + self._call_stdio('error', 'Use the --version option to specify the required OCP version.') + return False + + deploy_config = deploy.deploy_config + components = getattr(options, 'components', '') + if components: + components = components.split(',') + for component in components: + if component not in deploy_config.components: + self._call_stdio('error', 'No such component: %s' % component) + return False + else: + components = deploy_config.components.keys() + + self._call_stdio('start_loading', 'Get local repositories and plugins') + # Get the repository + repositories = self.load_local_repositories(deploy_info) + + ocp_check = self.search_py_script_plugin(repositories, 'ocp_check', no_found_act='ignore') + connect_plugins = self.search_py_script_plugin([repository for repository in ocp_check], 'connect') + + self._call_stdio('stop_loading', 'succeed') + + self._call_stdio('start_loading', 'Load cluster param plugin') + # Check whether the components have the parameter plugins and apply the plugins + self.search_param_plugin_and_apply(repositories, deploy_config) + if deploy_info.config_status != DeployConfigStatus.UNCHNAGE: + new_deploy_config = deploy.temp_deploy_config + change_user = deploy_config.user.username != new_deploy_config.user.username + self.search_param_plugin_and_apply(repositories, new_deploy_config) + else: + new_deploy_config = None + + self._call_stdio('stop_loading', 'succeed') + + # Get the client + ssh_clients = self.get_clients(deploy_config, repositories) + if new_deploy_config and deploy_config.user.username != new_deploy_config.user.username: + new_ssh_clients = self.get_clients(new_deploy_config, repositories) + else: + new_ssh_clients = None + + component_num = len(repositories) + for repository in repositories: + if repository.name not in components: + continue + if repository not in ocp_check: + component_num -= 1 + self._call_stdio('print', '%s No check plugin available.' % repository.name) + continue + + cluster_config = deploy_config.components[repository.name] + new_cluster_config = new_deploy_config.components[repository.name] if new_deploy_config else None + cluster_servers = cluster_config.servers + + self._call_stdio('verbose', 'Call %s for %s' % (connect_plugins[repository], repository)) + ret = connect_plugins[repository](deploy_config.components.keys(), ssh_clients, cluster_config, '', options, self.stdio) + if ret: + db = ret.get_return('connect') + cursor = ret.get_return('cursor') + else: + self._call_stdio('error', 'Failed to connect %s' % repository.name) + break + + self._call_stdio('verbose', 'Call %s for %s' % (ocp_check[repository], repository)) + if ocp_check[repository](deploy_config.components.keys(), ssh_clients, cluster_config, '', options, self.stdio, cursor=cursor, ocp_version=version, new_cluster_config=new_cluster_config, new_clients=new_ssh_clients): + component_num -= 1 + self._call_stdio('print', '%s Check passed.' % repository.name) + + return component_num == 0 + + def change_deploy_config_style(self, name, options=Values()): + self._call_stdio('verbose', 'Get Deploy by name') + deploy = self.deploy_manager.get_deploy_config(name) + if not deploy: + self._call_stdio('error', 'No such deploy: %s.' % name) + return False + + deploy_info = deploy.deploy_info + self._call_stdio('verbose', 'Deploy config status judge') + if deploy_info.config_status != DeployConfigStatus.UNCHNAGE: + self._call_stdio('error', 'Deploy %s %s' % (name, deploy_info.config_status.value)) + return False + deploy_config = deploy.deploy_config + if not deploy_config: + self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.') + return False + + style = getattr(options, 'style', '') + if not style: + self._call_stdio('error', 'Use the --style option to specify the preferred style.') + return False + + components = getattr(options, 'components', '') + if components: + components = components.split(',') + for component in components: + if component not in deploy_config.components: + self._call_stdio('error', 'No such component: %s' % component) + return False + else: + components = deploy_config.components.keys() + + self._call_stdio('start_loading', 'Change style') + try: + parsers = {} + for component_name in components: + parsers[component_name] = self.deploy_manager.config_parser_manager.get_parser(component_name, style) + self._call_stdio('verbose', 'get %s for %s' % (parsers[component_name], component_name)) + + for component_name in deploy_config.components: + if component_name in parsers: + deploy_config.change_component_config_style(component_name, style) + if deploy_config.dump(): + self._call_stdio('stop_loading', 'succeed') + return True + except Exception as e: + self._call_stdio('exception', e) + + self._call_stdio('stop_loading', 'fail') + return False + + def deploy_cluster(self, name, opt=Values()): self._call_stdio('verbose', 'Get Deploy by name') deploy = self.deploy_manager.get_deploy_config(name) @@ -977,10 +1143,8 @@ class ObdHome(object): self._call_stdio('error', 'Deploy needs redeploy') return False if deploy_info.config_status != DeployConfigStatus.UNCHNAGE and not getattr(options, 'without_parameter', False): - self._call_stdio('verbose', 'Apply temp deploy configuration') - if not deploy.apply_temp_deploy_config(): - self._call_stdio('error', 'Failed to apply new deploy configuration') - return False + self._call_stdio('error', 'Deploy %s.%s\nIf you still need to start the cluster, use the `obd cluster start %s --wop` option to start the cluster without loading parameters. ' % (deploy_info.config_status.value, deploy.effect_tip(), name)) + return False self._call_stdio('verbose', 'Get deploy config') deploy_config = deploy.deploy_config @@ -1006,8 +1170,8 @@ class ObdHome(object): # Get the repository repositories = self.load_local_repositories(deploy_info, False) - start_check_plugins = self.search_py_script_plugin(repositories, 'start_check', False) - create_tenant_plugins = self.search_py_script_plugin(repositories, 'create_tenant', False) if deploy_config.auto_create_tenant else {} + start_check_plugins = self.search_py_script_plugin(repositories, 'start_check', no_found_act='warn') + create_tenant_plugins = self.search_py_script_plugin(repositories, 'create_tenant', no_found_act='ignore') if deploy_config.auto_create_tenant else {} start_plugins = self.search_py_script_plugin(repositories, 'start') connect_plugins = self.search_py_script_plugin(repositories, 'connect') bootstrap_plugins = self.search_py_script_plugin(repositories, 'bootstrap') @@ -1082,7 +1246,7 @@ class ObdHome(object): self._call_stdio('print', 'Initialize cluster') self._call_stdio('verbose', 'Call %s for %s' % (bootstrap_plugins[repository], repository)) if not bootstrap_plugins[repository](deploy_config.components.keys(), ssh_clients, cluster_config, cmd, options, self.stdio, cursor): - self._call_stdio('print', 'Cluster init failed') + self._call_stdio('error', 'Cluster init failed') break if repository in create_tenant_plugins: create_tenant_options = Values({"variables": "ob_tcp_invited_nodes='%'"}) @@ -1131,7 +1295,7 @@ class ObdHome(object): self.search_param_plugin_and_apply(repositories, deploy_config) connect_plugins = self.search_py_script_plugin(repositories, 'connect') - create_tenant_plugins = self.search_py_script_plugin(repositories, 'create_tenant', False) + create_tenant_plugins = self.search_py_script_plugin(repositories, 'create_tenant', no_found_act='ignore') self._call_stdio('stop_loading', 'succeed') # Get the client @@ -1178,7 +1342,7 @@ class ObdHome(object): self.search_param_plugin_and_apply(repositories, deploy_config) connect_plugins = self.search_py_script_plugin(repositories, 'connect') - drop_tenant_plugins = self.search_py_script_plugin(repositories, 'drop_tenant', False) + drop_tenant_plugins = self.search_py_script_plugin(repositories, 'drop_tenant', no_found_act='ignore') self._call_stdio('stop_loading', 'succeed') # Get the client @@ -1219,10 +1383,27 @@ class ObdHome(object): self._call_stdio('print', 'Deploy config is UNCHNAGE') return True + if deploy_info.config_status != DeployConfigStatus.NEED_RELOAD: + self._call_stdio('error', 'Deploy `%s` %s%s' % (name, deploy_info.config_status.value, deploy.effect_tip())) + return False + + return self._reload_cluster(deploy) + + def _reload_cluster(self, deploy): + deploy_info = deploy.deploy_info self._call_stdio('verbose', 'Get deploy config') deploy_config = deploy.deploy_config self._call_stdio('verbose', 'Get new deploy config') - new_deploy_config = DeployConfig(deploy.get_temp_deploy_yaml_path(deploy.config_dir), YamlLoader(self.stdio)) + new_deploy_config = deploy.temp_deploy_config + + if deploy_config.components.keys() != new_deploy_config.components.keys(): + self._call_stdio('error', 'The deployment architecture is changed and cannot be reloaded.') + return False + + for component_name in deploy_config.components: + if deploy_config.components[component_name].servers != new_deploy_config.components[component_name].servers: + self._call_stdio('error', 'The deployment architecture is changed and cannot be reloaded.') + return False self._call_stdio('start_loading', 'Get local repositories and plugins') # Get the repository @@ -1247,7 +1428,7 @@ class ObdHome(object): cluster_status = self.cluster_status_check(ssh_clients, deploy_config, repositories, component_status) if cluster_status is False or cluster_status == 0: if self.stdio: - self._call_stdio('error', 'Some of the servers in the cluster have been stopped') + self._call_stdio('error', EC_SOME_SERVER_STOPED) for repository in component_status: cluster_status = component_status[repository] for server in cluster_status: @@ -1255,6 +1436,7 @@ class ObdHome(object): self._call_stdio('print', '%s %s is stopped' % (server, repository.name)) return False + repositories = self.sort_repositories_by_depends(deploy_config, repositories) component_num = len(repositories) for repository in repositories: cluster_config = deploy_config.components[repository.name] @@ -1277,7 +1459,7 @@ class ObdHome(object): component_num -= 1 if component_num == 0: if deploy.apply_temp_deploy_config(): - self._call_stdio('print', '%s reload' % name) + self._call_stdio('print', '%s reload' % deploy.name) return True else: deploy_config.dump() @@ -1318,7 +1500,7 @@ class ObdHome(object): cluster_status = self.cluster_status_check(ssh_clients, deploy_config, repositories, component_status) if cluster_status is False or cluster_status == 0: if self.stdio: - self._call_stdio('error', 'Some of the servers in the cluster have been stopped') + self._call_stdio('error', EC_SOME_SERVER_STOPED) for repository in component_status: cluster_status = component_status[repository] for server in cluster_status: @@ -1422,7 +1604,7 @@ class ObdHome(object): return True return False - def restart_cluster(self, name, opt=Values()): + def restart_cluster(self, name, options=Values()): self._call_stdio('verbose', 'Get Deploy by name') deploy = self.deploy_manager.get_deploy_config(name) if not deploy: @@ -1430,13 +1612,174 @@ class ObdHome(object): return False deploy_info = deploy.deploy_info - self._call_stdio('verbose', 'Check the deploy status') - if deploy_info.status == DeployStatus.STATUS_RUNNING: - # if deploy_info.config_status != DeployConfigStatus.UNCHNAGE and not getattr(opt, 'without_parameter', False): - # self.reload_cluster(name) - if not self.stop_cluster(name, options=opt): - return False - return self.start_cluster(name, options=opt) + if deploy_info.config_status == DeployConfigStatus.NEED_REDEPLOY: + self._call_stdio('error', 'Deploy needs redeploy') + return False + + self._call_stdio('start_loading', 'Get local repositories and plugins') + deploy_config = deploy.deploy_config + # Get the repository + repositories = self.load_local_repositories(deploy_info) + + restart_plugins = self.search_py_script_plugin(repositories, 'restart') + reload_plugins = self.search_py_script_plugin(repositories, 'reload') + start_plugins = self.search_py_script_plugin(repositories, 'start') + stop_plugins = self.search_py_script_plugin(repositories, 'stop') + connect_plugins = self.search_py_script_plugin(repositories, 'connect') + display_plugins = self.search_py_script_plugin(repositories, 'display') + + self._call_stdio('stop_loading', 'succeed') + + self._call_stdio('start_loading', 'Load cluster param plugin') + # Check whether the components have the parameter plugins and apply the plugins + self.search_param_plugin_and_apply(repositories, deploy_config) + if getattr(options, 'without_parameter', False) is False and deploy_info.config_status != DeployConfigStatus.UNCHNAGE: + apply_change = True + new_deploy_config = deploy.temp_deploy_config + change_user = deploy_config.user.username != new_deploy_config.user.username + self.search_param_plugin_and_apply(repositories, new_deploy_config) + else: + new_deploy_config = None + apply_change = change_user = False + + self._call_stdio('stop_loading', 'succeed') + + update_deploy_status = True + components = getattr(options, 'components', '') + if components: + components = components.split(',') + for component in components: + if component not in deploy_info.components: + self._call_stdio('error', 'No such component: %s' % component) + return False + if len(components) != len(deploy_info.components): + if apply_change: + self._call_stdio('error', 'Configurations are changed and must be applied to all components and servers.') + return False + update_deploy_status = False + else: + components = deploy_info.components.keys() + + servers = getattr(options, 'servers', '') + if servers: + server_list = servers.split(',') + if apply_change: + for repository in repositories: + cluster_config = deploy_config.components[repository.name] + for server in cluster_config.servers: + if server.name not in server_list: + self._call_stdio('error', 'Configurations are changed and must be applied to all components and servers.') + return False + else: + server_list = [] + + # Get the client + ssh_clients = self.get_clients(deploy_config, repositories) + if new_deploy_config and deploy_config.user.username != new_deploy_config.user.username: + new_ssh_clients = self.get_clients(new_deploy_config, repositories) + self._call_stdio('start_loading', 'Check sudo') + for server in new_ssh_clients: + client = new_ssh_clients[server] + ret = client.execute_command('sudo whoami') + if not ret: + self._call_stdio('error', ret.stderr) + self._call_stdio('stop_loading', 'fail') + return False + self._call_stdio('stop_loading', 'succeed') + else: + new_ssh_clients = None + + # Check the status for the deployed cluster + component_status = {} + cluster_status = self.cluster_status_check(ssh_clients, deploy_config, repositories, component_status) + if cluster_status is False or cluster_status == 0: + if self.stdio: + self._call_stdio('error', EC_SOME_SERVER_STOPED) + for repository in component_status: + cluster_status = component_status[repository] + for server in cluster_status: + if cluster_status[server] == 0: + self._call_stdio('print', '%s %s is stopped' % (server, repository.name)) + return False + + done_repositories = [] + cluster_configs = {} + component_num = len(components) + repositories = self.sort_repositories_by_depends(deploy_config, repositories) + for repository in repositories: + if repository.name not in components: + continue + cluster_config = deploy_config.components[repository.name] + new_cluster_config = new_deploy_config.components[repository.name] if new_deploy_config else None + if apply_change is False: + cluster_servers = cluster_config.servers + if servers: + cluster_config.servers = [srv for srv in cluster_servers if srv.ip in server_list or srv.name in server_list] + if not cluster_config.servers: + component_num -= 1 + continue + + start_all = cluster_servers == cluster_config.servers + update_deploy_status = update_deploy_status and start_all + + self._call_stdio('verbose', 'Call %s for %s' % (restart_plugins[repository], repository)) + if restart_plugins[repository]( + deploy_config.components.keys(), ssh_clients, cluster_config, [], options, self.stdio, + local_home_path=self.home_path, + start_plugin=start_plugins[repository], + reload_plugin=reload_plugins[repository], + stop_plugin=stop_plugins[repository], + connect_plugin=connect_plugins[repository], + display_plugin=display_plugins[repository], + repository=repository, + new_cluster_config=new_cluster_config, + new_clients=new_ssh_clients + ): + component_num -= 1 + done_repositories.append(repository) + if new_cluster_config: + cluster_configs[repository.name] = cluster_config + deploy_config.update_component(new_cluster_config) + else: + break + + if component_num == 0: + if len(components) != len(repositories) or servers: + self._call_stdio('print', "succeed") + return True + else: + if apply_change and not deploy.apply_temp_deploy_config(): + self._call_stdio('error', 'Failed to apply new deploy configuration') + return False + self._call_stdio('verbose', 'Set %s deploy status to running' % name) + if deploy.update_deploy_status(DeployStatus.STATUS_RUNNING): + self._call_stdio('print', '%s restart' % name) + return True + elif new_ssh_clients: + self._call_stdio('start_loading', 'Rollback') + component_num = len(done_repositories) + for repository in done_repositories: + new_cluster_config = new_deploy_config.components[repository.name] + cluster_config = cluster_configs[repository.name] + + self._call_stdio('verbose', 'Call %s for %s' % (restart_plugins[repository], repository)) + if restart_plugins[repository]( + deploy_config.components.keys(), ssh_clients, cluster_config, [], options, self.stdio, + local_home_path=self.home_path, + start_plugin=start_plugins[repository], + reload_plugin=reload_plugins[repository], + stop_plugin=stop_plugins[repository], + connect_plugin=connect_plugins[repository], + display_plugin=display_plugins[repository], + repository=repository, + new_cluster_config=new_cluster_config, + new_clients=new_ssh_clients, + rollback=True + ): + deploy_config.update_component(cluster_config) + + self._call_stdio('stop_loading', 'succeed') + return False def redeploy_cluster(self, name, opt=Values()): return self.destroy_cluster(name, opt) and self.deploy_cluster(name) and self.start_cluster(name) @@ -1609,7 +1952,7 @@ class ObdHome(object): route = [] use_images = [] - upgrade_route_plugins = self.search_py_script_plugin([current_repository], 'upgrade_route', no_found_exit=False) + upgrade_route_plugins = self.search_py_script_plugin([current_repository], 'upgrade_route', no_found_act='warn') if current_repository in upgrade_route_plugins: ret = upgrade_route_plugins[current_repository](deploy_config.components.keys(), ssh_clients, cluster_config, {}, options, self.stdio, current_repository, dest_repository) route = ret.get_return('route') @@ -1660,7 +2003,7 @@ class ObdHome(object): if not install_plugins: return False - upgrade_check_plugins = self.search_py_script_plugin(upgrade_repositories, 'upgrade_check', no_found_exit=False) + upgrade_check_plugins = self.search_py_script_plugin(upgrade_repositories, 'upgrade_check', no_found_act='warn') if current_repository in upgrade_check_plugins: connect_plugin = self.search_py_script_plugin(upgrade_repositories, 'connect')[current_repository] db = None @@ -1894,7 +2237,7 @@ class ObdHome(object): cluster_status = self.cluster_status_check(ssh_clients, deploy_config, repositories, component_status) if cluster_status is False or cluster_status == 0: if self.stdio: - self._call_stdio('error', 'Some of the servers in the cluster have been stopped') + self._call_stdio('error', EC_SOME_SERVER_STOPED) for repository in component_status: cluster_status = component_status[repository] for server in cluster_status: @@ -2058,7 +2401,7 @@ class ObdHome(object): cluster_status = self.cluster_status_check(ssh_clients, deploy_config, repositories, component_status) if cluster_status is False or cluster_status == 0: if self.stdio: - self._call_stdio('error', 'Some of the servers in the cluster have been stopped') + self._call_stdio('error', EC_SOME_SERVER_STOPED) for repository in component_status: cluster_status = component_status[repository] for server in cluster_status: @@ -2175,7 +2518,7 @@ class ObdHome(object): cluster_status = self.cluster_status_check(ssh_clients, deploy_config, repositories, component_status) if cluster_status is False or cluster_status == 0: if self.stdio: - self._call_stdio('error', 'Some of the servers in the cluster have been stopped') + self._call_stdio('error', EC_SOME_SERVER_STOPED) for repository in component_status: cluster_status = component_status[repository] for server in cluster_status: @@ -2224,3 +2567,240 @@ class ObdHome(object): self._call_stdio('print', 'Upgrade successful.\nCurrent version : %s' % pkg.version) return True return False + + def tpcc(self, name, opts): + self._call_stdio('verbose', 'Get Deploy by name') + deploy = self.deploy_manager.get_deploy_config(name) + if not deploy: + self._call_stdio('error', 'No such deploy: %s.' % name) + return False + + deploy_info = deploy.deploy_info + self._call_stdio('verbose', 'Check deploy status') + if deploy_info.status != DeployStatus.STATUS_RUNNING: + self._call_stdio('print', 'Deploy "%s" is %s' % (name, deploy_info.status.value)) + return False + self._call_stdio('verbose', 'Get deploy configuration') + deploy_config = deploy.deploy_config + + allow_components = ['obproxy', 'oceanbase', 'oceanbase-ce'] + if opts.component is None: + for component_name in allow_components: + if component_name in deploy_config.components: + opts.component = component_name + break + elif opts.component not in allow_components: + self._call_stdio('error', '%s not support. %s is allowed' % (opts.component, allow_components)) + return False + if opts.component not in deploy_config.components: + self._call_stdio('error', 'Can not find the component for tpcc, use `--component` to select component') + return False + + cluster_config = deploy_config.components[opts.component] + if not cluster_config.servers: + self._call_stdio('error', '%s server list is empty' % opts.component) + return False + if opts.test_server is None: + opts.test_server = cluster_config.servers[0] + else: + for server in cluster_config.servers: + if server.name == opts.test_server: + opts.test_server = server + break + else: + self._call_stdio('error', '%s is not a server in %s' % (opts.test_server, opts.component)) + return False + + self._call_stdio('start_loading', 'Get local repositories and plugins') + # Get the repository + repositories = self.get_local_repositories({opts.component: deploy_config.components[opts.component]}) + repository = repositories[0] + + # Check whether the components have the parameter plugins and apply the plugins + self.search_param_plugin_and_apply(repositories, deploy_config) + self._call_stdio('stop_loading', 'succeed') + + # Get the client + ssh_clients = self.get_clients(deploy_config, repositories) + + # Check the status for the deployed cluster + component_status = {} + cluster_status = self.cluster_status_check(ssh_clients, deploy_config, repositories, component_status) + if cluster_status is False or cluster_status == 0: + if self.stdio: + self._call_stdio('error', EC_SOME_SERVER_STOPED) + for repository in component_status: + cluster_status = component_status[repository] + for server in cluster_status: + if cluster_status[server] == 0: + self._call_stdio('print', '%s %s is stopped' % (server, repository.name)) + return False + + for repository in repositories: + if repository.name == opts.component: + break + + env = {'sys_root': False} + odp_db = None + odp_cursor = None + ob_optimization = True + ob_component = None + # ob_cluster_config = None + + connect_plugin = self.search_py_script_plugin(repositories, 'connect')[repository] + + if repository.name == 'obproxy': + ob_optimization = False + allow_components = ['oceanbase', 'oceanbase-ce'] + for component in deploy_info.components: + if component in allow_components: + ob_component = component + config = deploy_config.components[component] + env['user'] = 'root' + env['password'] = config.get_global_conf().get('root_password', '') + ob_optimization = True + break + ret = connect_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], {}, self.stdio, + target_server=opts.test_server) + if not ret or not ret.get_return('connect'): + self._call_stdio('error', 'Failed to connect to the server') + return False + odp_db = ret.get_return('connect') + odp_cursor = ret.get_return('cursor') + # ob_cluster_config = deploy_config.components[ob_component] + else: + ob_component = opts.component + # ob_cluster_config = cluster_config + + ret = connect_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], {}, self.stdio, + target_server=opts.test_server, **env) + if not ret or not ret.get_return('connect'): + self._call_stdio('error', 'Failed to connect to the server') + return False + db = ret.get_return('connect') + cursor = ret.get_return('cursor') + + pre_test_plugin = self.plugin_manager.get_best_py_script_plugin('pre_test', 'tpcc', repository.version) + optimize_plugin = self.plugin_manager.get_best_py_script_plugin('optimize', 'tpcc', repository.version) + build_plugin = self.plugin_manager.get_best_py_script_plugin('build', 'tpcc', repository.version) + run_test_plugin = self.plugin_manager.get_best_py_script_plugin('run_test', 'tpcc', repository.version) + recover_plugin = self.plugin_manager.get_best_py_script_plugin('recover', 'tpcc', repository.version) + + setattr(opts, 'host', opts.test_server.ip) + setattr(opts, 'port', db.port) + setattr(opts, 'ob_optimization', ob_optimization) + + kwargs = {} + + optimized = False + optimization = getattr(opts, 'optimization', 0) + test_only = getattr(opts, 'test_only', False) + components = [] + if getattr(self.stdio, 'sub_io'): + stdio = self.stdio.sub_io() + else: + stdio = None + obd = None + try: + self._call_stdio('verbose', 'Call %s for %s' % (pre_test_plugin, repository)) + ret = pre_test_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], opts, self.stdio, + cursor, odp_cursor, **kwargs) + if not ret: + return False + else: + kwargs.update(ret.kwargs) + if optimization: + optimized = True + kwargs['optimization_step'] = 'build' + self._call_stdio('verbose', 'Call %s for %s' % (optimize_plugin, repository)) + ret = optimize_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], opts, self.stdio, cursor, + odp_cursor, **kwargs) + if not ret: + return False + else: + kwargs.update(ret.kwargs) + if kwargs.get('odp_need_reboot'): + components.append('obproxy') + if kwargs.get('obs_need_reboot') and ob_component: + components.append(ob_component) + if components: + db.close() + cursor.close() + if odp_db: + odp_db.close() + if odp_cursor: + odp_cursor.close() + self._call_stdio('start_loading', 'Restart cluster') + obd = ObdHome(self.home_path, stdio=stdio, lock=False) + obd.lock_manager.set_try_times(-1) + option = Values({'components': ','.join(components), 'without_parameter': True}) + if obd.stop_cluster(name=name, options=option) and obd.start_cluster(name=name, options=option) and obd.display_cluster(name=name): + self._call_stdio('stop_loading', 'succeed') + else: + self._call_stdio('stop_loading', 'fail') + return False + if repository.name == 'obproxy': + ret = connect_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], {}, + self.stdio, + target_server=opts.test_server) + if not ret or not ret.get_return('connect'): + self._call_stdio('error', 'Failed to connect to the server') + return False + odp_db = ret.get_return('connect') + odp_cursor = ret.get_return('cursor') + ret = connect_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], {}, + self.stdio, + target_server=opts.test_server, **env) + if not ret or not ret.get_return('connect'): + self._call_stdio('error', 'Failed to connect to the server') + return False + db = ret.get_return('connect') + cursor = ret.get_return('cursor') + if not test_only: + self._call_stdio('verbose', 'Call %s for %s' % (build_plugin, repository)) + ret = build_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], opts, self.stdio, cursor, + odp_cursor, **kwargs) + if not ret: + return False + else: + kwargs.update(ret.kwargs) + if optimization: + kwargs['optimization_step'] = 'test' + self._call_stdio('verbose', 'Call %s for %s' % (optimize_plugin, repository)) + ret = optimize_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], opts, self.stdio, cursor, + odp_cursor, **kwargs) + if not ret: + return False + else: + kwargs.update(ret.kwargs) + self._call_stdio('verbose', 'Call %s for %s' % (run_test_plugin, repository)) + ret = run_test_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], opts, self.stdio, cursor, + odp_cursor, **kwargs) + if not ret: + return False + else: + kwargs.update(ret.kwargs) + return True + except Exception as e: + self._call_stdio('error', e) + return False + finally: + if optimization and optimized: + self._call_stdio('verbose', 'Call %s for %s' % (recover_plugin, repository)) + if not recover_plugin(deploy_config.components.keys(), ssh_clients, cluster_config, [], opts, self.stdio, + cursor, odp_cursor, **kwargs): + return False + if components and obd: + self._call_stdio('start_loading', 'Restart cluster') + option = Values({'components': ','.join(components), 'without_parameter': True}) + if obd.stop_cluster(name=name, options=option) and obd.start_cluster(name=name, options=option): + self._call_stdio('stop_loading', 'succeed') + else: + self._call_stdio('stop_loading', 'fail') + if db: + db.close() + if odp_db: + odp_db.close() + + + diff --git a/plugins/mysqltest/3.1.0/init.py b/plugins/mysqltest/3.1.0/init.py index 4b420fb8f108cf28018b263cd3945c1463ed025f..8aac225b648ed1dfe4d47e5d2a53ea6cf5e20f28 100644 --- a/plugins/mysqltest/3.1.0/init.py +++ b/plugins/mysqltest/3.1.0/init.py @@ -24,6 +24,8 @@ import re import os from ssh import LocalClient +from _errno import EC_MYSQLTEST_FAILE_NOT_FOUND, EC_MYSQLTEST_PARSE_CMD_FAILED + def parse_size(size): _bytes = 0 @@ -69,7 +71,7 @@ def init(plugin_context, env, *args, **kwargs): def exec_sql(cmd): ret = re.match('(.*\.sql)(?:\|([^\|]*))?(?:\|([^\|]*))?', cmd) if not ret: - stdio.error('parse cmd failed: %s' % cmd) + stdio.error(EC_MYSQLTEST_PARSE_CMD_FAILED.format(path=cmd)) return False cmd = ret.groups() sql_file_path1 = os.path.join(init_sql_dir, cmd[0]) @@ -79,7 +81,7 @@ def init(plugin_context, env, *args, **kwargs): elif os.path.isfile(sql_file_path2): sql_file_path = sql_file_path2 else: - stdio.error('%s not found in [%s, %s]' % (cmd[0], init_sql_dir, plugin_init_sql_dir)) + stdio.error(EC_MYSQLTEST_FAILE_NOT_FOUND.format(file=cmd[0], path='[%s, %s]' % (init_sql_dir, plugin_init_sql_dir))) return False exec_sql_cmd = exec_sql_temp % (cmd[1] if cmd[1] else 'root', cmd[2] if cmd[2] else 'oceanbase', sql_file_path) ret = LocalClient.execute_command(exec_sql_cmd, stdio=stdio) diff --git a/plugins/mysqltest/3.1.0/init_sql/init_tenant.sql b/plugins/mysqltest/3.1.0/init_sql/init_tenant.sql deleted file mode 100644 index 5935173458f98808f6a1e1e33aad841a7a87faeb..0000000000000000000000000000000000000000 --- a/plugins/mysqltest/3.1.0/init_sql/init_tenant.sql +++ /dev/null @@ -1,7 +0,0 @@ -create resource unit box1 max_cpu 2, max_memory 1073741824, max_iops 128, max_disk_size '5G', max_session_num 64, MIN_CPU=1, MIN_MEMORY=1073741824, MIN_IOPS=128; -create resource pool pool1 unit = 'box1', unit_num = 1; -create tenant ora_tt replica_num = 1, resource_pool_list=('pool1') set ob_tcp_invited_nodes='%', ob_compatibility_mode='oracle'; -alter tenant ora_tt set variables autocommit='on'; -alter tenant ora_tt set variables nls_date_format='YYYY-MM-DD HH24:MI:SS'; -alter tenant ora_tt set variables nls_timestamp_format='YYYY-MM-DD HH24:MI:SS.FF'; -alter tenant ora_tt set variables nls_timestamp_tz_format='YYYY-MM-DD HH24:MI:SS.FF TZR TZD'; \ No newline at end of file diff --git a/plugins/mysqltest/3.1.0/init_sql/init_tpch.sql b/plugins/mysqltest/3.1.0/init_sql/init_tpch.sql deleted file mode 100644 index 2d203fea20e61ebea12531ced6d6098f9b342ba1..0000000000000000000000000000000000000000 --- a/plugins/mysqltest/3.1.0/init_sql/init_tpch.sql +++ /dev/null @@ -1,46 +0,0 @@ -system sleep 5; -alter system set balancer_idle_time = '60s'; -create user 'admin' IDENTIFIED BY 'admin'; -use oceanbase; -create database if not exists test; - -use test; -grant all on *.* to 'admin' WITH GRANT OPTION; - - -set global ob_enable_jit='OFF'; -alter system set large_query_threshold='1s'; -alter system set syslog_level='info'; -alter system set syslog_io_bandwidth_limit='30M'; -alter system set trx_try_wait_lock_timeout='0'; -alter system set zone_merge_concurrency=0; -alter system set merger_completion_percentage=100; -alter system set trace_log_slow_query_watermark='500ms'; -alter system set minor_freeze_times=30; -alter system set clog_sync_time_warn_threshold = '1000ms'; -alter system set trace_log_slow_query_watermark = '10s'; -alter system set enable_sql_operator_dump = 'false'; -alter system set rpc_timeout=1000000000; - - -create resource unit tpch_box1 min_memory '100g', max_memory '100g', max_disk_size '1000g', max_session_num 64, min_cpu=9, max_cpu=9, max_iops 128, min_iops=128; -create resource pool tpch_pool1 unit = 'tpch_box1', unit_num = 1, zone_list = ('z1', 'z2', 'z3'); -create tenant oracle replica_num = 3, resource_pool_list=('tpch_pool1') set ob_tcp_invited_nodes='%', ob_compatibility_mode='oracle'; - -alter tenant oracle set variables autocommit='on'; -alter tenant oracle set variables nls_date_format='yyyy-mm-dd hh24:mi:ss'; -alter tenant oracle set variables nls_timestamp_format='yyyy-mm-dd hh24:mi:ss.ff'; -alter tenant oracle set variables nls_timestamp_tz_format='yyyy-mm-dd hh24:mi:ss.ff tzr tzd'; -alter tenant oracle set variables ob_query_timeout=7200000000; -alter tenant oracle set variables ob_trx_timeout=7200000000; -alter tenant oracle set variables max_allowed_packet=67108864; -alter tenant oracle set variables ob_enable_jit='OFF'; -alter tenant oracle set variables ob_sql_work_area_percentage=80; -alter tenant oracle set variables parallel_max_servers=512; -alter tenant oracle set variables parallel_servers_target=512; - -select count(*) from oceanbase.__all_server group by zone limit 1 into @num; -set @sql_text = concat('alter resource pool tpch_pool1', ' unit_num = ', @num); -prepare stmt from @sql_text; -execute stmt; -deallocate prepare stmt; diff --git a/plugins/obagent/0.1/destroy.py b/plugins/obagent/0.1/destroy.py index 9471bae4f03f7c909f6c5e796b44e44f0d6af889..65e745560003c63e4f34717760002d40a102a8ef 100644 --- a/plugins/obagent/0.1/destroy.py +++ b/plugins/obagent/0.1/destroy.py @@ -20,17 +20,19 @@ from __future__ import absolute_import, division, print_function +from _errno import EC_CLEAN_PATH_FAILED + global_ret = True def destroy(plugin_context, *args, **kwargs): def clean(server, path): client = clients[server] - ret = client.execute_command('rm -fr %s/*' % (path)) + ret = client.execute_command('rm -fr %s/' % (path)) if not ret: global global_ret global_ret = False - stdio.warn('fail to clean %s:%s' % (server, path)) + stdio.warn(EC_CLEAN_PATH_FAILED.format(server=server, path=path)) else: stdio.verbose('%s:%s cleaned' % (server, path)) cluster_config = plugin_context.cluster_config diff --git a/plugins/obagent/0.1/generate_config.py b/plugins/obagent/0.1/generate_config.py index e5435ca1302d385e011f4db6006ebb422b55ffae..fdab8711c7cde9cd849657367ce6e1fc77f678ab 100644 --- a/plugins/obagent/0.1/generate_config.py +++ b/plugins/obagent/0.1/generate_config.py @@ -46,7 +46,7 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): if comp in depends: have_depend = True for server in cluster_config.servers: - obs_config = cluster_config.get_depled_config(comp, server) + obs_config = cluster_config.get_depend_config(comp, server) if obs_config is not None: server_depends[server].append(comp) diff --git a/plugins/obagent/0.1/init.py b/plugins/obagent/0.1/init.py index 4409f5946f14b9cacb9449fbfc7bca7010cafbb8..c92a3243fe4bfcf43684864ee385d17a0f25aca2 100644 --- a/plugins/obagent/0.1/init.py +++ b/plugins/obagent/0.1/init.py @@ -20,6 +20,8 @@ from __future__ import absolute_import, division, print_function +from _errno import EC_FAIL_TO_INIT_PATH, InitDirFailedErrorMessage + def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): cluster_config = plugin_context.cluster_config @@ -32,25 +34,25 @@ def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): server_config = cluster_config.get_server_conf(server) client = clients[server] home_path = server_config['home_path'] - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_dir = repository_dir.replace(local_home_path, remote_home_path) stdio.verbose('%s init cluster work home', server) if force: - ret = client.execute_command('rm -fr %s/*' % home_path) + ret = client.execute_command('rm -fr %s' % home_path) if not ret: global_ret = False - stdio.error('failed to initialize %s home path: %s' % (server, ret.stderr)) + stdio.error(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=ret.stderr)) continue else: if client.execute_command('mkdir -p %s' % home_path): ret = client.execute_command('ls %s' % (home_path)) if not ret or ret.stdout.strip(): global_ret = False - stdio.error('fail to init %s home path: %s is not empty' % (server, home_path)) + stdio.error(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=InitDirFailedErrorMessage.NOT_EMPTY.format(path=home_path))) continue else: global_ret = False - stdio.error('fail to init %s home path: create %s failed' % (server, home_path)) + stdio.error(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=InitDirFailedErrorMessage.CREATE_FAILED.format(path=home_path))) continue if not (client.execute_command("bash -c 'mkdir -p %s/{run,bin,lib,conf,log}'" % (home_path)) \ @@ -58,7 +60,7 @@ def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): and client.execute_command("if [ -d %s/bin ]; then ln -fs %s/bin/* %s/bin; fi" % (remote_repository_dir, remote_repository_dir, home_path)) \ and client.execute_command("if [ -d %s/lib ]; then ln -fs %s/lib/* %s/lib; fi" % (remote_repository_dir, remote_repository_dir, home_path))): global_ret = False - stdio.error('fail to init %s home path', server) + stdio.error(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=InitDirFailedErrorMessage.PATH_ONLY.format(path=home_path))) if global_ret: stdio.stop_loading('succeed') diff --git a/plugins/obagent/0.1/reload.py b/plugins/obagent/0.1/reload.py index 599b1b4c6050f81ae599b8bfc2c5c59b844eb4c1..4d6605191cd9f92ee794278198e7cb6c7db49e0c 100644 --- a/plugins/obagent/0.1/reload.py +++ b/plugins/obagent/0.1/reload.py @@ -26,6 +26,8 @@ from copy import deepcopy from glob import glob from tool import YamlLoader +from _errno import * + def reload(plugin_context, repository_dir, new_cluster_config, *args, **kwargs): stdio = plugin_context.stdio @@ -46,8 +48,8 @@ def reload(plugin_context, repository_dir, new_cluster_config, *args, **kwargs): for comp in ['oceanbase', 'oceanbase-ce']: if comp in cluster_config.depends: root_servers = {} - ob_config = cluster_config.get_depled_config(comp) - new_ob_config = new_cluster_config.get_depled_config(comp) + ob_config = cluster_config.get_depend_config(comp) + new_ob_config = new_cluster_config.get_depend_config(comp) ob_config = {} if ob_config is None else ob_config new_ob_config = {} if new_ob_config is None else new_ob_config for key in config_map: @@ -66,6 +68,7 @@ def reload(plugin_context, repository_dir, new_cluster_config, *args, **kwargs): config_kv[key] = key global_ret = True + stdio.start_load('Reload obagent') for server in servers: change_conf = deepcopy(global_change_conf) client = clients[server] @@ -79,6 +82,18 @@ def reload(plugin_context, repository_dir, new_cluster_config, *args, **kwargs): if key not in config_kv: continue if key not in config or config[key] != new_config[key]: + item = cluster_config.get_temp_conf_item(key) + if item: + if item.need_redeploy or item.need_restart: + stdio.verbose('%s can not be reload' % key) + global_ret = False + continue + try: + item.modify_limit(config.get(key), new_config.get(key)) + except Exception as e: + stdio.verbose('%s: %s' % (server, str(e))) + global_ret = False + continue change_conf[config_kv[key]] = new_config[key] if change_conf: @@ -93,6 +108,11 @@ def reload(plugin_context, repository_dir, new_cluster_config, *args, **kwargs): ) if not client.execute_command(cmd): global_ret = False - stdio.error('fail to reload %s' % server) + stdio.error(EC_OBAGENT_RELOAD_FAILED.format(server=server)) - return plugin_context.return_true() if global_ret else None + if global_ret: + stdio.stop_load('succeed') + return plugin_context.return_true() + else: + stdio.stop_load('fail') + return diff --git a/plugins/obagent/0.1/restart.py b/plugins/obagent/0.1/restart.py new file mode 100644 index 0000000000000000000000000000000000000000..c2ef6258f9395ccba9a92cf5bb85ca25cf7009a5 --- /dev/null +++ b/plugins/obagent/0.1/restart.py @@ -0,0 +1,94 @@ + +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +import os + + +class Restart(object): + + def __init__(self, plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None): + self.local_home_path = local_home_path + self.plugin_context = plugin_context + self.components = plugin_context.components + self.clients = plugin_context.clients + self.cluster_config = plugin_context.cluster_config + self.stdio = plugin_context.stdio + self.repository = repository + self.start_plugin = start_plugin + self.reload_plugin = reload_plugin + self.connect_plugin = connect_plugin + self.display_plugin = display_plugin + self.stop_plugin = stop_plugin + self.new_clients = new_clients + self.new_cluster_config = new_cluster_config + self.sub_io = self.stdio.sub_io() + + def dir_read_check(self, client, path): + if not client.execute_command('cd %s' % path): + dirpath, name = os.path.split(path) + return self.dir_read_check(client, dirpath) and client.execute_command('sudo chmod +1 %s' % path) + return True + + def restart(self): + clients = self.clients + self.stdio.verbose('Call %s for %s' % (self.stop_plugin, self.repository)) + if not self.stop_plugin(self.components, clients, self.cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io): + self.stdio.stop_loading('stop_loading', 'fail') + return False + + if self.new_clients: + self.stdio.verbose('use new clients') + for server in self.cluster_config.servers: + new_client = self.new_clients[server] + server_config = self.cluster_config.get_server_conf(server) + home_path = server_config['home_path'] + if not new_client.execute_command('sudo chown -R %s: %s' % (new_client.config.username, home_path)): + self.stdio.stop_loading('stop_loading', 'fail') + return False + self.dir_read_check(new_client, home_path) + clients = self.new_clients + + cluster_config = self.new_cluster_config if self.new_cluster_config else self.cluster_config + self.stdio.verbose('Call %s for %s' % (self.start_plugin, self.repository)) + if not self.start_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, local_home_path=self.local_home_path, repository_dir=self.repository.repository_dir): + self.rollback() + self.stdio.stop_loading('stop_loading', 'fail') + return False + return self.display_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, cursor=None) + + def rollback(self): + if self.new_clients: + self.stop_plugin(self.components, self.new_clients, self.new_cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io) + for server in self.cluster_config.servers: + client = self.clients[server] + new_client = self.new_clients[server] + server_config = self.cluster_config.get_server_conf(server) + home_path = server_config['home_path'] + new_client.execute_command('sudo chown -R %s: %s' % (client.config.username, home_path)) + + +def restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None, rollback=False, *args, **kwargs): + task = Restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config, new_clients) + call = task.rollback if rollback else task.restart + if call(): + plugin_context.return_true() diff --git a/plugins/obagent/0.1/start.py b/plugins/obagent/0.1/start.py index 2dba8fb70e6f7dd97475f12162af68732cf8ee93..d17018c49901e74f642961310bf5400cc66881ee 100644 --- a/plugins/obagent/0.1/start.py +++ b/plugins/obagent/0.1/start.py @@ -34,6 +34,7 @@ from Crypto import Random from Crypto.Cipher import AES from tool import YamlLoader +from _errno import * stdio = None @@ -155,7 +156,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): "zone_name": "zone", } - stdio.start_loading('Start obproxy') + stdio.start_loading('Start obagent') for server in cluster_config.servers: client = clients[server] server_config = cluster_config.get_server_conf(server) @@ -203,7 +204,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): if use_parameter: for comp in ['oceanbase', 'oceanbase-ce']: - obs_config = cluster_config.get_depled_config(comp, server) + obs_config = cluster_config.get_depend_config(comp, server) if obs_config is not None: break @@ -244,7 +245,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): tf.write(text) tf.flush() if not client.put_file(tf.name, path.replace(repository_dir, home_path)): - stdio.error('Fail to send config file to %s' % server) + stdio.error(EC_OBAGENT_SEND_CONFIG_FAILED.format(server=server)) stdio.stop_loading('fail') return @@ -256,7 +257,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): else: ret = client.put_file(path, path.replace(repository_dir, home_path)) if not ret: - stdio.error('Fail to send config file to %s' % server) + stdio.error(EC_OBAGENT_SEND_CONFIG_FAILED.format(server=server)) stdio.stop_loading('fail') return @@ -284,7 +285,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): with tempfile.NamedTemporaryFile(suffix=".yaml") as tf: yaml.dump(config, tf) if not client.put_file(tf.name, os.path.join(home_path, 'conf/monagent.yaml')): - stdio.error('Fail to send config file to %s' % server) + stdio.error(EC_OBAGENT_SEND_CONFIG_FAILED.format(server=server)) stdio.stop_loading('fail') return diff --git a/plugins/obagent/0.1/start_check.py b/plugins/obagent/0.1/start_check.py index 6c963704c8a91d1fa47866a211ab95e5b957672e..d4d755cb81a6ad1a27cf81364104bad1cb1f6d23 100644 --- a/plugins/obagent/0.1/start_check.py +++ b/plugins/obagent/0.1/start_check.py @@ -20,6 +20,8 @@ from __future__ import absolute_import, division, print_function +from _errno import EC_CONFIG_CONFLICT_PORT + stdio = None success = True @@ -74,7 +76,7 @@ def start_check(plugin_context, strict_check=False, *args, **kwargs): port = int(server_config[key]) alert_f = alert if key == 'pprof_port' else critical if port in ports: - alert_f('Configuration conflict %s: %s port is used for %s\'s %s' % (server, port, ports[port]['server'], ports[port]['key'])) + alert_f(EC_CONFIG_CONFLICT_PORT.format(server1=server, port=port, server2=ports[port]['server'], key=ports[port]['key'])) continue ports[port] = { 'server': server, diff --git a/plugins/obagent/0.1/upgrade.py b/plugins/obagent/0.1/upgrade.py index fe81ef3255b3fcce7964c20484c000e509d93583..cc2dead231b3f986a2e8d5456beb4c8c46ff03c1 100644 --- a/plugins/obagent/0.1/upgrade.py +++ b/plugins/obagent/0.1/upgrade.py @@ -42,7 +42,7 @@ def upgrade(plugin_context, search_py_script_plugin, apply_param_plugin, *args, client = clients[server] server_config = cluster_config.get_server_conf(server) home_path = server_config['home_path'] - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_dir = repository_dir.replace(local_home_path, remote_home_path) client.execute_command("bash -c 'mkdir -p %s/{bin,lib}'" % (home_path)) client.execute_command("ln -fs %s/bin/* %s/bin" % (remote_repository_dir, home_path)) diff --git a/plugins/obagent/1.1.0/start.py b/plugins/obagent/1.1.0/start.py index 2820ea298b8bc5ec3fe7a75dc3526332272d2d68..8be952b357c4154472398d2c6c970eb96797268b 100644 --- a/plugins/obagent/1.1.0/start.py +++ b/plugins/obagent/1.1.0/start.py @@ -34,6 +34,7 @@ from Crypto import Random from Crypto.Cipher import AES from tool import YamlLoader +from _errno import * stdio = None @@ -156,7 +157,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): "ob_install_path": "home_path" } - stdio.start_loading('Start obproxy') + stdio.start_loading('Start obagent') for server in cluster_config.servers: client = clients[server] server_config = cluster_config.get_server_conf(server) @@ -198,7 +199,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): continue for comp in ['oceanbase', 'oceanbase-ce']: - obs_config = cluster_config.get_depled_config(comp, server) + obs_config = cluster_config.get_depend_config(comp, server) if obs_config is not None: break @@ -239,7 +240,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): tf.write(text) tf.flush() if not client.put_file(tf.name, path.replace(repository_dir, home_path)): - stdio.error('Fail to send config file to %s' % server) + stdio.error(EC_OBAGENT_SEND_CONFIG_FAILED.format(server=server)) stdio.stop_loading('fail') return @@ -251,7 +252,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): else: ret = client.put_file(path, path.replace(repository_dir, home_path)) if not ret: - stdio.error('Fail to send config file to %s' % server) + stdio.error(EC_OBAGENT_SEND_CONFIG_FAILED.format(server=server)) stdio.stop_loading('fail') return @@ -279,7 +280,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): with tempfile.NamedTemporaryFile(suffix=".yaml") as tf: yaml.dump(config, tf) if not client.put_file(tf.name, os.path.join(home_path, 'conf/monagent.yaml')): - stdio.error('Fail to send config file to %s' % server) + stdio.error(EC_OBAGENT_SEND_CONFIG_FAILED.format(server=server)) stdio.stop_loading('fail') return diff --git a/plugins/obagent/1.1.1/start_check.py b/plugins/obagent/1.1.1/start_check.py new file mode 100644 index 0000000000000000000000000000000000000000..3d83b34cc56ff4bfc5179d4e706dc0c68c9decba --- /dev/null +++ b/plugins/obagent/1.1.1/start_check.py @@ -0,0 +1,92 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +from _errno import EC_CONFIG_CONFLICT_PORT + + +stdio = None +success = True + + +def get_port_socket_inode(client, port): + port = hex(port)[2:].zfill(4).upper() + cmd = "bash -c 'cat /proc/net/{tcp,udp}' | awk -F' ' '{print $2,$10}' | grep '00000000:%s' | awk -F' ' '{print $2}' | uniq" % port + res = client.execute_command(cmd) + if not res or not res.stdout.strip(): + return False + stdio.verbose(res.stdout) + return res.stdout.strip().split('\n') + + +def start_check(plugin_context, strict_check=False, *args, **kwargs): + # def alert(*arg, **kwargs): + # global success + # if strict_check: + # success = False + # stdio.error(*arg, **kwargs) + # else: + # stdio.warn(*arg, **kwargs) + def critical(*arg, **kwargs): + global success + success = False + stdio.error(*arg, **kwargs) + global stdio + cluster_config = plugin_context.cluster_config + clients = plugin_context.clients + stdio = plugin_context.stdio + servers_port = {} + stdio.start_loading('Check before start obagent') + for server in cluster_config.servers: + ip = server.ip + client = clients[server] + server_config = cluster_config.get_server_conf(server) + port = int(server_config["server_port"]) + prometheus_port = int(server_config["pprof_port"]) + remote_pid_path = "%s/run/obagent-%s-%s.pid" % (server_config['home_path'], server.ip, server_config["server_port"]) + remote_pid = client.execute_command("cat %s" % remote_pid_path).stdout.strip() + if remote_pid: + if client.execute_command('ls /proc/%s' % remote_pid): + continue + + if ip not in servers_port: + servers_port[ip] = {} + ports = servers_port[ip] + server_config = cluster_config.get_server_conf_with_default(server) + stdio.verbose('%s port check' % server) + for key in ['server_port', 'pprof_port']: + port = int(server_config[key]) + # alert_f = alert if key == 'pprof_port' else critical + if port in ports: + critical(EC_CONFIG_CONFLICT_PORT.format(server1=server, port=port, server2=ports[port]['server'], key=ports[port]['key'])) + continue + ports[port] = { + 'server': server, + 'key': key + } + if get_port_socket_inode(client, port): + critical('%s:%s port is already used' % (ip, port)) + + if success: + stdio.stop_loading('succeed') + plugin_context.return_true() + else: + stdio.stop_loading('fail') \ No newline at end of file diff --git a/plugins/obproxy/3.1.0/connect.py b/plugins/obproxy/3.1.0/connect.py index d20e31047deaf5e5abeb502e6933523bfe594ae9..270aba5a1cec8dd048d2c9bd319fd84d22a50cc3 100644 --- a/plugins/obproxy/3.1.0/connect.py +++ b/plugins/obproxy/3.1.0/connect.py @@ -77,14 +77,14 @@ def connect(plugin_context, target_server=None, sys_root=True, *args, **kwargs): for comp in ['oceanbase', 'oceanbase-ce']: if comp in cluster_config.depends: - ob_config = cluster_config.get_depled_config(comp) + ob_config = cluster_config.get_depend_config(comp) if not ob_config: continue odp_config = cluster_config.get_global_conf() config_map = { 'observer_sys_password': 'proxyro_password', 'cluster_name': 'appname', - 'root_password': 'observer_root_password' + 'observer_root_password': 'root_password' } for key in config_map: ob_key = config_map[key] @@ -101,14 +101,12 @@ def connect(plugin_context, target_server=None, sys_root=True, *args, **kwargs): server_config = cluster_config.get_server_conf(server) if sys_root: pwd_key = 'obproxy_sys_password' - default_pwd = 'proxysys' else: pwd_key = 'observer_root_password' - default_pwd = '' r_password = password if password else server_config.get(pwd_key) if r_password is None: r_password = '' - db, cursor = _connect(server.ip, server_config['listen_port'], user, r_password if count % 2 else default_pwd) + db, cursor = _connect(server.ip, server_config['listen_port'], user, r_password if count % 2 else '') dbs[server] = db cursors[server] = cursor except: diff --git a/plugins/obproxy/3.1.0/destroy.py b/plugins/obproxy/3.1.0/destroy.py index 7402cbd4db61f120aa3e03e04e96ed102ab3226b..92532b5c2353b8981ee8413da48dcf9b6e8a36fc 100644 --- a/plugins/obproxy/3.1.0/destroy.py +++ b/plugins/obproxy/3.1.0/destroy.py @@ -20,18 +20,20 @@ from __future__ import absolute_import, division, print_function +from _errno import EC_CLEAN_PATH_FAILED + global_ret = True def destroy(plugin_context, *args, **kwargs): def clean(server, path): client = clients[server] - ret = client.execute_command('rm -fr %s/* %s/.conf' % (path, path)) + ret = client.execute_command('rm -fr %s/' % (path)) if not ret: # pring stderror global global_ret global_ret = False - stdio.warn('fail to clean %s:%s' % (server, path)) + stdio.warn(EC_CLEAN_PATH_FAILED.format(server=server, path=path)) else: stdio.verbose('%s:%s cleaned' % (server, path)) cluster_config = plugin_context.cluster_config diff --git a/plugins/obproxy/3.1.0/init.py b/plugins/obproxy/3.1.0/init.py index ba02b40bdd69b2fb8b186e5327faa40809775613..22192c6d3a1afcf07388583f26b0cce9224d9186 100644 --- a/plugins/obproxy/3.1.0/init.py +++ b/plugins/obproxy/3.1.0/init.py @@ -19,6 +19,8 @@ from __future__ import absolute_import, division, print_function +from _errno import EC_FAIL_TO_INIT_PATH, InitDirFailedErrorMessage + def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): cluster_config = plugin_context.cluster_config @@ -31,20 +33,20 @@ def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): server_config = cluster_config.get_server_conf(server) client = clients[server] home_path = server_config['home_path'] - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_dir = repository_dir.replace(local_home_path, remote_home_path) stdio.verbose('%s init cluster work home', server) if force: - ret = client.execute_command('rm -fr %s/*' % home_path) + ret = client.execute_command('rm -fr %s' % home_path) if not ret: global_ret = False - stdio.error('failed to initialize %s home path: %s' % (server, ret.stderr)) + stdio.error(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=ret.stderr)) continue if not (client.execute_command("bash -c 'mkdir -p %s/{run,bin,lib}'" % (home_path)) \ and client.execute_command("if [ -d %s/bin ]; then ln -fs %s/bin/* %s/bin; fi" % (remote_repository_dir, remote_repository_dir, home_path)) \ and client.execute_command("if [ -d %s/lib ]; then ln -fs %s/lib/* %s/lib; fi" % (remote_repository_dir, remote_repository_dir, home_path))): global_ret = False - stdio.error('fail to init %s home path', server) + stdio.error(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=InitDirFailedErrorMessage.NOT_EMPTY.format(path=home_path))) if global_ret: stdio.stop_loading('succeed') diff --git a/plugins/obproxy/3.1.0/obproxyd.sh b/plugins/obproxy/3.1.0/obproxyd.sh index d5dd90e8d50d5a6b94a2077e055bf494765550d3..bd9b1262f35867b0d4241b2a3968c64bc46d8184 100644 --- a/plugins/obproxy/3.1.0/obproxyd.sh +++ b/plugins/obproxy/3.1.0/obproxyd.sh @@ -19,6 +19,7 @@ function start() { if [ $? != 0 ]; then exit $? fi + kill -9 $pid while [ 1 ]; do diff --git a/plugins/obproxy/3.1.0/ocp_check.py b/plugins/obproxy/3.1.0/ocp_check.py new file mode 100644 index 0000000000000000000000000000000000000000..055be9969ee7a59467040f096f6a48ecac936af8 --- /dev/null +++ b/plugins/obproxy/3.1.0/ocp_check.py @@ -0,0 +1,60 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +from _rpm import Version + + +def ocp_check(plugin_context, ocp_version, cursor, new_cluster_config=None, new_clients=None, *args, **kwargs): + cluster_config = new_cluster_config if new_cluster_config else plugin_context.cluster_config + clients = new_clients if new_clients else plugin_context.clients + stdio = plugin_context.stdio + + is_admin = True + can_sudo = True + only_one = True + + min_version = Version('3.1.1') + max_version = min_version + ocp_version = Version(ocp_version) + + if ocp_version < min_version: + stdio.error('The current plugin version does not support OCP V%s' % ocp_version) + return + + if ocp_version > max_version: + stdio.warn('The plugin library does not support OCP V%s. The takeover requirements are not applicable to the current check.' % ocp_version) + + for server in cluster_config.servers: + client = clients[server] + if is_admin and client.config.username != 'admin': + is_admin = False + stdio.error('The current user must be the admin user. Run the edit-config command to modify the user.username field') + if can_sudo and not client.execute_command('sudo whoami'): + can_sudo = False + stdio.error('The user must have the privilege to run sudo commands without a password.') + if not client.execute_command('bash -c "if [ `pgrep obproxy | wc -l` -gt 1 ]; then exit 1; else exit 0;fi;"'): + only_one = False + stdio.error('%s Multiple OBProxies exist.' % server) + + if is_admin and can_sudo and only_one: + stdio.print('Configurations of the OBProxy can be taken over by OCP after they take effect.' if new_cluster_config else 'Configurations of the OBProxy can be taken over by OCP.') + return plugin_context.return_true() \ No newline at end of file diff --git a/plugins/obproxy/3.1.0/reload.py b/plugins/obproxy/3.1.0/reload.py index cfb593e929a11aad806c9ad4eee1df79dbdfa923..291a55aad1001c7451283c6d4dfb2d6634528803 100644 --- a/plugins/obproxy/3.1.0/reload.py +++ b/plugins/obproxy/3.1.0/reload.py @@ -37,8 +37,8 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): for comp in ['oceanbase', 'oceanbase-ce']: if comp in cluster_config.depends: root_servers = {} - ob_config = cluster_config.get_depled_config(comp) - new_ob_config = new_cluster_config.get_depled_config(comp) + ob_config = cluster_config.get_depend_config(comp) + new_ob_config = new_cluster_config.get_depend_config(comp) ob_config = {} if ob_config is None else ob_config new_ob_config = {} if new_ob_config is None else new_ob_config for key in config_map: @@ -56,6 +56,18 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): stdio.verbose('compare configuration of %s' % (server)) for key in new_config: if key not in config or config[key] != new_config[key]: + item = cluster_config.get_temp_conf_item(key) + if item: + if item.need_redeploy or item.need_restart: + stdio.verbose('%s can not be reload' % key) + global_ret = False + continue + try: + item.modify_limit(config.get(key), new_config.get(key)) + except Exception as e: + stdio.verbose('%s: %s' % (server, str(e))) + global_ret = False + continue change_conf[server][key] = new_config[key] if key not in global_change_conf: global_change_conf[key] = 1 @@ -64,6 +76,7 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): servers_num = len(servers) stdio.verbose('apply new configuration') + stdio.start_load('Reload obproxy') success_conf = {} sql = '' value = None @@ -87,4 +100,10 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): for server in success_conf[key]: value = change_conf[server][key] cluster_config.update_server_conf(server,key, value, False) - return plugin_context.return_true() if global_ret else None + + if global_ret: + stdio.stop_load('succeed') + return plugin_context.return_true() + else: + stdio.stop_load('fail') + return diff --git a/plugins/obproxy/3.1.0/restart.py b/plugins/obproxy/3.1.0/restart.py new file mode 100644 index 0000000000000000000000000000000000000000..3cc35628f66e04eb0fe96011c65a130e174fff79 --- /dev/null +++ b/plugins/obproxy/3.1.0/restart.py @@ -0,0 +1,128 @@ + +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +import os + + +class Restart(object): + + def __init__(self, plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None): + self.local_home_path = local_home_path + self.plugin_context = plugin_context + self.components = plugin_context.components + self.clients = plugin_context.clients + self.cluster_config = plugin_context.cluster_config + self.stdio = plugin_context.stdio + self.repository = repository + self.start_plugin = start_plugin + self.reload_plugin = reload_plugin + self.connect_plugin = connect_plugin + self.display_plugin = display_plugin + self.stop_plugin = stop_plugin + self.new_clients = new_clients + self.new_cluster_config = new_cluster_config + self.sub_io = self.stdio.sub_io() + self.dbs = None + self.cursors = None + + # def close(self): + # if self.dbs: + # for server in self.cursors: + # self.cursors[server].close() + # for db in self.dbs: + # self.dbs[server].close() + # self.cursors = None + # self.dbs = None + + def connect(self): + if self.cursors is None: + self.stdio.verbose('Call %s for %s' % (self.connect_plugin, self.repository)) + self.sub_io.start_loading('Connect to obproxy') + ret = self.connect_plugin(self.components, self.clients, self.cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io) + if not ret: + self.sub_io.stop_loading('fail') + return False + self.sub_io.stop_loading('succeed') + # self.close() + self.cursors = ret.get_return('cursor') + self.dbs = ret.get_return('connect') + return True + + def dir_read_check(self, client, path): + if not client.execute_command('cd %s' % path): + dirpath, name = os.path.split(path) + return self.dir_read_check(client, dirpath) and client.execute_command('sudo chmod +1 %s' % path) + return True + + def restart(self): + clients = self.clients + self.stdio.verbose('Call %s for %s' % (self.stop_plugin, self.repository)) + if not self.stop_plugin(self.components, clients, self.cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io): + self.stdio.stop_loading('stop_loading', 'fail') + return False + + if self.new_clients: + self.stdio.verbose('use new clients') + for server in self.cluster_config.servers: + client = clients[server] + new_client = self.new_clients[server] + server_config = self.cluster_config.get_server_conf(server) + home_path = server_config['home_path'] + if not new_client.execute_command('sudo chown -R %s: %s' % (new_client.config.username, home_path)): + self.stdio.stop_loading('stop_loading', 'fail') + return False + self.dir_read_check(new_client, home_path) + clients = self.new_clients + + cluster_config = self.new_cluster_config if self.new_cluster_config else self.cluster_config + self.stdio.verbose('Call %s for %s' % (self.start_plugin, self.repository)) + if not self.start_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, local_home_path=self.local_home_path, repository_dir=self.repository.repository_dir): + self.rollback() + self.stdio.stop_loading('stop_loading', 'fail') + return False + + if self.connect(): + ret = self.display_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, cursor=self.cursors) + if self.new_cluster_config: + self.stdio.verbose('Call %s for %s' % (self.reload_plugin, self.repository)) + self.reload_plugin(self.components, self.clients, self.cluster_config, [], {}, self.sub_io, + cursor=self.cursors, new_cluster_config=self.new_cluster_config, repository_dir=self.repository.repository_dir) + return ret + return False + + def rollback(self): + if self.new_clients: + self.stop_plugin(self.components, self.new_clients, self.new_cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io) + for server in self.cluster_config.servers: + client = self.clients[server] + new_client = self.new_clients[server] + server_config = self.cluster_config.get_server_conf(server) + home_path = server_config['home_path'] + new_client.execute_command('sudo chown -R %s: %s' % (client.config.username, home_path)) + + +def restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None, rollback=False, *args, **kwargs): + task = Restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config, new_clients) + call = task.rollback if rollback else task.restart + if call(): + plugin_context.return_true() diff --git a/plugins/obproxy/3.1.0/start.py b/plugins/obproxy/3.1.0/start.py index bf8ce010298d333f5bd1d2771ed540330f9537e9..c7b5ccce485007ae3d4d1e5c5f54761ddd2ae0dd 100644 --- a/plugins/obproxy/3.1.0/start.py +++ b/plugins/obproxy/3.1.0/start.py @@ -100,12 +100,12 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): for comp in ['oceanbase', 'oceanbase-ce']: if comp in cluster_config.depends: root_servers = {} - ob_config = cluster_config.get_depled_config(comp) + ob_config = cluster_config.get_depend_config(comp) if not ob_config: continue odp_config = cluster_config.get_global_conf() - for server in cluster_config.get_depled_servers(comp): - config = cluster_config.get_depled_config(comp, server) + for server in cluster_config.get_depend_servers(comp): + config = cluster_config.get_depend_config(comp, server) zone = config['zone'] if zone not in root_servers: root_servers[zone] = '%s:%s' % (server.ip, config['mysql_port']) @@ -124,7 +124,6 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): error = False for server in cluster_config.servers: - client = clients[server] server_config = cluster_config.get_server_conf(server) if 'rs_list' not in server_config and 'obproxy_config_server_url' not in server_config: error = True @@ -142,13 +141,20 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): need_bootstrap = True break + if getattr(options, 'without_parameter', False) and need_bootstrap is False: + use_parameter = False + else: + # Bootstrap is required when starting with parameter, ensure the passwords are correct. + need_bootstrap = True + use_parameter = True + for server in cluster_config.servers: client = clients[server] server_config = cluster_config.get_server_conf(server) home_path = server_config['home_path'] if client.execute_command("bash -c 'if [ -f %s/bin/obproxy ]; then exit 1; else exit 0; fi;'" % home_path): - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_dir = repository_dir.replace(local_home_path, remote_home_path) client.execute_command("bash -c 'mkdir -p %s/{bin,lib}'" % (home_path)) client.execute_command("ln -fs %s/bin/* %s/bin" % (remote_repository_dir, home_path)) @@ -156,11 +162,6 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): pid_path[server] = "%s/run/obproxy-%s-%s.pid" % (home_path, server.ip, server_config["listen_port"]) - if getattr(options, 'without_parameter', False) and need_bootstrap is False: - use_parameter = False - else: - use_parameter = True - if use_parameter: not_opt_str = [ 'listen_port', @@ -170,7 +171,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): ] start_unuse = ['home_path', 'observer_sys_password', 'obproxy_sys_password', 'observer_root_password'] get_value = lambda key: "'%s'" % server_config[key] if isinstance(server_config[key], str) else server_config[key] - opt_str = ["obproxy_sys_password=e3fd448c516073714189b57233c9cf428ccb1bed"] + opt_str = ["obproxy_sys_password=''"] for key in server_config: if key not in start_unuse and key not in not_opt_str: value = get_value(key) @@ -217,7 +218,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): stdio.start_loading('obproxy program health check') failed = [] servers = cluster_config.servers - count = 8 + count = 20 while servers and count: count -= 1 tmp_servers = [] @@ -236,7 +237,6 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): else: client.execute_command('echo %s > %s' % (pid, pid_path[server])) obproxyd(server_config["home_path"], client, server.ip, server_config["listen_port"]) - client.execute_command('cat %s | xargs kill -9' % pid_path[server]) tmp_servers.append(server) break stdio.verbose('failed to start %s obproxy, remaining retries: %d' % (server, count)) diff --git a/plugins/obproxy/3.1.0/start_check.py b/plugins/obproxy/3.1.0/start_check.py index 4efdedd46a8a9fa4f38cbacb37502fcd20c01019..18a052f706b13da38e91c3efbbba9a907076d480 100644 --- a/plugins/obproxy/3.1.0/start_check.py +++ b/plugins/obproxy/3.1.0/start_check.py @@ -20,6 +20,8 @@ from __future__ import absolute_import, division, print_function +from _errno import EC_CONFIG_CONFLICT_PORT + stdio = None success = True @@ -74,7 +76,7 @@ def start_check(plugin_context, strict_check=False, *args, **kwargs): port = int(server_config[key]) alert_f = alert if key == 'prometheus_listen_port' else critical if port in ports: - alert_f('Configuration conflict %s: %s port is used for %s\'s %s' % (server, port, ports[port]['server'], ports[port]['key'])) + alert_f(EC_CONFIG_CONFLICT_PORT.format(server1=server, port=port, server2=ports[port]['server'], key=ports[port]['key'])) continue ports[port] = { 'server': server, diff --git a/plugins/obproxy/3.1.0/upgrade.py b/plugins/obproxy/3.1.0/upgrade.py index fe81ef3255b3fcce7964c20484c000e509d93583..cc2dead231b3f986a2e8d5456beb4c8c46ff03c1 100644 --- a/plugins/obproxy/3.1.0/upgrade.py +++ b/plugins/obproxy/3.1.0/upgrade.py @@ -42,7 +42,7 @@ def upgrade(plugin_context, search_py_script_plugin, apply_param_plugin, *args, client = clients[server] server_config = cluster_config.get_server_conf(server) home_path = server_config['home_path'] - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_dir = repository_dir.replace(local_home_path, remote_home_path) client.execute_command("bash -c 'mkdir -p %s/{bin,lib}'" % (home_path)) client.execute_command("ln -fs %s/bin/* %s/bin" % (remote_repository_dir, home_path)) diff --git a/plugins/oceanbase/3.1.0/bootstrap.py b/plugins/oceanbase/3.1.0/bootstrap.py index 41b5499b239e573302e25a65d3d155dc3ba2f256..4a6b4501735f0a22d26ede2ddcf6bfcb8b9ce917 100644 --- a/plugins/oceanbase/3.1.0/bootstrap.py +++ b/plugins/oceanbase/3.1.0/bootstrap.py @@ -21,6 +21,7 @@ from __future__ import absolute_import, division, print_function import time +from _deploy import InnerConfigItem def bootstrap(plugin_context, cursor, *args, **kwargs): @@ -28,6 +29,11 @@ def bootstrap(plugin_context, cursor, *args, **kwargs): stdio = plugin_context.stdio bootstrap = [] floor_servers = {} + zones_config = {} + inner_config = { + InnerConfigItem('$_zone_idc'): 'idc' + } + inner_keys = inner_config.keys() for server in cluster_config.servers: server_config = cluster_config.get_server_conf(server) zone = server_config['zone'] @@ -35,7 +41,18 @@ def bootstrap(plugin_context, cursor, *args, **kwargs): floor_servers[zone].append('%s:%s' % (server.ip, server_config['rpc_port'])) else: floor_servers[zone] = [] + zones_config[zone] = {} bootstrap.append('REGION "sys_region" ZONE "%s" SERVER "%s:%s"' % (server_config['zone'], server.ip, server_config['rpc_port'])) + + zone_config = zones_config[zone] + for key in server_config: + if not isinstance(key, InnerConfigItem): + continue + if key not in inner_config: + continue + if key in zone_config: + continue + zone_config[key] = server_config[key] try: sql = 'set session ob_query_timeout=1000000000' stdio.verbose('execute sql: %s' % sql) @@ -52,16 +69,22 @@ def bootstrap(plugin_context, cursor, *args, **kwargs): global_conf = cluster_config.get_global_conf() if 'proxyro_password' in global_conf or 'obproxy' in plugin_context.components: value = global_conf['proxyro_password'] if global_conf.get('proxyro_password') is not None else '' - sql = 'create user "proxyro" IDENTIFIED BY "%s"' % value + sql = 'create user "proxyro" IDENTIFIED BY %s' stdio.verbose(sql) - cursor.execute(sql) - sql = 'grant select on oceanbase.* to proxyro IDENTIFIED BY "%s"' % value + cursor.execute(sql, [value]) + sql = 'grant select on oceanbase.* to proxyro IDENTIFIED BY %s' stdio.verbose(sql) - cursor.execute(sql) + cursor.execute(sql, [value]) if global_conf.get('root_password') is not None: sql = 'alter user "root" IDENTIFIED BY "%s"' % global_conf.get('root_password') stdio.verbose('execute sql: %s' % sql) cursor.execute(sql) + for zone in zones_config: + zone_config = zones_config[zone] + for key in zone_config: + sql = 'alter system modify zone %s set %s = %%s' % (zone, inner_config[key]) + stdio.verbose('execute sql: %s' % sql) + cursor.execute(sql, [zone_config[key]]) stdio.stop_loading('succeed') plugin_context.return_true() except: diff --git a/plugins/oceanbase/3.1.0/create_tenant.py b/plugins/oceanbase/3.1.0/create_tenant.py index 34433a0630d1c010fd9b8b286cf97e3c061c3e9e..d154157eabf0f096154000071dd45a41297a42e7 100644 --- a/plugins/oceanbase/3.1.0/create_tenant.py +++ b/plugins/oceanbase/3.1.0/create_tenant.py @@ -24,6 +24,8 @@ from __future__ import absolute_import, division, print_function import re import time +from _errno import EC_OBSERVER_CAN_NOT_MIGRATE_IN + def parse_size(size): _bytes = 0 @@ -36,7 +38,7 @@ def parse_size(size): return _bytes -def formate_size(size, precision=1): +def format_size(size, precision=1): units = ['B', 'K', 'M', 'G', 'T', 'P'] idx = 0 if precision: @@ -121,7 +123,7 @@ def create_tenant(plugin_context, cursor, *args, **kwargs): count -= 1 time.sleep(1) if count == 0: - stdio.error('server can not migrate in') + stdio.error(EC_OBSERVER_CAN_NOT_MIGRATE_IN) return except: exception('execute sql exception: %s' % sql) @@ -188,9 +190,9 @@ def create_tenant(plugin_context, cursor, *args, **kwargs): if cpu_total < MIN_CPU: return error('%s: resource not enough: cpu count less than %s' % (zone_list, MIN_CPU)) if mem_total < MIN_MEMORY: - return error('%s: resource not enough: memory less than %s' % (zone_list, formate_size(MIN_MEMORY))) + return error('%s: resource not enough: memory less than %s' % (zone_list, format_size(MIN_MEMORY))) if disk_total < MIN_DISK_SIZE: - return error('%s: resource not enough: disk space less than %s' % (zone_list, formate_size(MIN_DISK_SIZE))) + return error('%s: resource not enough: disk space less than %s' % (zone_list, format_size(MIN_DISK_SIZE))) max_cpu = get_option('max_cpu', cpu_total) max_memory = parse_size(get_option('max_memory', mem_total)) @@ -204,9 +206,9 @@ def create_tenant(plugin_context, cursor, *args, **kwargs): if cpu_total < max_cpu: return error('resource not enough: cpu (Avail: %s, Need: %s)' % (cpu_total, max_cpu)) if mem_total < max_memory: - return error('resource not enough: memory (Avail: %s, Need: %s)' % (formate_size(mem_total), formate_size(max_memory))) + return error('resource not enough: memory (Avail: %s, Need: %s)' % (format_size(mem_total), format_size(max_memory))) if disk_total < max_disk_size: - return error('resource not enough: disk space (Avail: %s, Need: %s)' % (formate_size(disk_total), formate_size(max_disk_size))) + return error('resource not enough: disk space (Avail: %s, Need: %s)' % (format_size(disk_total), format_size(max_disk_size))) if max_iops < MIN_IOPS: return error('max_iops must greater than %d' % MIN_IOPS) diff --git a/plugins/oceanbase/3.1.0/destroy.py b/plugins/oceanbase/3.1.0/destroy.py index 784897d1b140637aa077ec74ec8b075cf5038680..9094908ece43f2f8fcff04dce37016630053397e 100644 --- a/plugins/oceanbase/3.1.0/destroy.py +++ b/plugins/oceanbase/3.1.0/destroy.py @@ -20,18 +20,20 @@ from __future__ import absolute_import, division, print_function +from _errno import EC_CLEAN_PATH_FAILED + global_ret = True def destroy(plugin_context, *args, **kwargs): def clean(server, path): client = clients[server] - ret = client.execute_command('rm -fr %s/* %s/.conf' % (path, path)) + ret = client.execute_command('rm -fr %s/' % (path)) if not ret: # print stderror global global_ret global_ret = False - stdio.warn('fail to clean %s:%s' % (server, path)) + stdio.warn(EC_CLEAN_PATH_FAILED.format(server=server, path=path)) else: stdio.verbose('%s:%s cleaned' % (server, path)) cluster_config = plugin_context.cluster_config diff --git a/plugins/oceanbase/3.1.0/generate_config.py b/plugins/oceanbase/3.1.0/generate_config.py index 20dcbb2b47e8c34a978917795544eda1ddcdb75e..2e3e2162ebac908729ee5859a55b020d00136427 100644 --- a/plugins/oceanbase/3.1.0/generate_config.py +++ b/plugins/oceanbase/3.1.0/generate_config.py @@ -23,6 +23,8 @@ from __future__ import absolute_import, division, print_function import re, os +from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY + def parse_size(size): _bytes = 0 @@ -145,7 +147,7 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): free_memory = parse_size(str(v)) memory_limit = free_memory if memory_limit < MIN_MEMORY: - stdio.error('(%s) not enough memory. (Free: %s, Need: %s)' % (ip, format_size(free_memory), format_size(MIN_MEMORY))) + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(MIN_MEMORY))) success = False continue memory_limit = max(MIN_MEMORY, memory_limit * 0.9) @@ -173,7 +175,7 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): ret = client.execute_command("grep -e 'processor\s*:' /proc/cpuinfo | wc -l") if ret and ret.stdout.strip().isdigit(): cpu_num = int(ret.stdout) - server_config['cpu_count'] = max(16, int(cpu_num * - 2)) + server_config['cpu_count'] = max(16, int(cpu_num - 2)) else: server_config['cpu_count'] = 16 diff --git a/plugins/oceanbase/3.1.0/init.py b/plugins/oceanbase/3.1.0/init.py index f57e6539838f6147298abbf68ebc90daa0964921..2c94c68d48b19f4267477e721193cd890a47e09c 100644 --- a/plugins/oceanbase/3.1.0/init.py +++ b/plugins/oceanbase/3.1.0/init.py @@ -21,6 +21,8 @@ from __future__ import absolute_import, division, print_function import os +from _errno import EC_CONFIG_CONFLICT_DIR, EC_FAIL_TO_INIT_PATH, InitDirFailedErrorMessage + stdio = None force = False @@ -34,18 +36,18 @@ def critical(*arg, **kwargs): def init_dir(server, client, key, path, link_path=None): if force: - ret = client.execute_command('rm -fr %s/*' % path) + ret = client.execute_command('rm -fr %s' % path) if not ret: - critical('fail to initialize %s %s path: %s permission denied' % (server, key, ret.stderr)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='%s path' % key, msg=ret.stderr)) return False else: if client.execute_command('mkdir -p %s' % path): ret = client.execute_command('ls %s' % (path)) if not ret or ret.stdout.strip(): - critical('fail to initialize %s %s path: %s is not empty' % (server, key, path)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='%s path' % key, msg=InitDirFailedErrorMessage.NOT_EMPTY.format(path=path))) return False else: - critical('fail to initialize %s %s path: create %s failed' % (server, key, path)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='%s path' % key, msg=InitDirFailedErrorMessage.CREATE_FAILED.format(path=path))) return False ret = client.execute_command('mkdir -p %s' % path) if ret: @@ -53,7 +55,7 @@ def init_dir(server, client, key, path, link_path=None): client.execute_command("if [ ! '%s' -ef '%s' ]; then ln -sf %s %s; fi" % (path, link_path, path, link_path)) return True else: - critical('fail to initialize %s %s path: %s permission denied' % (server, key, ret.stderr)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='%s path' % key, msg=ret.stderr)) return False @@ -74,7 +76,7 @@ def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): server_config = cluster_config.get_server_conf(server) client = clients[server] home_path = server_config['home_path'] - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_dir = repository_dir.replace(local_home_path, remote_home_path) if not server_config.get('data_dir'): @@ -94,7 +96,7 @@ def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): for key in keys: path = server_config[key] if path in dirs: - critical('Configuration conflict %s: %s is used for %s\'s %s' % (server, path, dirs[path]['server'], dirs[path]['key'])) + critical(EC_CONFIG_CONFLICT_DIR.format(server1=server, path=path, server2=dirs[path]['server'], key=dirs[path]['key'])) continue dirs[path] = { 'server': server, @@ -105,16 +107,16 @@ def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): if force: ret = client.execute_command('rm -fr %s/*' % home_path) if not ret: - critical('failed to initialize %s home path: %s' % (server, ret.stderr)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=ret.stderr)) continue else: if client.execute_command('mkdir -p %s' % home_path): ret = client.execute_command('ls %s' % (home_path)) if not ret or ret.stdout.strip(): - critical('fail to init %s home path: %s is not empty' % (server, home_path)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=InitDirFailedErrorMessage.NOT_EMPTY.format(path=home_path))) continue else: - critical('fail to init %s home path: create %s failed' % (server, home_path)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=InitDirFailedErrorMessage.CREATE_FAILED.format(path=home_path))) ret = client.execute_command('bash -c "mkdir -p %s/{etc,admin,.conf,log,bin,lib}"' % home_path) \ and client.execute_command("if [ -d %s/bin ]; then ln -fs %s/bin/* %s/bin; fi" % (remote_repository_dir, remote_repository_dir, home_path)) \ and client.execute_command("if [ -d %s/lib ]; then ln -fs %s/lib/* %s/lib; fi" % (remote_repository_dir, remote_repository_dir, home_path)) @@ -123,16 +125,16 @@ def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): if force: ret = client.execute_command('rm -fr %s/*' % data_path) if not ret: - critical('fail to init %s data path: %s permission denied' % (server, ret.stderr)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='data dir', msg=InitDirFailedErrorMessage.PERMISSION_DENIED.format(path=data_path))) continue else: if client.execute_command('mkdir -p %s' % data_path): ret = client.execute_command('ls %s' % (data_path)) if not ret or ret.stdout.strip(): - critical('fail to init %s data path: %s is not empty' % (server, data_path)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='data dir', msg=InitDirFailedErrorMessage.NOT_EMPTY.format(path=data_path))) continue else: - critical('fail to init %s data path: create %s failed' % (server, data_path)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='data dir', msg=InitDirFailedErrorMessage.CREATE_FAILED.format(path=data_path))) ret = client.execute_command('bash -c "mkdir -p %s/sstable"' % data_path) if ret: link_path = '%s/store' % home_path @@ -143,26 +145,26 @@ def init(plugin_context, local_home_path, repository_dir, *args, **kwargs): if force: ret = client.execute_command('rm -fr %s/*' % log_dir) if not ret: - critical('fail to init %s %s dir: %s permission denied' % (server, key, ret.stderr)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='%s dir' % key, msg=InitDirFailedErrorMessage.PERMISSION_DENIED.format(path=log_dir))) continue else: if client.execute_command('mkdir -p %s' % log_dir): ret = client.execute_command('ls %s' % (log_dir)) if not ret or ret.stdout.strip(): - critical('fail to init %s %s dir: %s is not empty' % (server, key, log_dir)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='%s dir' % key, msg=InitDirFailedErrorMessage.NOT_EMPTY.format(path=log_dir))) continue else: - critical('fail to init %s %s dir: create %s failed' % (server, key, log_dir)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='%s dir' % key, msg=InitDirFailedErrorMessage.CREATE_FAILED.format(path=log_dir))) ret = client.execute_command('mkdir -p %s' % log_dir) if ret: link_path = '%s/%s' % (data_path, key) client.execute_command("if [ ! '%s' -ef '%s' ]; then ln -sf %s %s; fi" % (log_dir, link_path, log_dir, link_path)) else: - critical('failed to initialize %s %s dir' % (server, key)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='%s dir' % key, msg=ret.stderr)) else: - critical('failed to initialize %s date path' % (server)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='data dir', msg=InitDirFailedErrorMessage.PATH_ONLY.format(path=data_path))) else: - critical('fail to init %s home path: %s permission denied' % (server, ret.stderr)) + critical(EC_FAIL_TO_INIT_PATH.format(server=server, key='home path', msg=InitDirFailedErrorMessage.PERMISSION_DENIED.format(path=home_path))) if global_ret: stdio.stop_loading('succeed') diff --git a/plugins/oceanbase/3.1.0/ocp_check.py b/plugins/oceanbase/3.1.0/ocp_check.py new file mode 100644 index 0000000000000000000000000000000000000000..0523778ffc0e71bfc88da358054384a567670696 --- /dev/null +++ b/plugins/oceanbase/3.1.0/ocp_check.py @@ -0,0 +1,92 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +from _rpm import Version +from _deploy import InnerConfigItem + + +def ocp_check(plugin_context, ocp_version, cursor, new_cluster_config=None, new_clients=None, *args, **kwargs): + cluster_config = new_cluster_config if new_cluster_config else plugin_context.cluster_config + clients = new_clients if new_clients else plugin_context.clients + stdio = plugin_context.stdio + + is_admin = True + can_sudo = True + only_one = True + pwd_not_empty = True + + min_version = Version('3.1.1') + max_version = min_version + ocp_version = Version(ocp_version) + + if ocp_version < min_version: + stdio.error('The current plugin version does not support OCP V%s' % ocp_version) + return + + if ocp_version > max_version: + stdio.warn('The plugin library does not support OCP V%s. The takeover requirements are not applicable to the current check.' % ocp_version) + + for server in cluster_config.servers: + client = clients[server] + if is_admin and client.config.username != 'admin': + is_admin = False + stdio.error('The current user must be the admin user. Run the edit-config command to modify the user.username field') + if can_sudo and not client.execute_command('sudo whoami'): + can_sudo = False + stdio.error('The user must have the privilege to run sudo commands without a password.') + if not client.execute_command('bash -c "if [ `pgrep observer | wc -l` -gt 1 ]; then exit 1; else exit 0;fi;"'): + only_one = False + stdio.error('%s Multiple OBservers exist.' % server) + + try: + cursor.execute("select * from oceanbase.__all_user where user_name = 'root' and passwd = ''") + if cursor.fetchone() and not cluster_config.get_global_conf().get("root_password"): + pwd_not_empty = False + stdio.error('The password of root@sys is empty. Run the edit-config command to modify the root_password value of %s.' % cluster_config.name) + except: + if not cluster_config.get_global_conf().get("root_password"): + pwd_not_empty = False + + zones = {} + try: + cursor.execute("select zone from oceanbase.__all_zone where name = 'idc' and info = ''") + ret = cursor.fetchall() + if ret: + for row in ret: + zones[str(row['zone'])] = 1 + finally: + for server in cluster_config.servers: + config = cluster_config.get_server_conf(server) + zone = str(config.get('zone')) + if zone in zones and config.get('$_zone_idc'): + keys = list(config.keys()) + if '$_zone_idc' in keys and isinstance(keys[keys.index('$_zone_idc')], InnerConfigItem): + del zones[zone] + if zones: + if not cluster_config.parser or cluster_config.parser.STYLE == 'default': + stdio.error('Zone: IDC information is missing for %s. Run the chst command to change the configuration style of %s to cluster, and then run the edit-config command to add IDC information.' % (','.join(zones.keys()), cluster_config.name)) + else: + stdio.error('Zone: IDC information is missing for %s. Run the edit-config command to add IDC information.' % ','.join(zones.keys())) + + if is_admin and can_sudo and only_one and pwd_not_empty and not zones: + stdio.print('Configurations of the %s can be taken over by OCP after they take effect.' % cluster_config.name if new_cluster_config else 'Configurations of the %s can be taken over by OCP.' % cluster_config.name) + return plugin_context.return_true() \ No newline at end of file diff --git a/plugins/oceanbase/3.1.0/reload.py b/plugins/oceanbase/3.1.0/reload.py index 401b4dfb8b3e6d815687f8d95c13ffda7a359d65..0fb6bd95bef46310962c8e0db1a8f1a07d513efa 100644 --- a/plugins/oceanbase/3.1.0/reload.py +++ b/plugins/oceanbase/3.1.0/reload.py @@ -20,11 +20,19 @@ from __future__ import absolute_import, division, print_function +from _deploy import InnerConfigItem +from _errno import EC_OBSERVER_INVALID_MODFILY_GLOBAL_KEY + def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): stdio = plugin_context.stdio cluster_config = plugin_context.cluster_config servers = cluster_config.servers + inner_config = { + InnerConfigItem('$_zone_idc'): 'idc' + } + inner_keys = inner_config.keys() + zones_config = {} cluster_server = {} change_conf = {} global_change_conf = {} @@ -41,34 +49,65 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): for key in new_config: n_value = new_config[key] if key not in config or config[key] != n_value: - change_conf[server][key] = n_value - if key not in global_change_conf: - global_change_conf[key] = {'value': n_value, 'count': 1} - elif n_value == global_change_conf[key]['value']: - global_change_conf[key]['count'] += 1 + if isinstance(key, InnerConfigItem) and key in inner_keys: + zone = config['zone'] + if zone not in zones_config: + zones_config[zone] = {} + zones_config[zone][key] = n_value + else: + item = cluster_config.get_temp_conf_item(key) + if item: + if item.need_redeploy or item.need_restart: + stdio.verbose('%s can not be reload' % key) + global_ret = False + continue + try: + item.modify_limit(config.get(key), n_value) + except Exception as e: + global_ret = False + stdio.verbose('%s: %s' % (server, str(e))) + continue + change_conf[server][key] = n_value + if key not in global_change_conf: + global_change_conf[key] = {'value': n_value, 'count': 1} + elif n_value == global_change_conf[key]['value']: + global_change_conf[key]['count'] += 1 servers_num = len(servers) stdio.verbose('apply new configuration') + stdio.start_load('Reload observer') + for zone in zones_config: + zone_config = zones_config[zone] + for key in zone_config: + msg = '' + try: + msg = sql = 'alter system modify zone %s set %s = %%s' % (zone, inner_config[key]) + stdio.verbose('execute sql: %s' % sql) + cursor.execute(sql, [zone_config[key]]) + stdio.verbose('%s ok' % sql) + except: + global_ret = False + stdio.exception('execute sql exception: %s' % msg) + for key in global_change_conf: msg = '' try: if key in ['proxyro_password', 'root_password']: if global_change_conf[key]['count'] != servers_num: - stdio.warn('Invalid: proxyro_password is not a single server configuration item') + stdio.warn(EC_OBSERVER_INVALID_MODFILY_GLOBAL_KEY.format(key=key)) continue value = change_conf[server][key] if change_conf[server].get(key) is not None else '' user = key.split('_')[0] - msg = sql = 'CREATE USER IF NOT EXISTS %s IDENTIFIED BY "%s"' % (user, value) + msg = sql = 'CREATE USER IF NOT EXISTS %s IDENTIFIED BY %%s' % (user) stdio.verbose('execute sql: %s' % sql) - cursor.execute(sql) - msg = sql = 'alter user "%s" IDENTIFIED BY "%s"' % (user, value) + cursor.execute(sql, [value]) + msg = sql = 'alter user "%s" IDENTIFIED BY %%s' % (user) stdio.verbose('execute sql: %s' % sql) - cursor.execute(sql) + cursor.execute(sql, [value]) continue if global_change_conf[key]['count'] == servers_num: sql = 'alter system set %s = %%s' % key value = change_conf[server][key] - msg = sql % value stdio.verbose('execute sql: %s' % msg) cursor.execute(sql, [value]) cluster_config.update_global_conf(key, value, False) @@ -76,11 +115,9 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): for server in servers: if key not in change_conf[server]: continue - sql = 'alter system set %s = %%s server=%%s' % key - value = (change_conf[server][key], cluster_server[server]) - msg = sql % value + msg = sql = 'alter system set %s = %%s server=%%s' % key stdio.verbose('execute sql: %s' % msg) - cursor.execute(sql, value) + cursor.execute(sql, [change_conf[server][key], cluster_server[server]]) cluster_config.update_server_conf(server,key, value, False) except: global_ret = False @@ -89,4 +126,10 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): cursor.execute('alter system reload server') cursor.execute('alter system reload zone') cursor.execute('alter system reload unit') - return plugin_context.return_true() if global_ret else None + + if global_ret: + stdio.stop_load('succeed') + return plugin_context.return_true() + else: + stdio.stop_load('fail') + return diff --git a/plugins/oceanbase/3.1.0/restart.py b/plugins/oceanbase/3.1.0/restart.py new file mode 100644 index 0000000000000000000000000000000000000000..43c2f7b57bc6fab6d79f7060bb39d917c06cbd51 --- /dev/null +++ b/plugins/oceanbase/3.1.0/restart.py @@ -0,0 +1,289 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +import os +import time + + +class Restart(object): + + def __init__(self, plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None): + self.local_home_path = local_home_path + self.plugin_context = plugin_context + self.components = plugin_context.components + self.clients = plugin_context.clients + self.cluster_config = plugin_context.cluster_config + self.stdio = plugin_context.stdio + self.repository = repository + self.start_plugin = start_plugin + self.reload_plugin = reload_plugin + self.connect_plugin = connect_plugin + self.stop_plugin = stop_plugin + self.display_plugin = display_plugin + self.new_clients = new_clients + self.new_cluster_config = new_cluster_config + self.now_clients = {} + self.sub_io = self.stdio.sub_io() + self.db = None + self.cursor = None + for server in self.cluster_config.servers: + self.now_clients[server] = self.clients[server] + + def close(self): + if self.db: + self.cursor.close() + self.db.close() + self.cursor = None + self.db = None + + def connect(self): + if self.cursor is None or self.execute_sql('select version()', error=False) is False: + self.stdio.verbose('Call %s for %s' % (self.connect_plugin, self.repository)) + self.sub_io.start_loading('Connect to observer') + ret = self.connect_plugin(self.components, self.clients, self.cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io) + if not ret: + self.sub_io.stop_loading('fail') + return False + self.sub_io.stop_loading('succeed') + self.close() + self.cursor = ret.get_return('cursor') + self.db = ret.get_return('connect') + while self.execute_sql('use oceanbase', error=False) is False: + time.sleep(2) + self.execute_sql('set session ob_query_timeout=1000000000') + return True + + def execute_sql(self, query, args=None, one=True, error=True): + msg = query % tuple(args) if args is not None else query + self.stdio.verbose("query: %s. args: %s" % (query, args)) + try: + self.stdio.verbose('execute sql: %s' % msg) + self.cursor.execute(query, args) + result = self.cursor.fetchone() if one else self.cursor.fetchall() + result and self.stdio.verbose(result) + return result + except: + msg = 'execute sql exception: %s' % msg if error else '' + self.stdio.exception(msg) + return False + + def broken_sql(self, sql, sleep_time=3): + while True: + ret = self.execute_sql(sql, error=False) + if ret is None: + break + time.sleep(sleep_time) + + def wait(self): + if not self.connect(): + return False + self.stdio.verbose('server cneck') + self.broken_sql("select * from oceanbase.__all_server where status != 'active' or stop_time > 0 or start_service_time = 0") + self.broken_sql("select * from oceanbase.__all_virtual_clog_stat where is_in_sync= 0 and is_offline = 0") + return True + + def start_zone(self, zone=None): + if not self.connect(): + return False + if zone: + self.stdio.verbose('start zone %s' % zone) + start_sql = "alter system start zone %s" % zone + check_sql = "select * from oceanbase.__all_zone where name = 'status' and zone = '%s' and info != 'ACTIVE'" % zone + while True: + if self.execute_sql(start_sql, error=False) is None: + break + if self.execute_sql(check_sql, error=False) is None: + break + time.sleep(3) + self.wait() + return True + + def stop_zone(self, zone): + if not self.wait(): + return False + + self.stdio.verbose('stop zone %s' % zone) + stop_sql = "alter system stop zone %s" % zone + check_sql = "select * from oceanbase.__all_zone where name = 'status' and zone = '%s' and info = 'ACTIVE'" % zone + while True: + if self.execute_sql(stop_sql, error=False) is None: + break + if self.execute_sql(check_sql, error=False): + break + time.sleep(3) + return True + + def rollback(self): + if self.new_clients: + self.stdio.start_loading('Rollback') + self.stop_plugin(self.components, self.now_clients, self.new_cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io) + for server in self.cluster_config.servers: + client = self.clients[server] + new_client = self.now_clients[server] + server_config = self.cluster_config.get_server_conf(server) + home_path = server_config['home_path'] + chown_cmd = 'sudo chown -R %s:' % client.config.username + for key in ['home_path', 'data_dir', 'redo_dir']: + if key in server_config: + chown_cmd += ' %s' % server_config[key] + new_client.execute_command(chown_cmd) + self.stdio.stop_loading('succeed') + + def dir_read_check(self, client, path): + if not client.execute_command('cd %s' % path): + dirpath, name = os.path.split(path) + return self.dir_read_check(client, dirpath) and client.execute_command('sudo chmod +1 %s' % path) + return True + + def _restart(self): + clients = self.clients + self.stdio.verbose('Call %s for %s' % (self.stop_plugin, self.repository)) + if not self.stop_plugin(self.components, clients, self.cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io): + self.stdio.stop_loading('stop_loading', 'fail') + return False + + if self.new_clients: + self.stdio.verbose('use new clients') + for server in self.cluster_config.servers: + new_client = self.new_clients[server] + server_config = self.cluster_config.get_server_conf(server) + chown_cmd = 'sudo chown -R %s:' % new_client.config.username + for key in ['home_path', 'data_dir', 'redo_dir']: + if key in server_config: + chown_cmd += ' %s' % server_config[key] + if not new_client.execute_command(chown_cmd): + self.stdio.stop_loading('stop_loading', 'fail') + return False + self.dir_read_check(new_client, server_config['home_path']) + self.now_clients[server] = new_client + clients = self.new_clients + + cluster_config = self.new_cluster_config if self.new_cluster_config else self.cluster_config + self.stdio.verbose('Call %s for %s' % (self.start_plugin, self.repository)) + if not self.start_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, local_home_path=self.local_home_path, repository_dir=self.repository.repository_dir): + self.stdio.stop_loading('stop_loading', 'fail') + return False + return True + + def rolling(self, zones_servers): + self.stdio.start_loading('Observer rotation restart') + all_servers = self.cluster_config.servers + pre_zone = None + for zone in zones_servers: + self.cluster_config.servers = zones_servers[zone] + if self.new_cluster_config: + self.new_cluster_config.servers = zones_servers[zone] + if not self.start_zone(pre_zone): + self.stdio.stop_loading('stop_loading', 'fail') + return False + while True: + for server in zones_servers[zone]: + config = self.cluster_config.get_server_conf(server) + sql = ''' + select count(*) as cnt from oceanbase.__all_tenant as a left join ( + select tenant_id, refreshed_schema_version + from oceanbase.__all_virtual_server_schema_info + where svr_ip = %s and svr_port = %s and refreshed_schema_version > 1 + ) as b on a.tenant_id = b.tenant_id + where b.tenant_id is null''' + if self.execute_sql(sql, args=(server.ip, config['rpc_port'])).get('cnt'): + break + else: + break + time.sleep(3) + + while self.execute_sql("select * from oceanbase.__all_virtual_clog_stat where table_id = 1099511627777 and status != 'ACTIVE'"): + time.sleep(3) + + self.stop_zone(zone) + if not self._restart(): + return False + pre_zone = zone + + if not self.start_zone(pre_zone): + self.stdio.stop_loading('stop_loading', 'fail') + return False + + self.cluster_config.servers = all_servers + if self.new_cluster_config: + self.new_cluster_config.servers = all_servers + self.stdio.stop_loading('succeed') + return True + + def un_rolling(self): + self.stdio.start_loading('Observer restart') + + if not self._restart(): + return False + + self.wait() + self.stdio.stop_loading('succeed') + return True + + def restart(self): + zones_servers = {} + all_servers = self.cluster_config.servers + if self.connect(): + self.stdio.start_loading('Server check') + servers = self.execute_sql("select * from oceanbase.__all_server", one=False, error=False) + if len(self.cluster_config.servers) == len(servers): + for server in servers: + if server['status'] != 'active' or server['stop_time'] > 0 or server['start_service_time'] == 0: + break + else: + for server in self.cluster_config.servers: + config = self.cluster_config.get_server_conf_with_default(server) + zone = config['zone'] + if zone not in zones_servers: + zones_servers[zone] = [] + zones_servers[zone].append(server) + servers = self.cluster_config.servers + self.stdio.stop_loading('succeed') + ret = False + try: + if len(zones_servers) > 2: + ret = self.rolling(zones_servers) + else: + ret = self.un_rolling() + + if ret and self.connect(): + self.display_plugin(self.components, self.new_clients if self.new_clients else self.clients, self.new_cluster_config if self.new_cluster_config else self.cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, cursor=self.cursor) + if self.new_cluster_config: + self.stdio.verbose('Call %s for %s' % (self.reload_plugin, self.repository)) + self.reload_plugin(self.components, self.clients, self.cluster_config, [], {}, self.sub_io, + cursor=self.cursor, new_cluster_config=self.new_cluster_config, repository_dir=self.repository.repository_dir) + except Exception as e: + self.stdio.exception('Run Exception: %s' % e) + finally: + self.cluster_config.servers = all_servers + if self.new_cluster_config: + self.new_cluster_config.servers = all_servers + if not ret: + self.rollback() + return ret + + +def restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None, rollback=False, *args, **kwargs): + task = Restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config, new_clients) + call = task.rollback if rollback else task.restart + if call(): + plugin_context.return_true() diff --git a/plugins/oceanbase/3.1.0/start.py b/plugins/oceanbase/3.1.0/start.py index 53229d7ed574f1b942b86504d0db83d64e5402e6..56291ac105698827105dab9faa0477d3b68874f3 100644 --- a/plugins/oceanbase/3.1.0/start.py +++ b/plugins/oceanbase/3.1.0/start.py @@ -26,6 +26,8 @@ import time import requests from copy import deepcopy +from _errno import EC_OBSERVER_FAIL_TO_START + def config_url(ocp_config_server, appname, cid): cfg_url = '%s&Action=ObRootServiceInfo&ObCluster=%s' % (ocp_config_server, appname) @@ -97,7 +99,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): home_path = server_config['home_path'] if client.execute_command("bash -c 'if [ -f %s/bin/observer ]; then exit 1; else exit 0; fi;'" % home_path): - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_dir = repository_dir.replace(local_home_path, remote_home_path) client.execute_command("bash -c 'mkdir -p %s/{bin,lib}'" % (home_path)) client.execute_command("ln -fs %s/bin/* %s/bin" % (remote_repository_dir, home_path)) @@ -139,7 +141,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): } not_cmd_opt = [ 'home_path', 'obconfig_url', 'root_password', 'proxyro_password', - 'redo_dir', 'clog_dir', 'ilog_dir', 'slog_dir' + 'redo_dir', 'clog_dir', 'ilog_dir', 'slog_dir', '$_zone_idc' ] get_value = lambda key: "'%s'" % server_config[key] if isinstance(server_config[key], str) else server_config[key] opt_str = [] @@ -168,7 +170,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): client.add_env('LD_LIBRARY_PATH', '', True) if not ret: stdio.stop_loading('fail') - stdio.error('failed to start %s observer: %s' % (server, ret.stderr)) + stdio.error(EC_OBSERVER_FAIL_TO_START.format(server=server) + ': ' + ret.stderr) return stdio.stop_loading('succeed') @@ -185,7 +187,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): if remote_pid and client.execute_command('ls /proc/%s' % remote_pid): stdio.verbose('%s observer[pid: %s] started', server, remote_pid) else: - failed.append('failed to start %s observer' % server) + failed.append(EC_OBSERVER_FAIL_TO_START.format(server=server)) if failed: stdio.stop_loading('fail') for msg in failed: diff --git a/plugins/oceanbase/3.1.0/start_check.py b/plugins/oceanbase/3.1.0/start_check.py index 5873814aac30d671be7940296659bdb7911f1cc6..53cc6a845e38593c5e69932533a181168ca44ae9 100644 --- a/plugins/oceanbase/3.1.0/start_check.py +++ b/plugins/oceanbase/3.1.0/start_check.py @@ -24,6 +24,8 @@ import os import re import time +from _errno import EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, EC_OBSERVER_NOT_ENOUGH_MEMORY + stdio = None success = True @@ -50,7 +52,7 @@ def parse_size(size): return _bytes -def formate_size(size): +def format_size(size): units = ['B', 'K', 'M', 'G', 'T', 'P'] idx = 0 while idx < 5 and size >= 1024: @@ -120,7 +122,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): for key in ['mysql_port', 'rpc_port']: port = int(server_config[key]) if port in ports: - critical('Configuration conflict %s: %s port is used for %s\'s %s' % (server, port, ports[port]['server'], ports[port]['key'])) + critical(EC_CONFIG_CONFLICT_PORT.format(server1=server, port=port, server2=ports[port]['server'], key=ports[port]['key'])) continue ports[port] = { 'server': server, @@ -214,7 +216,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): free_memory = parse_size(str(v)) total_use = servers_memory[ip]['percentage'] * total_memory / 100 + servers_memory[ip]['num'] if total_use > free_memory: - critical('(%s) not enough memory. (Free: %s, Need: %s)' % (ip, formate_size(free_memory), formate_size(total_use))) + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY.formate(ip=ip, free=format_size(free_memory), need=format_size(total_use))) # disk disk = {'/': 0} ret = client.execute_command('df --block-size=1024') @@ -275,11 +277,12 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): if need > 0 and threshold < 2: alert('(%s) clog and data use the same disk (%s)' % (ip, p)) if need > avail: - critical('(%s) %s not enough disk space. (Avail: %s, Need: %s)' % (ip, p, formate_size(avail), formate_size(need))) + critical('(%s) %s not enough disk space. (Avail: %s, Need: %s)' % (ip, p, format_size(avail), format_size(need))) elif 1.0 * (total - avail + need) / total > disk[p]['threshold']: - msg = '(%s) %s not enough disk space for clog. Use `redo_dir` to set other disk for clog' % (ip, p) - msg += ', or reduce the value of `datafile_size`' if need > 0 else '.' - critical(msg) + # msg = '(%s) %s not enough disk space for clog. Use `redo_dir` to set other disk for clog' % (ip, p) + # msg += ', or reduce the value of `datafile_size`' if need > 0 else '.' + # critical(msg) + critical(EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG.format(ip=ip, path=p)) if success: for ip in servers_net_inferface: diff --git a/plugins/oceanbase/3.1.0/stop.py b/plugins/oceanbase/3.1.0/stop.py index 026378e36d724718472af60449f083852827f80b..a07d1a7c435aa14ed7684f887de63918dc0d746c 100644 --- a/plugins/oceanbase/3.1.0/stop.py +++ b/plugins/oceanbase/3.1.0/stop.py @@ -44,13 +44,16 @@ def get_port_socket_inode(client, port): return res.stdout.strip().split('\n') -def confirm_port(client, pid, port): +def port_release_check(client, pid, port, count): socket_inodes = get_port_socket_inode(client, port) if not socket_inodes: - return False - ret = client.execute_command("ls -l /proc/%s/fd/ |grep -E 'socket:\[(%s)\]'" % (pid, '|'.join(socket_inodes))) - if ret and ret.stdout.strip(): return True + if count < 5: + ret = client.execute_command("ls -l /proc/%s/fd/ |grep -E 'socket:\[(%s)\]'" % (pid, '|'.join(socket_inodes))) + if ret: + return not ret.stdout.strip() + else: + return not client.execute_command("ls -l /proc/%s" % pid) return False @@ -97,7 +100,6 @@ def stop(plugin_context, *args, **kwargs): else: stdio.verbose('%s observer is not running ...' % server) count = 30 - check = lambda client, pid, port: confirm_port(client, pid, port) if count < 5 else get_port_socket_inode(client, port) time.sleep(1) while count and servers: tmp_servers = {} @@ -106,7 +108,7 @@ def stop(plugin_context, *args, **kwargs): client = clients[server] stdio.verbose('%s check whether the port is released' % server) for key in ['rpc_port', 'mysql_port']: - if data[key] and check(data['client'], data['pid'], data[key]): + if data[key] and not port_release_check(data['client'], data['pid'], data[key], count): tmp_servers[server] = data break data[key] = '' diff --git a/plugins/oceanbase/3.1.0/upgrade.py b/plugins/oceanbase/3.1.0/upgrade.py index f1d4907ff73d0707b1ebdb9e4c8f588b62a24d83..6492f2c56843874321d0cf1b80e9f8965930f898 100644 --- a/plugins/oceanbase/3.1.0/upgrade.py +++ b/plugins/oceanbase/3.1.0/upgrade.py @@ -137,7 +137,7 @@ class Upgrader(object): self._connect_plugin = None self._start_plugin = None self._stop_plugin = None - self._display_plguin = None + self._display_plugin = None self.local_home_path = local_home_path self.exector_path = exector_path self.components = plugin_context.components @@ -193,15 +193,15 @@ class Upgrader(object): @property def display_plugin(self): - if self._display_plguin is None: - self._display_plguin = self.search_py_script_plugin(self.route_index - 1, 'display') - return self._display_plguin + if self._display_plugin is None: + self._display_plugin = self.search_py_script_plugin(self.route_index - 1, 'display') + return self._display_plugin def _clear_plugin(self): self._connect_plugin = None self._start_plugin = None self._stop_plugin = None - self._display_plguin = None + self._display_plugin = None def run(self): total = len(self.route) @@ -392,7 +392,7 @@ class Upgrader(object): client = self.clients[server] server_config = self.cluster_config.get_server_conf(server) home_path = server_config['home_path'] - remote_home_path = client.execute_command('echo $HOME/.obd').stdout.strip() + remote_home_path = client.execute_command('echo ${OBD_HOME:-"$HOME"}/.obd').stdout.strip() remote_repository_dir = repository_dir.replace(self.local_home_path, remote_home_path) client.execute_command("bash -c 'mkdir -p %s/{bin,lib}'" % (home_path)) client.execute_command("ln -fs %s/bin/* %s/bin" % (remote_repository_dir, home_path)) diff --git a/plugins/oceanbase/3.1.0/upgrade_check.py b/plugins/oceanbase/3.1.0/upgrade_check.py index 8825613f9f9cb37c0f43835f7debb13a711f4584..42ca225f2faf58e1109529bc65baa32b2a0d2af1 100644 --- a/plugins/oceanbase/3.1.0/upgrade_check.py +++ b/plugins/oceanbase/3.1.0/upgrade_check.py @@ -83,7 +83,7 @@ def upgrade_check(plugin_context, current_repository, repositories, route, curso succeed = False stdio.error('No such file: %s .' % path) if cant_use: - stdio.error('%s 不可用于升级，可以使用--disable禁用该镜像' % repository) + stdio.error('%s cannot be used for the upgrade. You can use the --disable option to disable the image.' % repository) i += 1 if succeed: diff --git a/plugins/oceanbase/3.1.0/upgrade_file_check.py b/plugins/oceanbase/3.1.0/upgrade_file_check.py index 1625b6fe42861658fb5f3e0b56675535f16e6419..1b909da181ced94b59bd0f5813cd76f30dd9f354 100644 --- a/plugins/oceanbase/3.1.0/upgrade_file_check.py +++ b/plugins/oceanbase/3.1.0/upgrade_file_check.py @@ -55,7 +55,7 @@ def upgrade_file_check(plugin_context, current_repository, repositories, route, succeed = False stdio.error('No such file: %s .' % path) if cant_use: - stdio.error('%s 不可用于升级，可以使用--disable禁用该镜像' % repository) + stdio.error('%s cannot be used for the upgrade. You can use the --disable option to disable the image.' % repository) i += 1 if succeed: diff --git a/plugins/sysbench/3.1.0/run_test.py b/plugins/sysbench/3.1.0/run_test.py index 198035e571d70d359aad012b224d74d86af9117a..2db95b35a703ab51401abfe5eeddbbe9d2c19408 100644 --- a/plugins/sysbench/3.1.0/run_test.py +++ b/plugins/sysbench/3.1.0/run_test.py @@ -170,7 +170,7 @@ def run_test(plugin_context, db, cursor, odp_db, odp_cursor=None, *args, **kwarg tenant_variables_done = [] odp_configs = [ # [配置名, 新值, 旧值, 替换条件: lambda n, o: n != o] - ['enable_compression_protocol', False, False, lambda n, o: n != o], + # ['enable_compression_protocol', False, False, lambda n, o: n != o], ['proxy_mem_limited', format_size(min(max(threads * (8 << 10), 2 << 30), 4 << 30), 0), 0, lambda n, o: parse_size(n) > parse_size(o)], ['enable_prometheus', False, False, lambda n, o: n != o], ['enable_metadb_used', False, False, lambda n, o: n != o], diff --git a/plugins/tpcc/3.1.0/build.py b/plugins/tpcc/3.1.0/build.py new file mode 100644 index 0000000000000000000000000000000000000000..676bf2e8cb92e5938dda92a1f953c04f7379ae06 --- /dev/null +++ b/plugins/tpcc/3.1.0/build.py @@ -0,0 +1,182 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +try: + import subprocess32 as subprocess +except: + import subprocess +import os +import time +import re + +from ssh import LocalClient + + +def build(plugin_context, cursor, odp_cursor, *args, **kwargs): + def get_option(key, default=''): + value = getattr(options, key, default) + if value is None: + value = default + stdio.verbose('get option: {} value {}'.format(key, value)) + return value + + def local_execute_command(command, env=None, timeout=None): + return LocalClient.execute_command(command, env, timeout, stdio) + + def run_sql(sql_file, force=False): + sql_cmd = "{obclient} -h{host} -P{port} -u{user}@{tenant} {password_arg} -A {db} {force_flag} < {sql_file}".format( + obclient=obclient_bin, host=host, port=port, user=user, tenant=tenant_name, + password_arg=("-p'%s'" % password) if password else '', + db=db_name, + force_flag='-f' if force else '', + sql_file=sql_file) + return local_execute_command(sql_cmd) + + def get_table_rows(table_name): + table_rows = 0 + ret = local_execute_command('%s "%s" -E' % (exec_sql_cmd, 'select count(*) from %s' % table_name)) + matched = re.match(r'.*count\(\*\):\s?(\d+)', ret.stdout, re.S) + if matched: + table_rows = int(matched.group(1)) + return table_rows + + stdio = plugin_context.stdio + options = plugin_context.options + + bmsql_jar = get_option('bmsql_jar') + bmsql_libs = get_option('bmsql_libs') + + host = get_option('host', '127.0.0.1') + port = get_option('port', 2881) + db_name = get_option('database', 'test') + user = get_option('user', 'root') + password = get_option('password', '') + tenant_name = get_option('tenant', 'test') + obclient_bin = get_option('obclient_bin', 'obclient') + java_bin = get_option('java_bin', 'java') + + bmsql_classpath = kwargs.get('bmsql_classpath') + if not bmsql_classpath: + jars = [bmsql_jar] + jars.extend(bmsql_libs.split(',')) + bmsql_classpath = '.:' + ':'.join(jars) + bmsql_prop_path = kwargs.get('bmsql_prop_path') + stdio.verbose('get bmsql_prop_path: {}'.format(bmsql_prop_path)) + warehouses = kwargs.get('warehouses', 0) + + stdio.verbose('Check connect ready') + exec_sql_cmd = "%s -h%s -P%s -u%s@%s %s -A %s -e" % ( + obclient_bin, host, port, user, tenant_name, ("-p'%s'" % password) if password else '', db_name) + stdio.start_loading('Connect to tenant %s' % tenant_name) + try: + while True: + ret = local_execute_command('%s "%s" -E' % (exec_sql_cmd, 'select version();')) + if ret: + break + time.sleep(10) + stdio.stop_loading('succeed') + except: + stdio.stop_loading('fail') + stdio.exception('') + return + + # drop old tables + bmsql_sql_path = kwargs.get('bmsql_sql_path', '') + run_sql(sql_file=os.path.join(bmsql_sql_path, 'tableDrops.sql'), force=True) + + retries = 300 + pending_free_count = -1 + while pending_free_count != 0 and retries > 0: + retries -= 1 + sql = 'select pending_free_count from oceanbase.__all_virtual_macro_block_marker_status' + stdio.verbose('execute sql: %s' % sql) + cursor.execute(sql) + ret = cursor.fetchone() + stdio.verbose('sql result: %s' % ret) + pending_free_count = ret.get('pending_free_count', 0) if ret else 0 + time.sleep(1) + + # create new tables + if not run_sql(sql_file=os.path.join(bmsql_sql_path, 'tableCreates.sql')): + stdio.error('create tables failed') + return False + + # load data + stdio.verbose('Start to load data.') + cmd = '{java_bin} -cp {cp} -Dprop={prop} LoadData'.format(java_bin=java_bin, cp=bmsql_classpath, prop=bmsql_prop_path) + stdio.start_progressbar('Load data ', warehouses, widget_type='simple_progress') + try: + stdio.verbose('local execute: %s' % cmd) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + while p.poll() is None: + count = get_table_rows('bmsql_warehouse') + if count: + stdio.update_progressbar(min(count, warehouses - 1)) + time.sleep(10) + code = p.returncode + output = p.stdout.read().decode() + verbose_msg = 'exited code %s' % code + verbose_msg += ', output:\n%s' % output + except: + output = '' + code = 255 + verbose_msg = 'unknown error' + stdio.exception('') + stdio.verbose(verbose_msg) + if code != 0: + stdio.interrupt_progressbar() + stdio.error('Failed to load data.') + return + if re.match(r'.*Worker \d+: ERROR: .*', output, re.S): + stdio.interrupt_progressbar() + stdio.error('Failed to load data.') + return + stdio.finish_progressbar() + + # create index + stdio.start_loading('create index') + if not run_sql(sql_file=os.path.join(bmsql_sql_path, 'indexCreates.sql')): + stdio.error('create index failed') + stdio.stop_loading('fail') + return + stdio.stop_loading('succeed') + + # build finish + stdio.start_loading('finish build') + if not run_sql(sql_file=os.path.join(bmsql_sql_path, 'buildFinish.sql')): + stdio.error('finish build failed') + stdio.stop_loading('fail') + return + stdio.stop_loading('succeed') + + # check result + stdio.start_loading('check data') + try: + assert get_table_rows('bmsql_warehouse') == warehouses, Exception('warehouse num wrong') + assert get_table_rows('bmsql_district') == warehouses * 10, Exception('district num wrong') + stdio.stop_loading('succeed') + except Exception as e: + stdio.stop_loading('fail') + stdio.verbose(e) + stdio.error('check data failed.') + return + return plugin_context.return_true() diff --git a/plugins/tpcc/3.1.0/optimize.py b/plugins/tpcc/3.1.0/optimize.py new file mode 100644 index 0000000000000000000000000000000000000000..22c6bac6f2c91b1abe0a726bb5fc66a0474a3c17 --- /dev/null +++ b/plugins/tpcc/3.1.0/optimize.py @@ -0,0 +1,333 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# + + +from __future__ import absolute_import, division, print_function + +from time import sleep + +from ssh import LocalClient + + +def optimize(plugin_context, cursor, odp_cursor, *args, **kwargs): + def get_option(key, default=''): + value = getattr(options, key, default) + if value is None: + value = default + return value + + def execute(cursor, query, args=None): + msg = query % tuple(args) if args is not None else query + stdio.verbose('execute sql: %s' % msg) + stdio.verbose("query: %s. args: %s" % (query, args)) + try: + cursor.execute(query, args) + return cursor.fetchone() + except Exception: + msg = 'execute sql exception: %s' % msg + stdio.exception(msg) + raise Exception(msg) + + def local_execute_command(command, env=None, timeout=None): + return LocalClient.execute_command(command, env, timeout, stdio) + + def type_transform(expect, current): + expect_type = type(expect) + if isinstance(current, expect_type): + return expect, current + elif isinstance(current, (int, float)) or str(current).isdigit(): + return expect, expect_type(current) + else: + return str(expect).lower(), str(current).lower() + + global stdio + cluster_config = plugin_context.cluster_config + stdio = plugin_context.stdio + options = plugin_context.options + optimization = get_option('optimization') + ob_optimization = get_option('ob_optimization') + not_test_only = not get_option('test_only') + tenant_name = get_option('tenant', 'test') + host = get_option('host', '127.0.0.1') + port = get_option('port', 2881) + mysql_db = get_option('database', 'test') + user = get_option('user', 'root') + password = get_option('password', '') + obclient_bin = get_option('obclient_bin', 'obclient') + + optimization_step = kwargs.get('optimization_step', 'build') + success = False + stdio.start_loading('Optimize for %s' % ('performance' if optimization_step == 'test' else 'server')) + if optimization_step == 'test': + exec_sql_cmd = "%s -h%s -P%s -u%s@%s %s -A %s -e" % ( + obclient_bin, host, port, user, tenant_name, ("-p'%s'" % password) if password else '', mysql_db) + stdio.start_loading('Connect to tenant %s' % tenant_name) + try: + while True: + ret = local_execute_command('%s "%s" -E' % (exec_sql_cmd, 'select version();')) + if ret: + break + sleep(10) + stdio.stop_loading('succeed') + except: + stdio.stop_loading('fail') + stdio.exception('') + return + sql = "select * from oceanbase.gv$tenant where tenant_name = %s" + try: + stdio.verbose('execute sql: %s' % (sql % tenant_name)) + cursor.execute(sql, [tenant_name]) + tenant_meta = cursor.fetchone() + if not tenant_meta: + stdio.error('Tenant %s not exists. Use `obd cluster tenant create` to create tenant.' % tenant_name) + return + except Exception as e: + stdio.verbose(e) + stdio.error('fail to get tenant info') + stdio.stop_loading('fail') + return + + if not_test_only: + sql_cmd_prefix = '%s -h%s -P%s -u%s@%s %s -A' % ( + obclient_bin, host, port, user, tenant_name, ("-p'%s'" % password) if password else '') + ret = local_execute_command('%s -e "%s"' % (sql_cmd_prefix, 'create database if not exists %s' % mysql_db)) + sql_cmd_prefix += ' -D %s' % mysql_db + if not ret: + stdio.error(ret.stderr) + stdio.stop_loading('fail') + return + else: + sql_cmd_prefix = '%s -h%s -P%s -u%s@%s %s -D %s -A' % ( + obclient_bin, host, port, user, tenant_name, ("-p'%s'" % password) if password else '', mysql_db) + + ret = LocalClient.execute_command('%s -e "%s"' % (sql_cmd_prefix, 'select version();'), stdio=stdio) + if not ret: + stdio.error(ret.stderr) + stdio.stop_loading('fail') + return + + odp_configs_done = kwargs.get('odp_configs_done', []) + system_configs_done = kwargs.get('system_configs_done', []) + tenant_variables_done = kwargs.get('tenant_variables_done', []) + odp_need_reboot = False + obs_need_reboot = False + variables_count_before = len(odp_configs_done) + len(system_configs_done) + len(tenant_variables_done) + + odp_configs_build = [ + # [配置名, 新值, 旧值, 替换条件: lambda n, o: n != o, 重启生效] + # ['enable_strict_kernel_release', False, False, lambda n, o: n != o, True], + ['automatic_match_work_thread', False, False, lambda n, o: n != o, True], + ['proxy_mem_limited', '4G', '4G', lambda n, o: n != o, False], + ['enable_compression_protocol', False, False, lambda n, o: n != o, True], + ['slow_proxy_process_time_threshold', '500ms', '500ms', lambda n, o: n != o, False], + ['enable_ob_protocol_v2', False, False, lambda n, o: n != o, True], + ['enable_qos', False, False, lambda n, o: n != o, False], + ['syslog_level', 'error', 'error', lambda n, o: n != o, False], + ] + + system_configs_build = [ + # [配置名, 新值, 旧值, 替换条件: lambda n, o: n != o, 是否租户级, 重启生效] + ['memory_chunk_cache_size', '0', '0', lambda n, o: n != o, False, False], + ['trx_try_wait_lock_timeout', '0ms', '0ms', lambda n, o: n != o, False, False], + ['large_query_threshold', '1s', '1s', lambda n, o: n != o, False, False], + ['trace_log_slow_query_watermark', '500ms', '500ms', lambda n, o: n != o, False, False], + ['syslog_io_bandwidth_limit', '30m', '30m', lambda n, o: n != o, False, False], + ['enable_async_syslog', 'true', 'true', lambda n, o: n != o, False, False], + ['merger_warm_up_duration_time', 0, 0, lambda n, o: n != o, False, False], + ['merger_switch_leader_duration_time', 0, 0, lambda n, o: n != o, False, False], + ['large_query_worker_percentage', 10, 10, lambda n, o: n != o, False, False], + ['builtin_db_data_verify_cycle', 0, 0, lambda n, o: n != o, False, False], + ['enable_merge_by_turn', False, False, lambda n, o: n != o, False, False], + ['minor_merge_concurrency', 30, 30, lambda n, o: n != o, False, False], + ['memory_limit_percentage', 85, 85, lambda n, o: n != o, False, False], + ['memstore_limit_percentage', 80, 80, lambda n, o: n != o, False, False], + ['freeze_trigger_percentage', 60, 60, lambda n, o: n != o, False, False], + ['enable_syslog_recycle', True, True, lambda n, o: n != o, False, False], + ['max_syslog_file_count', 100, 100, lambda n, o: n != o, False, False], + ['minor_freeze_times', 500, 500, lambda n, o: n != o, False, False], + ['minor_compact_trigger', 0, 0, lambda n, o: n != o, False, False], + ['max_kept_major_version_number', 1, 1, lambda n, o: n != o, False, False], + ['sys_bkgd_io_high_percentage', 90, 90, lambda n, o: n != o, False, False], + ['sys_bkgd_io_low_percentage', 70, 70, lambda n, o: n != o, False, False], + ['merge_thread_count', 45, 45, lambda n, o: n != o, False, False], + ['merge_stat_sampling_ratio', 1, 1, lambda n, o: n != o, False, False], + ['writing_throttling_trigger_percentage', 75, 75, lambda n, o: n != o, True, False], + ['writing_throttling_maximum_duration', '15m', '15m', lambda n, o: n != o, False, False], + ['enable_sql_audit', 'false', 'false', lambda n, o: n != o, False, False], + ['_enable_clog_rpc_aggregation', 'true', 'true', lambda n, o: n != o, False, False], + ['enable_early_lock_release', 'false', 'false', lambda n, o: n != o, True, False], + ['enable_auto_leader_switch', 'false', 'false', lambda n, o: n != o, False, False], + ['clog_transport_compress_all', 'false', 'false', lambda n, o: n != o, False, False], + ['sleep', 2], + ['enable_perf_event', False, False, lambda n, o: n != o, False, False], + ['use_large_pages', 'true', 'true', lambda n, o: n != o, False, True, False], + ['micro_block_merge_verify_level', 0, 0, lambda n, o: n != o, False, False], + ['builtin_db_data_verify_cycle', 0, 0, lambda n, o: n != o, False, False], + ['net_thread_count', 6, 6, lambda n, o: n != o, False, True], + ['_clog_aggregation_buffer_amount', 4, 4, lambda n, o: n != o, True, False], + ['_flush_clog_aggregation_buffer_timeout', '2ms', '2ms', lambda n, o: n != o, True, False], + ] + + odp_configs_test = [] + + system_configs_test = [ + # [配置名, 新值, 旧值, 替换条件: lambda n, o: n != o, 是否租户级, 重启生效] + ['writing_throttling_trigger_percentage', 100, 100, lambda n, o: n != o, True, False], + ['writing_throttling_maximum_duration', '1h', '1h', lambda n, o: n != o, False, False], + ['memstore_limit_percentage', 80, 80, lambda n, o: n != o, False, False], + ['freeze_trigger_percentage', 30, 30, lambda n, o: n != o, False, False], + ['large_query_threshold', '200s', '200s', lambda n, o: n != o, False, False], + ['trx_try_wait_lock_timeout', '0ms', '0ms', lambda n, o: n != o, False, False], + ['cpu_quota_concurrency', 4, 4, lambda n, o: n != o, False, False], + ['minor_warm_up_duration_time', 0, 0, lambda n, o: n != o, False, False], + ['minor_freeze_times', 500, 500, lambda n, o: n != o, False, False], + ['minor_compact_trigger', 3, 3, lambda n, o: n != o, False, False], + ['sys_bkgd_io_high_percentage', 90, 90, lambda n, o: n != o, False, False], + ['sys_bkgd_io_low_percentage', 70, 70, lambda n, o: n != o, False, False], + ['minor_merge_concurrency', 20, 20, lambda n, o: n != o, False, False], + ['builtin_db_data_verify_cycle', 0, 0, lambda n, o: n != o, False, False], + ['trace_log_slow_query_watermark', '10s', '10s', lambda n, o: n != o, False, False], + ['gts_refresh_interval', '500us', '500us', lambda n, o: n != o, False, False], + ['server_permanent_offline_time', '36000s', '36000s', lambda n, o: n != o, False, False], + ['weak_read_version_refresh_interval', 0, 0, lambda n, o: n != o, False, False], + ['_ob_get_gts_ahead_interval', '1ms', '1ms', lambda n, o: n != o, False, False], + ['bf_cache_priority', 10, 10, lambda n, o: n != o, False, False], + ['user_block_cache_priority', 5, 5, lambda n, o: n != o, False, False], + ['merge_stat_sampling_ratio', 1, 1, lambda n, o: n != o, False, False], + ['enable_sql_audit', 'false', 'false', lambda n, o: n != o, False, False], + ['bf_cache_miss_count_threshold', 1, 1, lambda n, o: n != o, False, False], + ['__easy_memory_limit', '20G', '20G', lambda n, o: n != o, False, False], + ['_enable_defensive_check', 'false', 'false', lambda n, o: n != o, False, False], + ['binlog_row_image', 'MINIMAL', 'MINIMAL', lambda n, o: n != o, False, False], + ['sleep', 2], + ['syslog_level', 'PERF', 'PERF', lambda n, o: n != o, False, False], + ['max_syslog_file_count', 100, 100, lambda n, o: n != o, False, False], + ['enable_syslog_recycle', True, True, lambda n, o: n != o, False, False], + ['ob_enable_batched_multi_statement', True, True, lambda n, o: n != o, True, False], + ['_cache_wash_interval', '1m', '1m', lambda n, o: n != o, False, False], + ['cache_wash_threshold', '10G', '10G', lambda n, o: n != o, False, False], + ['plan_cache_evict_interval', '30s', '30s', lambda n, o: n != o, False, False], + ['enable_one_phase_commit', 'false', 'false', lambda n, o: n != o, False, False], + ['use_large_pages', 'true', 'true', lambda n, o: n != o, False, False], + ['enable_monotonic_weak_read', 'false', 'false', lambda n, o: n != o, False, False], + ] + + odp_configs = odp_configs_build if optimization_step == 'build' else odp_configs_test + system_configs = system_configs_build if optimization_step == 'build' else system_configs_test + + try: + if odp_cursor and optimization: + for config in odp_configs: + if not config[4] or optimization > 1: + sql = 'show proxyconfig like "%s"' % config[0] + ret = execute(odp_cursor, sql) + if ret: + if config[0] in ['syslog_level']: + config[2] = ret['level'] + else: + config[2] = ret['value'] + if config[3](*type_transform(config[1], config[2])): + sql = 'alter proxyconfig set %s=%%s' % config[0] + if config[4]: + odp_need_reboot = True + execute(odp_cursor, sql, [config[1]]) + odp_configs_done.append(config) + else: + stdio.verbose("proxy config %s not found, skip" % config[0]) + + tenant_q = ' tenant="%s"' % tenant_name + server_num = len(cluster_config.servers) + + if optimization and ob_optimization: + for config in system_configs: + if config[0] == 'sleep': + sleep(config[1]) + continue + if not config[5] or optimization > 1: + sql = 'select value from oceanbase.__all_virtual_sys_parameter_stat where name="%s"' % config[0] + if config[4]: + sql = 'select * from oceanbase.__all_virtual_tenant_parameter_info ' \ + 'where name like "%s" and tenant_id=%d' % (config[0], tenant_meta['tenant_id']) + ret = execute(cursor, sql) + if ret: + config[2] = ret['value'] + if config[3](*type_transform(config[1], config[2])): + sql = 'alter system set %s=%%s' % config[0] + if config[4]: + sql += tenant_q + if config[5]: + obs_need_reboot = True + execute(cursor, sql, [config[1]]) + system_configs_done.append(config) + else: + stdio.verbose("system parameter %s not found, skip" % config[0]) + + sql = "select count(1) server_num from oceanbase.__all_server where status = 'active'" + ret = execute(cursor, sql) + if ret: + server_num = ret.get("server_num", server_num) + + max_cpu = kwargs.get('cpu_total') + parallel_servers_target = int(max_cpu * server_num * 8) + + tenant_variables_build = [ + # [变量名, 新值, 旧值, 替换条件: lambda n, o: n != o] + ['ob_plan_cache_percentage', 20, 20, lambda n, o: n != o], + ['autocommit', 1, 1, lambda n, o: n != o], + ['ob_query_timeout', 36000000000, 36000000000, lambda n, o: n != o], + ['ob_trx_timeout', 36000000000, 36000000000, lambda n, o: n != o], + ['max_allowed_packet', 67108864, 67108864, lambda n, o: n != o], + ['ob_sql_work_area_percentage', 100, 100, lambda n, o: n != o], + ['parallel_servers_target', parallel_servers_target, parallel_servers_target, lambda n, o: n != o], + ] + tenant_variables_test = [] + tenant_variables = tenant_variables_build if optimization_step == 'build' else tenant_variables_test + + select_sql_t = "select value from oceanbase.__all_virtual_sys_variable where tenant_id = %d and name = '%%s'" % \ + tenant_meta['tenant_id'] + update_sql_t = "ALTER TENANT %s SET VARIABLES %%s = %%%%s" % tenant_name + + for config in tenant_variables: + sql = select_sql_t % config[0] + ret = execute(cursor, sql) + if ret: + value = ret['value'] + config[2] = int(value) if isinstance(value, str) and value.isdigit() else value + if config[3](*type_transform(config[1], config[2])): + sql = update_sql_t % config[0] + tenant_variables_done.append(config) + execute(cursor, sql, [config[1]]) + else: + stdio.verbose("tenant config %s not found, skip" % config[0]) + if len(odp_configs_done) + len(system_configs_done) + len(tenant_variables_done) - variables_count_before: + sleep(3) + + success = True + + except KeyboardInterrupt as e: + stdio.exception(e) + except Exception: + stdio.exception('') + finally: + if success: + stdio.stop_loading('succeed') + plugin_context.return_true( + odp_configs_done=odp_configs_done, + system_configs_done=system_configs_done, + tenant_variables_done=tenant_variables_done, + tenant_id=tenant_meta['tenant_id'], + odp_need_reboot=odp_need_reboot, + obs_need_reboot=obs_need_reboot + ) + else: + stdio.stop_loading('fail') + plugin_context.return_false( + odp_configs_done=odp_configs_done, + system_configs_done=system_configs_done, + tenant_variables_done=tenant_variables_done, + tenant_id=tenant_meta['tenant_id'], + odp_need_reboot=odp_need_reboot, + obs_need_reboot=obs_need_reboot + ) \ No newline at end of file diff --git a/plugins/tpcc/3.1.0/pre_test.py b/plugins/tpcc/3.1.0/pre_test.py new file mode 100644 index 0000000000000000000000000000000000000000..7a6d18a184e3a4de71ec530563b97f73c3d3580a --- /dev/null +++ b/plugins/tpcc/3.1.0/pre_test.py @@ -0,0 +1,282 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +import os +import re + +from ssh import LocalClient +from tool import DirectoryUtil + +PROPS4OB_TEMPLATE = """ +db=oceanbase +driver=com.mysql.jdbc.Driver +conn=jdbc:mysql://{host_ip}:{port}/{db_name}?rewriteBatchedStatements=true&allowMultiQueries=true&useLocalSessionState=true&useUnicode=true&characterEncoding=utf-8&socketTimeout=30000000&useSSL=false +user={user} +password={password} +warehouses={warehouses} +loadWorkers={load_workers} +terminals={terminals} +database={db_name} +runTxnsPerTerminal=0 +runMins={run_mins} +limitTxnsPerMin=0 +terminalWarehouseFixed=true +newOrderWeight=45 +paymentWeight=43 +orderStatusWeight=4 +deliveryWeight=4 +stockLevelWeight=4 +resultDirectory=my_result_%tY-%tm-%td_%tH%tM%tS +osCollectorScript=./misc/os_collector_linux.py +osCollectorInterval=1 +""" + + +def pre_test(plugin_context, cursor, odp_cursor, *args, **kwargs): + def get_option(key, default=''): + value = getattr(options, key, default) + if value is None: + value = default + stdio.verbose('get option: {} value {}'.format(key, value)) + return value + + def execute(cursor, query, args=None): + msg = query % tuple(args) if args is not None else query + stdio.verbose('execute sql: %s' % msg) + stdio.verbose("query: %s. args: %s" % (query, args)) + try: + cursor.execute(query, args) + return cursor.fetchone() + except: + msg = 'execute sql exception: %s' % msg + stdio.exception(msg) + raise Exception(msg) + + def local_execute_command(command, env=None, timeout=None): + return LocalClient.execute_command(command, env, timeout, stdio) + + stdio = plugin_context.stdio + options = plugin_context.options + + tmp_dir = os.path.abspath(get_option('tmp_dir', './tmp')) + tenant_name = get_option('tenant', 'test') + + if tenant_name == 'sys': + stdio.error('DO NOT use sys tenant for testing.') + return + + bmsql_path = get_option('bmsql_dir') + bmsql_jar = get_option('bmsql_jar', None) + bmsql_libs = get_option('bmsql_libs', None) + bmsql_sql_path = get_option('bmsql_sql_dir') + if bmsql_path: + if bmsql_jar is None: + bmsql_jar = os.path.join(bmsql_path, 'dist') if bmsql_path else '/usr/ob-benchmarksql/OB-BenchmarkSQL-5.0.jar' + if bmsql_libs is None: + bmsql_libs = '%s,%s' % (os.path.join(bmsql_path, 'lib'), os.path.join(bmsql_path, 'lib/oceanbase')) + else: + if bmsql_jar is None: + bmsql_jar = '/usr/ob-benchmarksql/OB-BenchmarkSQL-5.0.jar' + + if not os.path.exists(tmp_dir) and not DirectoryUtil.mkdir(tmp_dir): + stdio.error('Create tmp dir failed') + return + + if not os.path.exists(bmsql_jar): + stdio.error( + 'BenchmarkSQL jar file not found at %s. Please use `--bmsql-jar` to set BenchmarkSQL jar file' % bmsql_jar) + return + + jars = [os.path.join(bmsql_jar, '*') if os.path.isdir(bmsql_jar) else bmsql_jar] + if bmsql_libs: + for lib in bmsql_libs.split(','): + if lib: + if os.path.isdir(lib): + jars.append(os.path.join(lib, '*')) + else: + jars.append(lib) + bmsql_classpath = ':'.join(jars) + + obclient_bin = get_option('obclient_bin', 'obclient') + ret = LocalClient.execute_command('%s --help' % obclient_bin, stdio=stdio) + if not ret: + stdio.error( + '%s\n%s is not an executable file. please use `--obclient-bin` to set.\nYou may not have obclient installed' % ( + ret.stderr, obclient_bin)) + return + + java_bin = get_option('java_bin', 'java') + ret = local_execute_command('{java_bin} -version'.format(java_bin=java_bin)) + if not ret: + stdio.error( + '%s\n%s is not an executable file. please use `--java-bin` to set.\nYou may not have java installed' % ( + ret.stderr, java_bin)) + return + exec_classes = ['jTPCC', 'LoadData', 'ExecJDBC'] + passed = True + for exec_class in exec_classes: + ret = local_execute_command('%s -cp %s %s' % (java_bin, bmsql_classpath, exec_class)) + if 'Could not find or load main class %s' % exec_class in ret.stderr: + stdio.error('Main class %s not found.' % exec_class) + passed = False + if not passed: + stdio.error('Please use `--bmsql-libs` to infer all the depends') + return + + local_dir = os.path.dirname(os.path.abspath(__file__)) + run_path = os.path.join(tmp_dir, 'run') + if not DirectoryUtil.copy(os.path.join(local_dir, 'run'), run_path, stdio): + return + + stdio.verbose('Start to get bmsql sqls...') + if bmsql_sql_path: + miss_sql = [] + for sql_file in ['buildFinish.sql', 'indexCreates.sql', 'indexDrops.sql', 'tableCreates.sql', 'tableDrops.sql']: + if not os.path.exists(os.path.join(bmsql_sql_path, sql_file)): + miss_sql.append(sql_file) + if miss_sql: + stdio.error('Cannot find %s in scripts path %s.' % (','.join(miss_sql), bmsql_sql_path)) + stdio.stop_loading('fail') + return + + cpu_total = 0 + min_cpu = None + try: + sql = "select a.id , b.cpu_total from oceanbase.__all_server a " \ + "join oceanbase.__all_virtual_server_stat b on a.id=b.id " \ + "where a.status = 'active' and a.stop_time = 0 and a.start_service_time > 0;" + stdio.verbose('execute sql: %s' % sql) + cursor.execute(sql) + all_services = cursor.fetchall() + if not all_services: + stdio.error('No active server available.') + return + for serv in all_services: + cpu_count = int(serv.get('cpu_total', 0) + 2) + min_cpu = cpu_count if min_cpu is None else min(cpu_count, min_cpu) + cpu_total += cpu_count + except Exception as e: + stdio.exception(e) + stdio.error('fail to get server status') + return + + stdio.verbose('cpu total in all servers is %d' % cpu_total) + if not bmsql_sql_path: + bmsql_sql_path = os.path.join(tmp_dir, 'sql.oceanbase') + if not DirectoryUtil.copy(os.path.join(local_dir, 'sql.oceanbase'), bmsql_sql_path, stdio): + return + create_table_sql = os.path.join(bmsql_sql_path, 'tableCreates.sql') + local_execute_command("sed -i 's/{{partition_num}}/%d/g' %s" % (cpu_total, create_table_sql)) + + sql = "select * from oceanbase.gv$tenant where tenant_name = %s" + try: + stdio.verbose('execute sql: %s' % (sql % tenant_name)) + cursor.execute(sql, [tenant_name]) + tenant_meta = cursor.fetchone() + if not tenant_meta: + stdio.error('Tenant %s not exists. Use `obd cluster tenant create` to create tenant.' % tenant_name) + return + sql = "select * from oceanbase.__all_resource_pool where tenant_id = %d" % tenant_meta['tenant_id'] + pool = execute(cursor, sql) + sql = "select * from oceanbase.__all_unit_config where unit_config_id = %d" % pool['unit_config_id'] + tenant_unit = execute(cursor, sql) + max_memory = tenant_unit['max_memory'] + max_cpu = int(tenant_unit['max_cpu']) + except Exception as e: + stdio.verbose(e) + stdio.error('fail to get tenant info') + return + + host = get_option('host', '127.0.0.1') + port = get_option('port', 2881) + db_name = get_option('database', 'test') + user = get_option('user', 'root') + password = get_option('password', '') + warehouses = get_option('warehouses', cpu_total * 20) + load_workers = get_option('load_workers', int(min(min_cpu, (max_memory >> 30) / 2))) + terminals = get_option('terminals', min(cpu_total * 15, warehouses * 10)) + run_mins = get_option('run_mins', 10) + test_only = get_option('test_only') + + stdio.verbose('Check connect ready') + exec_sql_cmd = "%s -h%s -P%s -u%s@%s %s -A %s -e" % ( + obclient_bin, host, port, user, tenant_name, ("-p'%s'" % password) if password else '', db_name) + ret = local_execute_command('%s "%s" -E' % (exec_sql_cmd, 'select version();')) + if not ret: + stdio.error('Connect to tenant %s failed' % tenant_name) + return + + if warehouses <= 0: + stdio.error('warehouses should more than 0') + return + if terminals <= 0 or terminals > 10 * warehouses: + stdio.error('terminals should more than 0 and less than 10 * warehouses') + return + if run_mins <= 0: + stdio.error('run-mins should more than 0') + return + if test_only: + exec_sql_cmd = "%s -h%s -P%s -u%s@%s %s -A %s -e" % ( + obclient_bin, host, port, user, tenant_name, ("-p'%s'" % password) if password else '', db_name) + table_rows = 0 + ret = local_execute_command('%s "%s" -E' % (exec_sql_cmd, 'select count(*) from bmsql_warehouse')) + matched = re.match(r'.*count\(\*\):\s?(\d+)', ret.stdout, re.S) + if matched: + table_rows = int(matched.group(1)) + if table_rows <= 0: + stdio.error('No warehouse found. Please load data first.') + return + elif table_rows != warehouses: + stdio.error('Warehouse num do not match. Expect: {} ,actual: {}'.format(warehouses, table_rows)) + return + try: + bmsql_prop_path = os.path.join(tmp_dir, 'props.oceanbase') + stdio.verbose('set bmsql_prop_path: {}'.format(bmsql_prop_path)) + with open(bmsql_prop_path, 'w') as f: + f.write(PROPS4OB_TEMPLATE.format( + host_ip=host, + port=port, + db_name=db_name, + user=user + '@' + tenant_name, + password=password, + warehouses=warehouses, + load_workers=load_workers, + terminals=terminals, + run_mins=run_mins + )) + except Exception as e: + stdio.exception(e) + stdio.error('Failed to generate config file props.oceanbase.') + stdio.stop_loading('fail') + return + + stdio.stop_loading('succeed') + return plugin_context.return_true( + bmsql_prop_path=bmsql_prop_path, + bmsql_classpath=bmsql_classpath, + run_path=run_path, + bmsql_sql_path=bmsql_sql_path, + warehouses=warehouses, + cpu_total=cpu_total, + max_memory=max_memory, + max_cpu=max_cpu + ) diff --git a/plugins/tpcc/3.1.0/recover.py b/plugins/tpcc/3.1.0/recover.py new file mode 100644 index 0000000000000000000000000000000000000000..21db2ffe4118594c358a3f34ad41f1ac1424f7fd --- /dev/null +++ b/plugins/tpcc/3.1.0/recover.py @@ -0,0 +1,90 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +from time import sleep + + +def recover(plugin_context, cursor, odp_cursor, *args, **kwargs): + def get_option(key, default=''): + value = getattr(options, key, default) + if value is None: + value = default + return value + + def execute(cursor, query, args=None): + msg = query % tuple(args) if args is not None else query + stdio.verbose('execute sql: %s' % msg) + stdio.verbose("query: %s. args: %s" % (query, args)) + try: + cursor.execute(query, args) + return cursor.fetchone() + except: + msg = 'execute sql exception: %s' % msg + stdio.exception(msg) + raise Exception(msg) + + global stdio + stdio = plugin_context.stdio + options = plugin_context.options + optimization = get_option('optimization') > 0 + tenant_name = get_option('tenant', 'test') + + tenant_variables_done = kwargs.get('tenant_variables_done', []) + system_configs_done = kwargs.get('system_configs_done', []) + odp_configs_done = kwargs.get('odp_configs_done', []) + tenant_id = kwargs.get('tenant_id') + stdio.verbose(cursor) + stdio.verbose(vars(cursor)) + if optimization: + stdio.start_loading('Recover') + update_sql_t = "ALTER TENANT %s SET VARIABLES %%s = %%%%s" % tenant_name + tenant_q = ' tenant="%s"' % tenant_name + if not tenant_id: + sql = "select * from oceanbase.gv$tenant where tenant_name = %s" + stdio.verbose('execute sql: %s' % (sql % tenant_name)) + cursor.execute(sql, [tenant_name]) + tenant_meta = cursor.fetchone() + if not tenant_meta: + stdio.error('Tenant %s not exists. Use `obd cluster tenant create` to create tenant.' % tenant_name) + return + + for config in tenant_variables_done[::-1]: + if config[3](config[1], config[2]): + sql = update_sql_t % config[0] + execute(cursor, sql, [config[2]]) + for config in system_configs_done[::-1]: + if config[0] == 'sleep': + sleep(config[1]) + continue + if config[3](config[1], config[2]): + sql = 'alter system set %s=%%s' % config[0] + if config[4]: + sql += tenant_q + execute(cursor, sql, [config[2]]) + + if odp_cursor: + for config in odp_configs_done[::-1]: + if config[3](config[1], config[2]): + sql = 'alter proxyconfig set %s=%%s' % config[0] + execute(odp_cursor, sql, [config[2]]) + stdio.stop_loading('succeed') + return plugin_context.return_true() diff --git a/plugins/tpcc/3.1.0/run/log4j.properties b/plugins/tpcc/3.1.0/run/log4j.properties new file mode 100644 index 0000000000000000000000000000000000000000..68d6db5c58bea94bc0d8099f70c34026a206dfcb --- /dev/null +++ b/plugins/tpcc/3.1.0/run/log4j.properties @@ -0,0 +1,22 @@ +# log4j.rootLogger=TRACE, CONSOLE, E, T +log4j.rootLogger=INFO, CONSOLE, E + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern= %d{HH:mm:ss,SSS} [%t] %-5p %x %C{1} : %m%n + +log4j.appender.E=org.apache.log4j.RollingFileAppender +log4j.appender.E.Threshold=WARN +log4j.appender.E.File=benchmarksql-error.log +log4j.appender.E.MaxFileSize=100MB +log4j.appender.E.MaxBackupIndex=1 +log4j.appender.E.layout=org.apache.log4j.PatternLayout +log4j.appender.E.layout.ConversionPattern= %d{HH:mm:ss,SSS} [%t] %-5p %x %C{1} : %m%n + +log4j.appender.T=org.apache.log4j.FileAppender +log4j.appender.T.Threshold=TRACE +log4j.appender.T.File=benchmarksql-trace.log +log4j.appender.T.append=false +log4j.appender.T.layout=org.apache.log4j.PatternLayout +log4j.appender.T.layout.ConversionPattern= %d{HH:mm:ss,SSS} [%t] %-5p %x %C{1} : %m%n diff --git a/plugins/tpcc/3.1.0/run/misc/os_collector_linux.py b/plugins/tpcc/3.1.0/run/misc/os_collector_linux.py new file mode 100644 index 0000000000000000000000000000000000000000..0a40e43a4416eb79630b653d3a25c6d6bc52b07c --- /dev/null +++ b/plugins/tpcc/3.1.0/run/misc/os_collector_linux.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python +# ---------------------------------------------------------------------- +# os_collector_linux.py - +# +# Script used to collect OS level resource utilization data like +# CPU usage and disk IO. +# +# This code is used in the jTPCCOSCollect class. It is launched as +# a separate process, possibly via ssh(1) on the remote database +# server. The ability of Python to receive a script to execute on +# stdin allows us to execute this script via ssh(1) on the database +# server without installing any programs/scripts there. +# +# The command line arguments for this script are the runID, the +# interval in seconds at which to collect information and a variable +# number of devices in the form "blk_" "net_", +# for example "blk_sda" for the first SCSI disk or "net_eth0". +# +# The output on stdout is one line for CPU/VM info, followed by a +# line for each of the specified devices in CSV format. The first +# set of lines are the CSV headers. The output is prefixed with the +# runID, elapsed_ms and for the devices the blk_ or net_ name that +# was specified on the command line. This format makes it easy to +# load the data into a result database where it can be analyzed +# together with the BenchmarkSQL per transaction results and compared +# to other benchmark runs. +# +# It is the caller's responsibility to split the output lines into +# separate result CSV files. +# ---------------------------------------------------------------------- + +import errno +import math +import os +import sys +import time + + +# ---- +# main +# ---- +def main(argv): + global deviceFDs + global lastDeviceData + + # ---- + # Get the runID and collection interval from the command line + # ---- + runID = (int)(argv[0]) + interval = (float)(argv[1]) + + # ---- + # Our start time is now. Since most of the information is deltas + # we can only produce the first data after the first interval. + # ---- + startTime = time.time() + nextDue = startTime + interval + + # ---- + # Initialize CPU and vmstat collection and output the CSV header. + # ---- + sysInfo = ['run', 'elapsed', ] + sysInfo += initSystemUsage() + print(",".join([str(x) for x in sysInfo])) + + # ---- + # Get all the devices from the command line. + # ---- + devices = [] + deviceFDs = {} + lastDeviceData = {} + for dev in argv[2:]: + if dev.startswith('blk_'): + devices.append(dev) + elif dev.startswith('net_'): + devices.append(dev) + else: + raise Exception("unknown device type '" + dev + "'") + print('devices') + print(devices) + # ---- + # Initialize usage collection per device depending on the type. + # Output all the headers in the order, the devices are given. + # ---- + for dev in devices: + if dev.startswith('blk_'): + devInfo = ['run', 'elapsed', 'device', ] + devInfo += initBlockDevice(dev) + print(",".join([str(x) for x in devInfo])) + elif dev.startswith('net_'): + devInfo = ['run', 'elapsed', 'device', ] + devInfo += initNetDevice(dev) + print(",".join([str(x) for x in devInfo])) + + # ---- + # Flush all header lines. + # ---- + sys.stdout.flush() + + try: + while True: + # ---- + # Wait until our next collection interval and calculate the + # elapsed time in milliseconds. + # ---- + now = time.time() + if nextDue > now: + time.sleep(nextDue - now) + elapsed = (int)((nextDue - startTime) * 1000.0) + + # ---- + # Collect CPU and vmstat information. + # ---- + sysInfo = [runID, elapsed, ] + sysInfo += getSystemUsage() + print(",".join([str(x) for x in sysInfo])) + + # ---- + # Collect all device utilization data. + # ---- + for dev in devices: + if dev.startswith('blk_'): + devInfo = [runID, elapsed, dev, ] + devInfo += getBlockUsage(dev, interval) + print(",".join([str(x) for x in devInfo])) + elif dev.startswith('net_'): + devInfo = [runID, elapsed, dev, ] + devInfo += getNetUsage(dev, interval) + print(",".join([str(x) for x in devInfo])) + + # ---- + # Bump the time when we are next due. + # ---- + nextDue += interval + + sys.stdout.flush() + + # ---- + # Running on the command line for test purposes? + # ---- + except KeyboardInterrupt: + print("") + return 0 + + # ---- + # The OSCollector class will just close our stdout on the other + # side, so this is expected. + # ---- + except IOError as e: + if e.errno == errno.EPIPE: + return 0 + else: + raise e + + +def initSystemUsage(): + global procStatFD + global procVMStatFD + global lastStatData + global lastVMStatData + + procStatFD = open("/proc/stat", "rb", buffering=0) + for line in procStatFD: + line = line.decode().split() + if line[0] == "cpu": + lastStatData = [int(x) for x in line[1:]] + break + if len(lastStatData) != 10: + raise Exception("cpu line in /proc/stat too short"); + + procVMStatFD = open("/proc/vmstat", "rb", buffering=0) + lastVMStatData = {} + for line in procVMStatFD: + line = line.decode().split() + if line[0] in ['nr_dirty', ]: + lastVMStatData['vm_' + line[0]] = int(line[1]) + if len(lastVMStatData.keys()) != 1: + raise Exception("not all elements found in /proc/vmstat") + + return [ + 'cpu_user', 'cpu_nice', 'cpu_system', + 'cpu_idle', 'cpu_iowait', 'cpu_irq', + 'cpu_softirq', 'cpu_steal', + 'cpu_guest', 'cpu_guest_nice', + 'vm_nr_dirty', + ] + + +def getSystemUsage(): + global procStatFD + global procVMStatFD + global lastStatData + global lastVMStatData + + procStatFD.seek(0, 0) + for line in procStatFD: + line = line.decode().split() + if line[0] != "cpu": + continue + statData = [int(x) for x in line[1:]] + deltaTotal = (float)(sum(statData) - sum(lastStatData)) + if deltaTotal == 0: + result = [0.0 for x in statData] + else: + result = [] + for old, new in zip(lastStatData, statData): + result.append((float)(new - old) / deltaTotal) + procStatLast = statData + break + + procVMStatFD.seek(0, 0) + newVMStatData = {} + for line in procVMStatFD: + line = line.decode().split() + if line[0] in ['nr_dirty', ]: + newVMStatData['vm_' + line[0]] = int(line[1]) + + for key in ['vm_nr_dirty', ]: + result.append(newVMStatData[key]) + + return result + + +def initBlockDevice(dev): + global deviceFDs + global lastDeviceData + + devPath = os.path.join("/sys/block", dev[4:], "stat") + deviceFDs[dev] = open(devPath, "rb", buffering=0) + line = deviceFDs[dev].readline().decode().split() + + newData = [] + for idx, mult in [ + (0, 1.0), (1, 1.0), (2, 0.5), + (4, 1.0), (5, 1.0), (6, 0.5), + ]: + newData.append((int)(line[idx])) + lastDeviceData[dev] = newData + + return ['rdiops', 'rdmerges', 'rdkbps', 'wriops', 'wrmerges', 'wrkbps', ] + + +def getBlockUsage(dev, interval): + global deviceFDs + + deviceFDs[dev].seek(0, 0) + line = deviceFDs[dev].readline().split() + + oldData = lastDeviceData[dev] + newData = [] + result = [] + ridx = 0 + for idx, mult in [ + (0, 1.0), (1, 1.0), (2, 0.5), + (4, 1.0), (5, 1.0), (6, 0.5), + ]: + newData.append((int)(line[idx])) + result.append((float)(newData[ridx] - oldData[ridx]) * mult / interval) + ridx += 1 + lastDeviceData[dev] = newData + return result + + +def initNetDevice(dev): + global deviceFDs + global lastDeviceData + + devPath = os.path.join("/sys/class/net", dev[4:], "statistics") + deviceData = [] + for fname in ['rx_packets', 'rx_bytes', 'tx_packets', 'tx_bytes', ]: + key = dev + "." + fname + deviceFDs[key] = open(os.path.join(devPath, fname), + "rb", buffering=0) + deviceData.append((int)(deviceFDs[key].read().decode())) + + lastDeviceData[dev] = deviceData + + return ['rxpktsps', 'rxkbps', 'txpktsps', 'txkbps', ] + + +def getNetUsage(dev, interval): + global deviceFDs + global lastDeviceData + + oldData = lastDeviceData[dev] + newData = [] + for fname in ['rx_packets', 'rx_bytes', 'tx_packets', 'tx_bytes', ]: + key = dev + "." + fname + deviceFDs[key].seek(0, 0) + newData.append((int)(deviceFDs[key].read())) + + result = [ + (float)(newData[0] - oldData[0]) / interval, + (float)(newData[1] - oldData[1]) / interval / 1024.0, + (float)(newData[2] - oldData[2]) / interval, + (float)(newData[3] - oldData[3]) / interval / 1024.0, + ] + lastDeviceData[dev] = newData + return result + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/plugins/tpcc/3.1.0/run_test.py b/plugins/tpcc/3.1.0/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..cb8b8e8f96d451baf13303a6513c1c57450dd874 --- /dev/null +++ b/plugins/tpcc/3.1.0/run_test.py @@ -0,0 +1,183 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + + +from __future__ import absolute_import, division, print_function + +import datetime +import os +import re +import time + +try: + import subprocess32 as subprocess +except: + import subprocess + +from ssh import LocalClient + +stdio = None + + +def run_test(plugin_context, cursor, odp_cursor=None, *args, **kwargs): + def get_option(key, default=''): + value = getattr(options, key, default) + if value is None: + value = default + return value + + def execute(cursor, query, args=None): + msg = query % tuple(args) if args is not None else query + stdio.verbose('execute sql: %s' % msg) + stdio.verbose("query: %s. args: %s" % (query, args)) + try: + cursor.execute(query, args) + return cursor.fetchone() + except: + msg = 'execute sql exception: %s' % msg + stdio.exception(msg) + raise Exception(msg) + + def local_execute_command(command, env=None, timeout=None): + return LocalClient.execute_command(command, env, timeout, stdio) + + global stdio + stdio = plugin_context.stdio + options = plugin_context.options + bmsql_jar = get_option('bmsql_jar') + bmsql_libs = get_option('bmsql_libs') + bmsql_classpath = kwargs.get('bmsql_classpath') + if not bmsql_classpath: + jars = [bmsql_jar] + jars.extend(bmsql_libs.split(',')) + bmsql_classpath = ':'.join(jars) + bmsql_prop_path = kwargs.get('bmsql_prop_path') + stdio.verbose('get bmsql_prop_path: {}'.format(bmsql_prop_path)) + run_path = kwargs.get('run_path') + host = get_option('host', '127.0.0.1') + port = get_option('port', 2881) + db_name = get_option('database', 'test') + user = get_option('user', 'root') + password = get_option('password', '') + tenant_name = get_option('tenant', 'test') + obclient_bin = get_option('obclient_bin', 'obclient') + run_mins = get_option('run_mins', 10) + java_bin = get_option('java_bin', 'java') + + stdio.verbose('Check connect ready') + exec_sql_cmd = "%s -h%s -P%s -u%s@%s %s -A %s -e" % ( + obclient_bin, host, port, user, tenant_name, ("-p'%s'" % password) if password else '', db_name) + stdio.start_loading('Connect to tenant %s' % tenant_name) + try: + while True: + ret = local_execute_command('%s "%s" -E' % (exec_sql_cmd, 'select version();')) + if ret: + break + time.sleep(10) + stdio.stop_loading('succeed') + except: + stdio.stop_loading('fail') + stdio.exception('') + return + + merge_version = execute(cursor, "select value from oceanbase.__all_zone where name='frozen_version'")['value'] + stdio.start_loading('Merge') + execute(cursor, 'alter system major freeze') + sql = "select value from oceanbase.__all_zone where name='frozen_version' and value != %s" % merge_version + while True: + if execute(cursor, sql): + break + time.sleep(1) + + while True: + if not execute(cursor, """select * from oceanbase.__all_zone + where name='last_merged_version' + and value != (select value from oceanbase.__all_zone where name='frozen_version' limit 1) + and zone in (select zone from oceanbase.__all_zone where name='status' and info = 'ACTIVE') + """): + break + time.sleep(5) + stdio.stop_loading('succeed') + + stdio.verbose('Benchmark run') + seq_file = os.path.join(run_path, '.jTPCC_run_seq.dat') + try: + with open(seq_file) as f: + seq = int(f.read()) + except Exception as e: + stdio.verbose(e) + seq = 0 + seq += 1 + with open(seq_file, 'w') as f: + f.write(str(seq)) + log_path = os.path.join(run_path, 'tpcc_out_{}_{}'.format(seq, datetime.datetime.now().strftime('%Y%m%d%H%M%S'))) + cmd = '{java_bin} -cp {cp} -Dprop={prop} -DrunID={seq} jTPCC'.format( + java_bin=java_bin, + run_path=run_path, + cp=bmsql_classpath, prop=bmsql_prop_path, seq=seq) + try: + stdio.verbose('local execute: %s' % cmd) + with open(log_path, 'wb', 0) as logf: + p = subprocess.Popen(cmd, shell=True, stdout=logf, stderr=subprocess.STDOUT, cwd=run_path) + stdio.start_loading('Benchmark run') + start_time = datetime.datetime.now() + timeout = datetime.timedelta(seconds=int(run_mins * 60 * 1.2)) + while p.poll() is None: + time.sleep(1) + ret = local_execute_command("tail -c 1000 %s" % log_path) + if ret: + stdio.update_loading_text(ret.stdout.strip('\b\r\n ').split('\n')[-1].split('\b')[-1].strip()) + if datetime.datetime.now() - start_time > timeout: + p.terminate() + stdio.verbose('Run benchmark sql timeout.') + stdio.error('Failed to run TPC-C benchmark.') + stdio.verbose('return code: {}'.format(p.returncode)) + with open(log_path, 'r') as f: + out = f.read() + stdio.verbose('output: {}'.format(out)) + return + stdio.update_loading_text('Benchmark run') + code = p.returncode + with open(log_path, 'r') as f: + out = f.read() + verbose_msg = 'exited code %s' % code + if code: + verbose_msg += ', output: %s' % out + stdio.verbose(verbose_msg) + stdio.error('Failed to run TPC-C benchmark.') + stdio.stop_loading('fail') + return + stdio.verbose('stdout: %s' % out) + output = 'TPC-C Result\n' + for k in [r'Measured tpmC \(NewOrders\)', r'Measured tpmTOTAL', r'Session Start', r'Session End', + r'Transaction Count']: + matched = re.match(r'.*(%s)\s+=\s+(.*?)\n' % k, out, re.S) + if not matched: + stdio.error('Failed to run TPC-C benchmark.') + return + output += '{} : {}\n'.format(matched.group(1), matched.group(2)) + stdio.print(output) + stdio.stop_loading('succeed') + return plugin_context.return_true() + except Exception as e: + error = str(e) + verbose_msg = 'exited code 255, error output:\n%s' % error + stdio.verbose(verbose_msg) + stdio.exception('') + stdio.stop_loading('fail') diff --git a/plugins/tpcc/3.1.0/sql.oceanbase/buildFinish.sql b/plugins/tpcc/3.1.0/sql.oceanbase/buildFinish.sql new file mode 100644 index 0000000000000000000000000000000000000000..19b4eac8086f198cff9e0ffbebe5bbd19660ad2d --- /dev/null +++ b/plugins/tpcc/3.1.0/sql.oceanbase/buildFinish.sql @@ -0,0 +1,4 @@ +-- ---- +-- Extra commands to run after the tables are created, loaded, +-- indexes built and extra's created. +-- ---- diff --git a/plugins/tpcc/3.1.0/sql.oceanbase/indexCreates.sql b/plugins/tpcc/3.1.0/sql.oceanbase/indexCreates.sql new file mode 100644 index 0000000000000000000000000000000000000000..468fdaec1227fb3c8be02b07da33f8c93b00560c --- /dev/null +++ b/plugins/tpcc/3.1.0/sql.oceanbase/indexCreates.sql @@ -0,0 +1,2 @@ +create index bmsql_customer_idx1 on bmsql_customer (c_w_id, c_d_id, c_last, c_first) local; +create index bmsql_oorder_idx1 on bmsql_oorder (o_w_id, o_d_id, o_carrier_id, o_id) local; \ No newline at end of file diff --git a/plugins/tpcc/3.1.0/sql.oceanbase/indexDrops.sql b/plugins/tpcc/3.1.0/sql.oceanbase/indexDrops.sql new file mode 100644 index 0000000000000000000000000000000000000000..5ebfa94fbedf12a8e9e1851af7cb17a0dabebe4a --- /dev/null +++ b/plugins/tpcc/3.1.0/sql.oceanbase/indexDrops.sql @@ -0,0 +1,2 @@ +alter table bmsql_customer drop index bmsql_customer_idx1; +alter table bmsql_oorder drop index bmsql_oorder_idx1; \ No newline at end of file diff --git a/plugins/tpcc/3.1.0/sql.oceanbase/tableCreates.sql b/plugins/tpcc/3.1.0/sql.oceanbase/tableCreates.sql new file mode 100644 index 0000000000000000000000000000000000000000..a239642a30a97458cb27d2fa9e9abec06d045fac --- /dev/null +++ b/plugins/tpcc/3.1.0/sql.oceanbase/tableCreates.sql @@ -0,0 +1,135 @@ +create table bmsql_config ( + cfg_name varchar(30) primary key, + cfg_value varchar(50) +); + +create tablegroup if not exists tpcc_group binding true partition by hash partitions {{partition_num}}; + +create table bmsql_warehouse ( + w_id integer not null, + w_ytd decimal(12,2), + w_tax decimal(4,4), + w_name varchar(10), + w_street_1 varchar(20), + w_street_2 varchar(20), + w_city varchar(20), + w_state char(2), + w_zip char(9), + primary key(w_id) +)tablegroup='tpcc_group' partition by hash(w_id) partitions {{partition_num}}; + +create table bmsql_district ( + d_w_id integer not null, + d_id integer not null, + d_ytd decimal(12,2), + d_tax decimal(4,4), + d_next_o_id integer, + d_name varchar(10), + d_street_1 varchar(20), + d_street_2 varchar(20), + d_city varchar(20), + d_state char(2), + d_zip char(9), + PRIMARY KEY (d_w_id, d_id) +)tablegroup='tpcc_group' partition by hash(d_w_id) partitions {{partition_num}}; + +create table bmsql_customer ( + c_w_id integer not null, + c_d_id integer not null, + c_id integer not null, + c_discount decimal(4,4), + c_credit char(2), + c_last varchar(16), + c_first varchar(16), + c_credit_lim decimal(12,2), + c_balance decimal(12,2), + c_ytd_payment decimal(12,2), + c_payment_cnt integer, + c_delivery_cnt integer, + c_street_1 varchar(20), + c_street_2 varchar(20), + c_city varchar(20), + c_state char(2), + c_zip char(9), + c_phone char(16), + c_since timestamp, + c_middle char(2), + c_data varchar(500), + PRIMARY KEY (c_w_id, c_d_id, c_id) +)tablegroup='tpcc_group' partition by hash(c_w_id) partitions {{partition_num}}; + + +create table bmsql_history ( + hist_id integer AUTO_INCREMENT, + h_c_id integer, + h_c_d_id integer, + h_c_w_id integer, + h_d_id integer, + h_w_id integer, + h_date timestamp, + h_amount decimal(6,2), + h_data varchar(24) +)tablegroup='tpcc_group' partition by hash(h_w_id) partitions {{partition_num}}; + +create table bmsql_new_order ( + no_w_id integer not null , + no_d_id integer not null, + no_o_id integer not null, + PRIMARY KEY (no_w_id, no_d_id, no_o_id) +)tablegroup='tpcc_group' partition by hash(no_w_id) partitions {{partition_num}}; + +create table bmsql_oorder ( + o_w_id integer not null, + o_d_id integer not null, + o_id integer not null, + o_c_id integer, + o_carrier_id integer, + o_ol_cnt integer, + o_all_local integer, + o_entry_d timestamp, + PRIMARY KEY (o_w_id, o_d_id, o_id) +)tablegroup='tpcc_group' partition by hash(o_w_id) partitions {{partition_num}}; + +create table bmsql_order_line ( + ol_w_id integer not null, + ol_d_id integer not null, + ol_o_id integer not null, + ol_number integer not null, + ol_i_id integer not null, + ol_delivery_d timestamp, + ol_amount decimal(6,2), + ol_supply_w_id integer, + ol_quantity integer, + ol_dist_info char(24), + PRIMARY KEY (ol_w_id, ol_d_id, ol_o_id, ol_number) +)tablegroup='tpcc_group' partition by hash(ol_w_id) partitions {{partition_num}}; + +create table bmsql_item ( + i_id integer not null, + i_name varchar(24), + i_price decimal(5,2), + i_data varchar(50), + i_im_id integer, + PRIMARY KEY (i_id) +) duplicate_scope='cluster'; + +create table bmsql_stock ( + s_w_id integer not null, + s_i_id integer not null, + s_quantity integer, + s_ytd integer, + s_order_cnt integer, + s_remote_cnt integer, + s_data varchar(50), + s_dist_01 char(24), + s_dist_02 char(24), + s_dist_03 char(24), + s_dist_04 char(24), + s_dist_05 char(24), + s_dist_06 char(24), + s_dist_07 char(24), + s_dist_08 char(24), + s_dist_09 char(24), + s_dist_10 char(24), + PRIMARY KEY (s_w_id, s_i_id) +)tablegroup='tpcc_group' partition by hash(s_w_id) partitions {{partition_num}}; \ No newline at end of file diff --git a/plugins/tpcc/3.1.0/sql.oceanbase/tableDrops.sql b/plugins/tpcc/3.1.0/sql.oceanbase/tableDrops.sql new file mode 100644 index 0000000000000000000000000000000000000000..61fe21d4e43092987984054885c753d2f52d34da --- /dev/null +++ b/plugins/tpcc/3.1.0/sql.oceanbase/tableDrops.sql @@ -0,0 +1,21 @@ +drop table bmsql_config; + +drop table bmsql_new_order; + +drop table bmsql_order_line; + +drop table bmsql_oorder; + +drop table bmsql_history; + +drop table bmsql_customer; + +drop table bmsql_stock; + +drop table bmsql_item; + +drop table bmsql_district; + +drop table bmsql_warehouse; + +drop tablegroup tpcc_group; \ No newline at end of file diff --git a/plugins/tpch/3.1.0/create_tpch_mysql_table_part.ddl b/plugins/tpch/3.1.0/create_tpch_mysql_table_part.ddl index 520986b0c75e15fbc59ed545c6da2475cfb075c1..f0ac9b7e77c0328dc7e2752275ad9df691378120 100644 --- a/plugins/tpch/3.1.0/create_tpch_mysql_table_part.ddl +++ b/plugins/tpch/3.1.0/create_tpch_mysql_table_part.ddl @@ -1,5 +1,16 @@ -create tablegroup if not exists tpch_tg_lineitem_order_group binding true partition by key 1 partitions 192; -create tablegroup if not exists tpch_tg_partsupp_part binding true partition by key 1 partitions 192; +drop table if exists lineitem; +drop table if exists orders; +drop table if exists partsupp; +drop table if exists part; +drop table if exists customer; +drop table if exists supplier; +drop table if exists nation; +drop table if exists region; +drop tablegroup if exists tpch_tg_lineitem_order_group; +drop tablegroup if exists tpch_tg_partsupp_part; + +create tablegroup if not exists tpch_tg_lineitem_order_group binding true partition by key 1 partitions cpu_num; +create tablegroup if not exists tpch_tg_partsupp_part binding true partition by key 1 partitions cpu_num; drop table if exists lineitem; create table lineitem ( @@ -21,7 +32,7 @@ drop table if exists lineitem; l_comment varchar(44) default null, primary key(l_orderkey, l_linenumber)) tablegroup = tpch_tg_lineitem_order_group - partition by key (l_orderkey) partitions 192; + partition by key (l_orderkey) partitions cpu_num; create index I_L_ORDERKEY on lineitem(l_orderkey) local; create index I_L_SHIPDATE on lineitem(l_shipdate) local; @@ -38,7 +49,7 @@ drop table if exists orders; o_comment varchar(79) default null, primary key (o_orderkey)) tablegroup = tpch_tg_lineitem_order_group - partition by key(o_orderkey) partitions 192; + partition by key(o_orderkey) partitions cpu_num; create index I_O_ORDERDATE on orders(o_orderdate) local; @@ -51,7 +62,7 @@ drop table if exists partsupp; ps_comment varchar(199) default null, primary key (ps_partkey, ps_suppkey)) tablegroup tpch_tg_partsupp_part - partition by key(ps_partkey) partitions 192; + partition by key(ps_partkey) partitions cpu_num; drop table if exists part; @@ -67,7 +78,7 @@ drop table if exists part; p_comment varchar(23) default null, primary key (p_partkey)) tablegroup tpch_tg_partsupp_part - partition by key(p_partkey) partitions 192; + partition by key(p_partkey) partitions cpu_num; drop table if exists customer; @@ -81,7 +92,7 @@ drop table if exists customer; c_mktsegment char(10) default null, c_comment varchar(117) default null, primary key (c_custkey)) - partition by key(c_custkey) partitions 192; + partition by key(c_custkey) partitions cpu_num; drop table if exists supplier; create table supplier ( @@ -93,7 +104,7 @@ drop table if exists supplier; s_acctbal bigint default null, s_comment varchar(101) default null, primary key (s_suppkey) -) partition by key(s_suppkey) partitions 192; +) partition by key(s_suppkey) partitions cpu_num; drop table if exists nation; diff --git a/plugins/tpch/3.1.0/queries/db10.sql b/plugins/tpch/3.1.0/queries/db10.sql index 65924d87e3c5c5bdac3b265ec32da389f55e37c9..239206264df553dcb0685e3f42ee5b89d236a141 100644 --- a/plugins/tpch/3.1.0/queries/db10.sql +++ b/plugins/tpch/3.1.0/queries/db10.sql @@ -1,7 +1,8 @@ -- using default substitutions + select /*+ TPCH_Q10 parallel(cpu_num) */ - c_custkey + c_custkey, c_name, sum(l_extendedprice * (1 - l_discount)) as revenue, c_acctbal, @@ -30,4 +31,5 @@ group by c_address, c_comment order by - revenue desc; + revenue desc +limit 20; diff --git a/plugins/tpch/3.1.0/run_test.py b/plugins/tpch/3.1.0/run_test.py index 82dd9b46037b8eb3ebe0ce1504c5fbf58cd229b9..518288e47f941e3aa87765a1c3b13426cdb2f657 100644 --- a/plugins/tpch/3.1.0/run_test.py +++ b/plugins/tpch/3.1.0/run_test.py @@ -218,7 +218,7 @@ def run_test(plugin_context, db, cursor, *args, **kwargs): if ret: server_num = ret.get("server_num", server_num) - parallel_max_servers = int(max_cpu * 10) + parallel_max_servers = min(int(max_cpu * 10), 1800) parallel_servers_target = int(max_cpu * server_num * 8) tenant_variables = [ # [变量名, 新值, 旧值, 替换条件: lambda n, o: n != o] @@ -260,6 +260,19 @@ def run_test(plugin_context, db, cursor, *args, **kwargs): parallel_num = int(max_cpu * unit_count) if not_test_only: + # 替换并发数 + stdio.start_loading('Format DDL') + n_ddl_path = [] + for fp in ddl_path: + _, fn = os.path.split(fp) + nfp = os.path.join(tmp_dir, fn) + ret = local_execute_command("sed %s -e 's/partitions cpu_num/partitions %d/' > %s" % (fp, cpu_total, nfp)) + if not ret: + raise Exception(ret.stderr) + n_ddl_path.append(nfp) + ddl_path = n_ddl_path + stdio.stop_loading('succeed') + stdio.start_loading('Create table') for path in ddl_path: path = os.path.abspath(path) diff --git a/profile/obd.sh b/profile/obd.sh index 44c8f94a3114aaf3fba4bdc1aa948105a637378f..577e06b06399af77bd97f4791cd873f014322c15 100644 --- a/profile/obd.sh +++ b/profile/obd.sh @@ -6,16 +6,21 @@ fi function _obd_complete_func { - local cur prev cmd obd_cmd cluster_cmd tenant_cmd mirror_cmd test_cmd + local cur prev cmd obd_cmd cluster_cmd tenant_cmd mirror_cmd test_cmd devmode_cmd COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" obd_cmd="mirror cluster test update repo" - cluster_cmd="autodeploy tenant start deploy redeploy restart reload destroy stop edit-config list display upgrade" + cluster_cmd="autodeploy tenant start deploy redeploy restart reload destroy stop edit-config list display upgrade chst check4ocp" tenant_cmd="create drop" - mirror_cmd="clone create list update" + mirror_cmd="clone create list update enable disable" repo_cmd="list" - test_cmd="mysqltest sysbench tpch" + test_cmd="mysqltest sysbench tpch tpcc" + if [ -f "${OBD_HOME:-"$HOME"}/.obd/.dev_mode" ]; then + obd_cmd="$obd_cmd devmode" + devmode_cmd="enable disable" + fi + if [[ ${cur} == * ]] ; then case "${prev}" in obd);& @@ -23,6 +28,7 @@ function _obd_complete_func cluster);& tenant);& mirror);& + devmode);& repo) cmd=$(eval echo \$"${prev}_cmd") COMPREPLY=( $(compgen -W "${cmd}" -- ${cur}) ) @@ -41,9 +47,9 @@ function _obd_complete_func return 0 else prev="${COMP_WORDS[COMP_CWORD-2]}" - obd_home=${OBD_HOME:-~/.obd} + obd_home=${OBD_HOME:-~} if [[ "$prev" == "cluster" || "$prev" == "test" || "$prev" == "tenant" ]]; then - res=`ls -p $obd_home/cluster 2>/dev/null | sed "s#/##"` + res=`ls -p $obd_home/.obd/cluster 2>/dev/null | sed "s#/##"` compopt -o nospace COMPREPLY=( $(compgen -o filenames -W "${res}" -- ${cur}) ) fi diff --git a/rpm/build.sh b/rpm/build.sh index 96a0e0e7bc5cd8896724247ba7037c47f79d0487..fe8e6891f429ff9455de0876ce2d1778baa61b9d 100755 --- a/rpm/build.sh +++ b/rpm/build.sh @@ -106,7 +106,7 @@ function build() cd .. VERSION=`grep 'Version:' rpm/ob-deploy.spec | head -n1 | awk -F' ' '{print $2}'` CID=`git log |head -n1 | awk -F' ' '{print $2}'` - BRANCH=`git branch | grep -e "^\*" | awk -F' ' '{print $2}'` + BRANCH=`git rev-parse --abbrev-ref HEAD` DATE=`date '+%b %d %Y %H:%M:%S'` VERSION="$VERSION".`date +%s` BUILD_DIR="$DIR/.build" @@ -120,11 +120,17 @@ function build() pyinstaller --hidden-import=decimal --hidden-import=configparser -F obd.py || exit 1 rm -f obd.py obd.spec cp -r plugins $BUILD_DIR/plugins + cp -r config_parser $BUILD_DIR/config_parser + rm -fr $BUILD_DIR/plugins/oceanbase-ce + rm -fr $BUILD_DIR/plugins/obproxy-ce + rm -fr $BUILD_DIR/config_parser/oceanbase-ce rm -fr /usr/obd /usr/bin/obd cp ./dist/obd /usr/bin/obd cp -fr ./profile/* /etc/profile.d/ mv $BUILD_DIR /usr/obd rm -fr dist + cd $BUILD_DIR/plugins && ln -s oceanbase oceanbase-ce && mkdir -p obproxy-ce && cp -fr obproxy/3.1.0 obproxy-ce/3.1.0 + cd $BUILD_DIR/config_parser && ln -s oceanbase oceanbase-ce chmod +x /usr/bin/obd chmod -R 755 /usr/obd/* chown -R root:root /usr/obd/* diff --git a/rpm/ob-deploy.spec b/rpm/ob-deploy.spec index fa86cc8dd58056b01eecd3ab6ba6146d7e07450e..83c99a649f4745eb2759d8f1f3979d3387af93ca 100644 --- a/rpm/ob-deploy.spec +++ b/rpm/ob-deploy.spec @@ -1,5 +1,5 @@ Name: ob-deploy -Version: 1.2.1 +Version: 1.3.0 Release: %(echo $RELEASE)%{?dist} # if you want use the parameter of rpm_create on build time, # uncomment below @@ -43,7 +43,7 @@ wget https://mirrors.aliyun.com/oceanbase/OceanBase.repo cd $SRC_DIR/ rm -rf build.log build dist obd.spec CID=`git log |head -n1 | awk -F' ' '{print $2}'` -BRANCH=`git branch | grep -e "^\*" | awk -F' ' '{print $2}'` +BRANCH=`git rev-parse --abbrev-ref HEAD` DATE=`date '+%b %d %Y %H:%M:%S'` VERSION="$RPM_PACKAGE_VERSION" if [ "$OBD_DUBUG" ]; then @@ -59,11 +59,14 @@ pyinstaller --hidden-import=decimal --hidden-import=configparser -F obd.py rm -f obd.py obd.spec \cp -rf $SRC_DIR/dist/obd ${RPM_BUILD_ROOT}/usr/bin/obd \cp -rf $SRC_DIR/plugins $BUILD_DIR/SOURCES/plugins +\cp -rf $SRC_DIR/config_parser $BUILD_DIR/SOURCES/config_parser \rm -fr $BUILD_DIR/SOURCES/plugins/oceanbase-ce \rm -fr $BUILD_DIR/SOURCES/plugins/obproxy-ce +\rm -fr $BUILD_DIR/SOURCES/config_parser/oceanbase-ce \cp -rf $SRC_DIR/profile/ $BUILD_DIR/SOURCES/ \cp -rf $SRC_DIR/mirror/ $BUILD_DIR/SOURCES/ \cp -rf $BUILD_DIR/SOURCES/plugins ${RPM_BUILD_ROOT}/usr/obd/ +\cp -rf $BUILD_DIR/SOURCES/config_parser ${RPM_BUILD_ROOT}/usr/obd/ \cp -rf $BUILD_DIR/SOURCES/mirror ${RPM_BUILD_ROOT}/usr/obd/ mkdir -p ${RPM_BUILD_ROOT}/etc/profile.d/ \cp -rf $BUILD_DIR/SOURCES/profile/* ${RPM_BUILD_ROOT}/etc/profile.d/ @@ -72,6 +75,7 @@ mkdir -p ${RPM_BUILD_ROOT}/usr/obd/lib/ mkdir -p ${RPM_BUILD_ROOT}/usr/obd/lib/executer \cp -rf ${RPM_DIR}/executer27 ${RPM_BUILD_ROOT}/usr/obd/lib/executer/ cd ${RPM_BUILD_ROOT}/usr/obd/plugins && ln -s oceanbase oceanbase-ce && mkdir -p obproxy-ce && cp -fr obproxy/3.1.0 obproxy-ce/3.1.0 +cd ${RPM_BUILD_ROOT}/usr/obd/config_parser && ln -s oceanbase oceanbase-ce # package infomation %files @@ -95,8 +99,6 @@ cd ${RPM_BUILD_ROOT}/usr/obd/plugins && ln -s oceanbase oceanbase-ce && mkdir -p %post # chkconfig: 2345 10 90 # description: obd .... -#mkdir -p /usr/obd/ && cp -rf /root/.obd/plugins /usr/obd/plugins -#chmod 744 /root/.obd/plugins/* chmod -R 755 /usr/obd/* chown -R root:root /usr/obd/* find /usr/obd -type f -exec chmod 644 {} \; @@ -107,6 +109,24 @@ echo -e 'Installation of obd finished successfully\nPlease source /etc/profile.d #/sbin/chkconfig obd on %changelog +* Wed Mar 30 2022 obd 1.3.0 + - new features: support rotation restart + - new features: support switching deployment users + - new features: obd cluster chst + - new features: obd cluster check4ocp + - fix bug: fixed the default path in tpch + - fix bug: fixed the default component in sysbench +* Wed Jan 05 2022 obd 1.2.1 +- fix bug: fixed the upgrade path encoding error when you use the obd cluster upgrade command without setting the Chinese environment. +- fix bug: fixed the problem caused by no mysqlt.connector dependency when you use the obd cluster upgrade command. +- fix bug: fixed OBD cannot choose and upgrade the target component when you have only one component. +* Fri Dec 31 2021 obd 1.2.0 + - new features: obd mirror disable/enable + - new features: support obagent 1.1.0 + - new features: parameter check + - new features: new option "--wp/--with-parameter" for restart + - new features: support cross version upgrade and rolling upgrade for oceanbase/oceanbase-ce + - fix bug: can not connect to root when sysbench useing obproxy node * Fri Dec 31 2021 obd 1.2.0 - new features: obd mirror disable/enable - new features: support obagent 1.1.0 diff --git a/ssh.py b/ssh.py index c4846798dfa357194c7f5c17cbd8d3936cd5e8db..42b083b591be5f0c2c84f3c84a1eb23a7e9c4831 100644 --- a/ssh.py +++ b/ssh.py @@ -31,7 +31,9 @@ warnings.filterwarnings("ignore") from paramiko import AuthenticationException, SFTPClient from paramiko.client import SSHClient, AutoAddPolicy -from paramiko.ssh_exception import NoValidConnectionsError +from paramiko.ssh_exception import NoValidConnectionsError, SSHException + +from tool import DirectoryUtil __all__ = ("SshClient", "SshConfig", "LocalClient") @@ -102,6 +104,14 @@ class LocalClient(object): return True return False + @staticmethod + def get_file(local_path, remote_path, stdio=None): + return LocalClient.put_file(remote_path, local_path, stdio=stdio) + + @staticmethod + def get_dir(local_path, remote_path, stdio=None): + return LocalClient.put_dir(remote_path, local_path, stdio=stdio) + class SshClient(object): @@ -201,45 +211,63 @@ class SshClient(object): def __del__(self): self.close() + + def _execute_command(self, command, retry, stdio): + if not self._login(stdio): + return SshReturn(255, '', 'connect failed') + + stdio = stdio if stdio else self.stdio + try: + stdin, stdout, stderr = self.ssh_client.exec_command(command) + output = stdout.read().decode(errors='replace') + error = stderr.read().decode(errors='replace') + if output: + idx = output.rindex('\n') + code = int(output[idx:]) + stdout = output[:idx] + verbose_msg = 'exited code %s' % code + else: + code, stdout = 1, '' + if code: + verbose_msg = 'exited code %s, error output:\n%s' % (code, error) + stdio and getattr(stdio, 'verbose', print)(verbose_msg) + return SshReturn(code, stdout, error) + except SSHException as e: + if retry: + self.close() + return self._execute_command(command, retry-1, stdio) + else: + stdio and getattr(stdio, 'exception', print)('') + stdio and getattr(stdio, 'critical', print)('%s@%s connect failed: %s' % (self.config.username, self.config.host, e)) + raise e + except Exception as e: + stdio and getattr(stdio, 'exception', print)('') + stdio and getattr(stdio, 'critical', print)('%s@%s connect failed: %s' % (self.config.username, self.config.host, e)) + raise e def execute_command(self, command, stdio=None): if self._is_local(): - return LocalClient.execute_command(command, self.env, self.config.timeout, stdio) - if not self._login(stdio): - return SshReturn(255, '', 'connect failed') + return LocalClient.execute_command(command, self.env, self.config.timeout, stdio=stdio) stdio = stdio if stdio else self.stdio verbose_msg = '%s execute: %s ' % (self.config, command) stdio and getattr(stdio, 'verbose', print)(verbose_msg, end='') command = '%s %s;echo -e "\n$?\c"' % (self.env_str, command.strip(';')) - stdin, stdout, stderr = self.ssh_client.exec_command(command) - output = stdout.read().decode(errors='replace') - error = stderr.read().decode(errors='replace') - if output: - idx = output.rindex('\n') - code = int(output[idx:]) - stdout = output[:idx] - verbose_msg = 'exited code %s' % code - else: - code, stdout = 1, '' - if code: - verbose_msg += ', error output:\n%s' % error - stdio and getattr(stdio, 'verbose', print)(verbose_msg) - return SshReturn(code, stdout, error) - + return self._execute_command(command, 3, stdio=stdio) + def put_file(self, local_path, remote_path, stdio=None): stdio = stdio if stdio else self.stdio - if self._is_local(): - return LocalClient.put_file(local_path, remote_path, stdio) if not os.path.isfile(local_path): - stdio and getattr(stdio, 'critical', print)('%s is not file' % local_path) + stdio and getattr(stdio, 'error', print)('%s is not file' % local_path) return False + if self._is_local(): + return LocalClient.put_file(local_path, remote_path, stdio=stdio) if not self._open_sftp(stdio): return False - return self._put_file(local_path, remote_path, stdio) + return self._put_file(local_path, remote_path, stdio=stdio) def _put_file(self, local_path, remote_path, stdio=None): - if self.execute_command('mkdir -p %s && rm -fr %s' % (os.path.dirname(remote_path), remote_path), stdio): + if self.execute_command('mkdir -p %s && rm -fr %s' % (os.path.dirname(remote_path), remote_path), stdio=stdio): stdio and getattr(stdio, 'verbose', print)('send %s to %s' % (local_path, remote_path)) if self.sftp.put(local_path, remote_path): return self.execute_command('chmod %s %s' % (oct(os.stat(local_path).st_mode)[-3: ], remote_path)) @@ -248,10 +276,10 @@ class SshClient(object): def put_dir(self, local_dir, remote_dir, stdio=None): stdio = stdio if stdio else self.stdio if self._is_local(): - return LocalClient.put_dir(local_dir, remote_dir, stdio) + return LocalClient.put_dir(local_dir, remote_dir, stdio=stdio) if not self._open_sftp(stdio): return False - if not self.execute_command('mkdir -p %s' % remote_dir, stdio): + if not self.execute_command('mkdir -p %s' % remote_dir, stdio=stdio): return False failed = [] @@ -267,16 +295,87 @@ class SshClient(object): for name in files: local_path = os.path.join(root, name) remote_path = os.path.join(remote_dir, root[local_dir_path_len:].lstrip('/'), name) - if not self._put_file(local_path, remote_path, stdio): + if not self._put_file(local_path, remote_path, stdio=stdio): failed.append(remote_path) for name in dirs: local_path = os.path.join(root, name) remote_path = os.path.join(remote_dir, root[local_dir_path_len:].lstrip('/'), name) - if not self.execute_command('mkdir -p %s' % remote_path, stdio): + if not self.execute_command('mkdir -p %s' % remote_path, stdio=stdio): failed_dirs.append(local_dir) failed.append(remote_path) for path in failed: - stdio and getattr(stdio, 'critical', print)('send %s to %s@%s failed' % (path, self.config.username, self.config.host)) + stdio and getattr(stdio, 'error', print)('send %s to %s@%s failed' % (path, self.config.username, self.config.host)) return True + def get_file(self, local_path, remote_path, stdio=None): + stdio = stdio if stdio else self.stdio + dirname, _ = os.path.split(local_path) + if not dirname: + dirname = os.getcwd() + local_path = os.path.join(dirname, local_path) + if os.path.exists(dirname): + if not os.path.isdir(dirname): + stdio and getattr(stdio, 'error', print)('%s is not directory' % dirname) + return False + elif not DirectoryUtil.mkdir(dirname, stdio=stdio): + return False + if os.path.exists(local_path) and not os.path.isfile(local_path): + stdio and getattr(stdio, 'error', print)('%s is not file' % local_path) + return False + if self._is_local(): + return LocalClient.get_file(local_path, remote_path, stdio=stdio) + if not self._open_sftp(stdio): + return False + return self._get_file(local_path, remote_path, stdio=stdio) + + def _get_file(self, local_path, remote_path, stdio=None): + stdio and getattr(stdio, 'verbose', print)('get %s to %s' % (remote_path, local_path)) + try: + self.sftp.get(remote_path, local_path) + stat = self.sftp.stat(remote_path) + os.chmod(local_path, stat.st_mode) + return True + except Exception as e: + stdio and getattr(stdio, 'exception', print)('from %s@%s get %s to %s failed: %s' % (self.config.username, self.config.host, remote_path, local_path, e)) + return False + + def get_dir(self, local_dir, remote_dir, stdio=None): + stdio = stdio if stdio else self.stdio + dirname, _ = os.path.split(local_dir) + if not dirname: + dirname = os.getcwd() + local_dir = os.path.join(dirname, local_dir) + if os.path.exists(dirname): + if not os.path.isdir(dirname): + stdio and getattr(stdio, 'error', print)('%s is not directory' % dirname) + return False + elif not DirectoryUtil.mkdir(dirname, stdio=stdio): + return False + if os.path.exists(local_dir) and not os.path.isdir(local_dir): + stdio and getattr(stdio, 'error', print)('%s is not directory' % local_dir) + return False + if self._is_local(): + return LocalClient.get_dir(local_dir, remote_dir, stdio=stdio) + if not self._open_sftp(stdio): + return False + return self._get_dir(local_dir, remote_dir, stdio=stdio) + + def _get_dir(self, local_dir, remote_dir, failed=[], stdio=None): + if DirectoryUtil.mkdir(local_dir, stdio=stdio): + try: + for fn in self.sftp.listdir(remote_dir): + remote_path = os.path.join(remote_dir, fn) + local_path = os.path.join(local_dir, fn) + if self.execute_command('bash -c "if [ -f %s ]; then exit 0; else exit 1; fi;"' % remote_path): + if not self._get_file(local_path, remote_path, stdio=stdio): + failed.append(remote_path) + else: + self._get_dir(local_path, remote_path, failed=failed, stdio=stdio.sub_io()) + except Exception as e: + stdio and getattr(stdio, 'exception', print)('Fail to get %s: %s' % (remote_dir, e)) + failed.append(remote_dir) + else: + failed.append(remote_dir) + return not failed + diff --git a/tool.py b/tool.py index f46928969d2c727f754f8b937bf09c4cb02f97f0..4a83bcfca3cd34256d8d9732263cff84c84a7943 100644 --- a/tool.py +++ b/tool.py @@ -27,20 +27,54 @@ import sys import stat import gzip import fcntl +import signal import shutil -from ssh import LocalClient from ruamel.yaml import YAML, YAMLContextManager, representer if sys.version_info.major == 2: + from collections import OrderedDict from backports import lzma + class TimeoutError(OSError): + + def __init__(self, *args, **kwargs): + super(TimeoutError, self).__init__(*args, **kwargs) else: import lzma + class OrderedDict(dict): + pass + + +__all__ = ("timeout", "DynamicLoading", "ConfigUtil", "DirectoryUtil", "FileUtil", "YamlLoader", "OrderedDict") _WINDOWS = os.name == 'nt' +class Timeout: + + def __init__(self, seconds=1, error_message='Timeout'): + self.seconds = seconds + self.error_message = error_message + + def handle_timeout(self, signum, frame): + raise TimeoutError(self.error_message) + + def _is_timeout(self): + return self.seconds and self.seconds > 0 + + def __enter__(self): + if self._is_timeout(): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + if self._is_timeout(): + signal.alarm(0) + +timeout = Timeout + + class DynamicLoading(object): class Module(object): @@ -257,10 +291,11 @@ class FileUtil(object): return True except Exception as e: if int(getattr(e, 'errno', -1)) == 26: + from ssh import LocalClient if LocalClient.execute_command('/usr/bin/cp -f %s %s' % (src, dst), stdio=stdio): return True elif stdio: - getattr(stdio, 'exception', print)('copy error') + getattr(stdio, 'exception', print)('copy error: %s' % e) else: raise e return False @@ -274,7 +309,7 @@ class FileUtil(object): return True except Exception as e: if stdio: - getattr(stdio, 'exception', print)('link error') + getattr(stdio, 'exception', print)('link error: %s' % e) else: raise e return False