diff --git a/_cmd.py b/_cmd.py index 8d24210c16a7f421caf53e3ec0b312c019d0be2d..07ac50fa4c2d78ce5b5808ab15eebbfafd91db72 100644 --- a/_cmd.py +++ b/_cmd.py @@ -183,7 +183,7 @@ class ObdCommand(BaseCommand): handler.setFormatter(logging.Formatter("[%%(asctime)s.%%(msecs)03d] [%s] [%%(levelname)s] %%(message)s" % trace_id, "%Y-%m-%d %H:%M:%S")) logger.addHandler(handler) ROOT_IO.trace_logger = logger - obd = ObdHome(self.OBD_PATH, ROOT_IO) + obd = ObdHome(self.OBD_PATH, self.dev_mode, ROOT_IO) ROOT_IO.track_limit += 1 ROOT_IO.verbose('cmd: %s' % self.cmds) ROOT_IO.verbose('opts: %s' % self.opts) diff --git a/_errno.py b/_errno.py index f4dc987766c1f5e650fab2e3f478b25a33031fae..331fc19a44bca52b97a3ab95418f96c564290460 100644 --- a/_errno.py +++ b/_errno.py @@ -49,7 +49,7 @@ class InitDirFailedErrorMessage(object): PERMISSION_DENIED = ': {path} permission denied .' -DOC_LINK_MSG = 'See https://open.oceanbase.com/docs/obd-cn/V1.2.0/10000000000017237.' +DOC_LINK_MSG = 'See https://open.oceanbase.com/docs/obd-cn/V1.3.0/10000000000099584.' EC_CONFIG_CONFLICT_PORT = OBDErrorCode(1000, 'Configuration conflict {server1}:{port} port is used for {server2}\'s {key}') EC_CONFLICT_PORT = OBDErrorCode(1001, '{server}:{port} port is already used') diff --git a/_plugin.py b/_plugin.py index 25f7a5c705affa24c6894983c437bfa2b77196fc..58b8b79349bfda3163730279ce15ec3e59216958 100644 --- a/_plugin.py +++ b/_plugin.py @@ -48,12 +48,13 @@ class Plugin(object): PLUGIN_TYPE = None FLAG_FILE = None - def __init__(self, component_name, plugin_path, version): + def __init__(self, component_name, plugin_path, version, dev_mode): if not self.PLUGIN_TYPE or not self.FLAG_FILE: raise NotImplementedError self.component_name = component_name self.plugin_path = plugin_path self.version = Version(version) + self.dev_mode = dev_mode def __str__(self): return '%s-%s-%s' % (self.component_name, self.PLUGIN_TYPE.name.lower(), self.version) @@ -115,12 +116,14 @@ class PluginReturn(object): class PluginContext(object): - def __init__(self, components, clients, cluster_config, cmd, options, stdio): + def __init__(self, components, clients, cluster_config, cmd, options, dev_mode, stdio): self.components = components self.clients = clients self.cluster_config = cluster_config self.cmd = cmd self.options = options + # todo 怎么传入开发模式标识 + self.dev_mode = dev_mode self.stdio = stdio self._return = PluginReturn() @@ -169,8 +172,8 @@ class ScriptPlugin(Plugin): return new_method return attr - def __init__(self, component_name, plugin_path, version): - super(ScriptPlugin, self).__init__(component_name, plugin_path, version) + def __init__(self, component_name, plugin_path, version, dev_mode): + super(ScriptPlugin, self).__init__(component_name, plugin_path, version, dev_mode) self.context = None def __call__(self): @@ -191,7 +194,7 @@ class ScriptPlugin(Plugin): sub_clients = {} for server in clients: sub_clients[server] = ScriptPlugin.ClientForScriptPlugin(clients[server], sub_stdio) - self.context = PluginContext(components, sub_clients, cluster_config, cmd, options, sub_stdio) + self.context = PluginContext(components, sub_clients, cluster_config, cmd, options, self.dev_mode, sub_stdio) def after_do(self, stdio, *arg, **kwargs): self._export(stdio) @@ -221,10 +224,10 @@ class PyScriptPlugin(ScriptPlugin): LIBS_PATH = [] PLUGIN_COMPONENT_NAME = None - def __init__(self, component_name, plugin_path, version): + def __init__(self, component_name, plugin_path, version, dev_mode): if not self.PLUGIN_COMPONENT_NAME: raise NotImplementedError - super(PyScriptPlugin, self).__init__(component_name, plugin_path, version) + super(PyScriptPlugin, self).__init__(component_name, plugin_path, version, dev_mode) self.module = None self.libs_path = deepcopy(self.LIBS_PATH) self.libs_path.append(self.plugin_path) @@ -493,8 +496,8 @@ class ParamPlugin(Plugin): DEF_PARAM_YAML = 'parameter.yaml' FLAG_FILE = DEF_PARAM_YAML - def __init__(self, component_name, plugin_path, version): - super(ParamPlugin, self).__init__(component_name, plugin_path, version) + def __init__(self, component_name, plugin_path, version, dev_mode): + super(ParamPlugin, self).__init__(component_name, plugin_path, version, dev_mode) self.def_param_yaml_path = os.path.join(self.plugin_path, self.DEF_PARAM_YAML) self._src_data = None self._need_redploy_items = None @@ -605,8 +608,8 @@ class InstallPlugin(Plugin): FLAG_FILE = FILES_MAP_YAML _KEYCRE = re.compile(r"\$(\w+)") - def __init__(self, component_name, plugin_path, version): - super(InstallPlugin, self).__init__(component_name, plugin_path, version) + def __init__(self, component_name, plugin_path, version, dev_mode): + super(InstallPlugin, self).__init__(component_name, plugin_path, version, dev_mode) self.file_map_path = os.path.join(self.plugin_path, self.FILES_MAP_YAML) self._file_map = {} @@ -671,7 +674,7 @@ class ComponentPluginLoader(object): PLUGIN_TYPE = None - def __init__(self, home_path, plugin_type=PLUGIN_TYPE, stdio=None): + def __init__(self, home_path, plugin_type=PLUGIN_TYPE, dev_mode=False, stdio=None): if plugin_type: self.PLUGIN_TYPE = plugin_type if not self.PLUGIN_TYPE: @@ -679,6 +682,7 @@ class ComponentPluginLoader(object): self.plguin_cls = getattr(sys.modules[__name__], self.PLUGIN_TYPE.value, False) if not self.plguin_cls: raise ImportError(self.PLUGIN_TYPE.value) + self.dev_mode = dev_mode self.stdio = stdio self.path = home_path self.component_name = os.path.split(self.path)[1] @@ -692,7 +696,7 @@ class ComponentPluginLoader(object): else: path, _ = os.path.split(flag_path) _, version = os.path.split(path) - plugin = self.plguin_cls(self.component_name, path, version) + plugin = self.plguin_cls(self.component_name, path, version, self.dev_mode) self._plugins[flag_path] = plugin plugins.append(plugin) return plugins @@ -725,7 +729,7 @@ class PyScriptPluginLoader(ComponentPluginLoader): PLUGIN_TYPE = PluginType.PY_SCRIPT - def __init__(self, home_path, script_name=None, stdio=None): + def __init__(self, home_path, script_name=None, dev_mode=False, stdio=None): if not script_name: raise NotImplementedError type_name = 'PY_SCRIPT_%s' % script_name.upper() @@ -733,7 +737,7 @@ class PyScriptPluginLoader(ComponentPluginLoader): self.PLUGIN_TYPE = PyScriptPluginLoader.PyScriptPluginType(type_name, type_value) if not getattr(sys.modules[__name__], type_value, False): self._create_(script_name) - super(PyScriptPluginLoader, self).__init__(home_path, stdio=stdio) + super(PyScriptPluginLoader, self).__init__(home_path, dev_mode=dev_mode, stdio=stdio) def _create_(self, script_name): exec(''' @@ -742,8 +746,8 @@ class %s(PyScriptPlugin): FLAG_FILE = '%s.py' PLUGIN_COMPONENT_NAME = '%s' - def __init__(self, component_name, plugin_path, version): - super(%s, self).__init__(component_name, plugin_path, version) + def __init__(self, component_name, plugin_path, version, dev_mode): + super(%s, self).__init__(component_name, plugin_path, version, dev_mode) @staticmethod def set_plugin_type(plugin_type): @@ -764,8 +768,9 @@ class PluginManager(Manager): RELATIVE_PATH = 'plugins' # The directory structure for plugin is ./plugins/{component_name}/{version} - def __init__(self, home_path, stdio=None): + def __init__(self, home_path, dev_mode=False, stdio=None): super(PluginManager, self).__init__(home_path, stdio=stdio) + self.dev_mode = dev_mode self.component_plugin_loaders = {} self.py_script_plugin_loaders = {} for plugin_type in PluginType: @@ -779,7 +784,7 @@ class PluginManager(Manager): return None loaders = self.component_plugin_loaders[plugin_type] if component_name not in loaders: - loaders[component_name] = ComponentPluginLoader(os.path.join(self.path, component_name), plugin_type, self.stdio) + loaders[component_name] = ComponentPluginLoader(os.path.join(self.path, component_name), plugin_type, self.dev_mode, self.stdio) loader = loaders[component_name] return loader.get_best_plugin(version) @@ -792,6 +797,6 @@ class PluginManager(Manager): self.py_script_plugin_loaders[script_name] = {} loaders = self.py_script_plugin_loaders[script_name] if component_name not in loaders: - loaders[component_name] = PyScriptPluginLoader(os.path.join(self.path, component_name), script_name, self.stdio) + loaders[component_name] = PyScriptPluginLoader(os.path.join(self.path, component_name), script_name, self.dev_mode, self.stdio) loader = loaders[component_name] return loader.get_best_plugin(version) diff --git a/core.py b/core.py index 67bca4a32a9532b631507877019f2ba339440e82..29e5bdc270052e8c20309ff7ce190a451a74f4f3 100644 --- a/core.py +++ b/core.py @@ -52,8 +52,9 @@ class ObdHome(object): HOME_LOCK_RELATIVE_PATH = 'obd.conf' - def __init__(self, home_path, stdio=None, lock=True): + def __init__(self, home_path, dev_mode=False, stdio=None): self.home_path = home_path + self.dev_mode = dev_mode self._lock = None self._home_conf = None self._mirror_manager = None @@ -81,7 +82,7 @@ class ObdHome(object): @property def plugin_manager(self): if not self._plugin_manager: - self._plugin_manager = PluginManager(self.home_path, self.stdio) + self._plugin_manager = PluginManager(self.home_path, self.dev_mode, self.stdio) return self._plugin_manager @property @@ -1627,6 +1628,7 @@ class ObdHome(object): stop_plugins = self.search_py_script_plugin(repositories, 'stop') connect_plugins = self.search_py_script_plugin(repositories, 'connect') display_plugins = self.search_py_script_plugin(repositories, 'display') + bootstrap_plugins = self.search_py_script_plugin(repositories, 'bootstrap') self._call_stdio('stop_loading', 'succeed') @@ -1774,7 +1776,8 @@ class ObdHome(object): repository=repository, new_cluster_config=new_cluster_config, new_clients=new_ssh_clients, - rollback=True + rollback=True, + bootstrap_plugin=bootstrap_plugins[repository], ): deploy_config.update_component(cluster_config) @@ -2296,7 +2299,7 @@ class ObdHome(object): else: stdio = None self._call_stdio('start_loading', 'Reboot') - obd = ObdHome(self.home_path, stdio=stdio, lock=False) + obd = ObdHome(self.home_path, self.dev_mode, stdio=stdio) obd.lock_manager.set_try_times(-1) if obd.redeploy_cluster(name): self._call_stdio('stop_loading', 'succeed') @@ -2645,11 +2648,13 @@ class ObdHome(object): odp_cursor = None ob_optimization = True ob_component = None + odp_component = None # ob_cluster_config = None connect_plugin = self.search_py_script_plugin(repositories, 'connect')[repository] if repository.name in ['obproxy', 'obproxy-ce']: + odp_component = repository.name ob_optimization = False allow_components = ['oceanbase', 'oceanbase-ce'] for component in deploy_info.components: @@ -2719,8 +2724,8 @@ class ObdHome(object): return False else: kwargs.update(ret.kwargs) - if kwargs.get('odp_need_reboot'): - components.append('obproxy') + if kwargs.get('odp_need_reboot') and odp_component: + components.append(odp_component) if kwargs.get('obs_need_reboot') and ob_component: components.append(ob_component) if components: @@ -2731,7 +2736,7 @@ class ObdHome(object): if odp_cursor: odp_cursor.close() self._call_stdio('start_loading', 'Restart cluster') - obd = ObdHome(self.home_path, stdio=stdio, lock=False) + obd = ObdHome(self.home_path, self.dev_mode, stdio=stdio) obd.lock_manager.set_try_times(-1) option = Values({'components': ','.join(components), 'without_parameter': True}) if obd.stop_cluster(name=name, options=option) and obd.start_cluster(name=name, options=option) and obd.display_cluster(name=name): diff --git a/example/autodeploy/distributed-with-obproxy-and-obagent-example.yaml b/example/autodeploy/distributed-with-obproxy-and-obagent-example.yaml index e3b8f75c72b351f3d585bcdb640069534d191df2..8bf54683016d2f1c76623aad8ada4baf7bfb84f5 100644 --- a/example/autodeploy/distributed-with-obproxy-and-obagent-example.yaml +++ b/example/autodeploy/distributed-with-obproxy-and-obagent-example.yaml @@ -61,7 +61,7 @@ oceanbase-ce: zone: zone2 server3: zone: zone3 -obproxy: +obproxy-ce: depends: - oceanbase-ce servers: @@ -103,6 +103,7 @@ obagent: # The working directory for obagent. obagent is started under this directory. This is a required field. home_path: /root/observer skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # The port that pulls and manages the metrics. The default port number is 8088. # server_port: 8088 # Debug port for pprof. The default port number is 8089. diff --git a/example/autodeploy/distributed-with-obproxy-example.yaml b/example/autodeploy/distributed-with-obproxy-example.yaml index afc14a1cfdc8c05298e88b49486848712330e632..b38969c83f5d2cc781caae41ba24cfdd677a0abe 100644 --- a/example/autodeploy/distributed-with-obproxy-example.yaml +++ b/example/autodeploy/distributed-with-obproxy-example.yaml @@ -61,7 +61,7 @@ oceanbase-ce: zone: zone2 server3: zone: zone3 -obproxy: +obproxy-ce: depends: - oceanbase-ce servers: diff --git a/example/autodeploy/single-with-obproxy-example.yaml b/example/autodeploy/single-with-obproxy-example.yaml index c48ee54c432a24328a042a5bb18811e13a39c0cb..6f6f6e8b1e274e1b1f566aea7ec6df609d8cc380 100644 --- a/example/autodeploy/single-with-obproxy-example.yaml +++ b/example/autodeploy/single-with-obproxy-example.yaml @@ -50,7 +50,7 @@ oceanbase-ce: # root_password: # Password for proxyro. proxyro_password must be the same as observer_sys_password. The default value is empty. # proxyro_password: -obproxy: +obproxy-ce: depends: - oceanbase-ce servers: diff --git a/example/distributed-example.yaml b/example/distributed-example.yaml index fd886ccd83a8620b2e7586769304895cf60d4dff..e0f59abb2b1123ec6e258e6cac0abbe2ee8d6fe7 100644 --- a/example/distributed-example.yaml +++ b/example/distributed-example.yaml @@ -29,6 +29,7 @@ oceanbase-ce: enable_syslog_recycle: true # Enable auto system log recycling or not. The default value is false. max_syslog_file_count: 4 # The maximum number of reserved log files before enabling auto recycling. The default value is 0. skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # root_password: # root user password # In this example , support multiple ob process in single node, so different process use different ports. # If deploy ob cluster in multiple nodes, the port and path setting can be same. diff --git a/example/distributed-with-obproxy-example.yaml b/example/distributed-with-obproxy-example.yaml index dfca58bfd6c52d84f999ec0e170b491ab1fb23e6..ec47344d2951de4a4343e8f3d941a6ee238fe2a8 100644 --- a/example/distributed-with-obproxy-example.yaml +++ b/example/distributed-with-obproxy-example.yaml @@ -64,7 +64,7 @@ oceanbase-ce: # The directory for clog, ilog, and slog. The default value is the same as the data_dir value. # redo_dir: /redo zone: zone3 -obproxy: +obproxy-ce: # Set dependent components for the component. # When the associated configurations are not done, OBD will automatically get the these configurations from the dependent components. depends: @@ -82,5 +82,6 @@ obproxy: # observer cluster name, consistent with oceanbase-ce's appname. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # cluster_name: obcluster skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # obproxy_sys_password: # obproxy sys user password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # observer_sys_password: # proxyro user pasword, consistent with oceanbase-ce's proxyro_password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. diff --git a/example/mini-distributed-with-obproxy-example.yaml b/example/mini-distributed-with-obproxy-example.yaml index 4cf208066ed46cf738f0d0259a994ae3686553da..7e65294c738e0fde8bc3935b915d928c4316a2e4 100644 --- a/example/mini-distributed-with-obproxy-example.yaml +++ b/example/mini-distributed-with-obproxy-example.yaml @@ -74,7 +74,7 @@ oceanbase-ce: # The directory for clog, ilog, and slog. The default value is the same as the data_dir value. # redo_dir: /redo zone: zone3 -obproxy: +obproxy-ce: # Set dependent components for the component. # When the associated configurations are not done, OBD will automatically get the these configurations from the dependent components. depends: @@ -92,5 +92,6 @@ obproxy: # observer cluster name, consistent with oceanbase-ce's appname. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # cluster_name: obcluster skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # obproxy_sys_password: # obproxy sys user password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # observer_sys_password: # proxyro user pasword, consistent with oceanbase-ce's proxyro_password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. diff --git a/example/mini-single-with-obproxy-example.yaml b/example/mini-single-with-obproxy-example.yaml index 37162ab4847a5feaf1d2706ac0978f4574e8735a..1a7cd72eaf7b2501992c47ba0caf23f0d7a7912e 100644 --- a/example/mini-single-with-obproxy-example.yaml +++ b/example/mini-single-with-obproxy-example.yaml @@ -48,7 +48,7 @@ oceanbase-ce: appname: obcluster # root_password: # root user password, can be empty # proxyro_password: # proxyro user pasword, consistent with obproxy's observer_sys_password, can be empty -obproxy: +obproxy-ce: # Set dependent components for the component. # When the associated configurations are not done, OBD will automatically get the these configurations from the dependent components. depends: @@ -66,5 +66,6 @@ obproxy: # observer cluster name, consistent with oceanbase-ce's appname. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # cluster_name: obcluster skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # obproxy_sys_password: # obproxy sys user password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # observer_sys_password: # proxyro user pasword, consistent with oceanbase-ce's proxyro_password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. diff --git a/example/obagent/distributed-with-obproxy-and-obagent-example.yaml b/example/obagent/distributed-with-obproxy-and-obagent-example.yaml index ece21ebb447da919c2f8db2d8b51a4d54e5700f0..9341f493db204c813a7e54d43e8f32e77a94c985 100644 --- a/example/obagent/distributed-with-obproxy-and-obagent-example.yaml +++ b/example/obagent/distributed-with-obproxy-and-obagent-example.yaml @@ -61,7 +61,7 @@ oceanbase-ce: # The directory for clog, ilog, and slog. The default value is the same as the data_dir value. # redo_dir: /redo zone: zone3 -obproxy: +obproxy-ce: servers: - 192.168.1.5 # Set dependent components for the component. @@ -79,6 +79,7 @@ obproxy: # observer cluster name, consistent with oceanbase-ce's appname. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # cluster_name: obcluster skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # obproxy_sys_password: # obproxy sys user password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # observer_sys_password: # proxyro user pasword, consistent with oceanbase-ce's proxyro_password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. obagent: diff --git a/example/obproxy/distributed-with-obproxy-example.yaml b/example/obproxy/distributed-with-obproxy-example.yaml index dfca58bfd6c52d84f999ec0e170b491ab1fb23e6..ec47344d2951de4a4343e8f3d941a6ee238fe2a8 100644 --- a/example/obproxy/distributed-with-obproxy-example.yaml +++ b/example/obproxy/distributed-with-obproxy-example.yaml @@ -64,7 +64,7 @@ oceanbase-ce: # The directory for clog, ilog, and slog. The default value is the same as the data_dir value. # redo_dir: /redo zone: zone3 -obproxy: +obproxy-ce: # Set dependent components for the component. # When the associated configurations are not done, OBD will automatically get the these configurations from the dependent components. depends: @@ -82,5 +82,6 @@ obproxy: # observer cluster name, consistent with oceanbase-ce's appname. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # cluster_name: obcluster skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # obproxy_sys_password: # obproxy sys user password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # observer_sys_password: # proxyro user pasword, consistent with oceanbase-ce's proxyro_password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. diff --git a/example/obproxy/obproxy-only-example.yaml b/example/obproxy/obproxy-only-example.yaml index 585c699a446d14b4b9cbefaa3a3e3b4f71c45801..b5e9d8e82d8e8329578894c66a5772593985a9c1 100644 --- a/example/obproxy/obproxy-only-example.yaml +++ b/example/obproxy/obproxy-only-example.yaml @@ -5,7 +5,7 @@ # key_file: your ssh-key file path if need # port: your ssh port, default 22 # timeout: ssh connection timeout (second), default 30 -obproxy: +obproxy-ce: servers: - 192.168.1.5 global: @@ -19,5 +19,6 @@ obproxy: # observer cluster name, consistent with oceanbase-ce's appname. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. cluster_name: obcluster skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # obproxy_sys_password: # obproxy sys user password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # observer_sys_password: # proxyro user pasword, consistent with oceanbase-ce's proxyro_password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. diff --git a/example/single-with-obproxy-example.yaml b/example/single-with-obproxy-example.yaml index c4e244d55ad5453a5b6eaa6dcb1a06e5a69514f2..975bafe9b9b35d0eca1f786cb630fd03b7f420eb 100644 --- a/example/single-with-obproxy-example.yaml +++ b/example/single-with-obproxy-example.yaml @@ -36,7 +36,7 @@ oceanbase-ce: appname: obcluster # root_password: # root user password, can be empty # proxyro_password: # proxyro user pasword, consistent with obproxy's observer_sys_password, can be empty -obproxy: +obproxy-ce: # Set dependent components for the component. # When the associated configurations are not done, OBD will automatically get the these configurations from the dependent components. depends: @@ -54,5 +54,6 @@ obproxy: # observer cluster name, consistent with oceanbase-ce's appname. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # cluster_name: obcluster skip_proxy_sys_private_check: true + enable_strict_kernel_release: false # obproxy_sys_password: # obproxy sys user password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. # observer_sys_password: # proxyro user pasword, consistent with oceanbase-ce's proxyro_password, can be empty. When a depends exists, OBD gets this value from the oceanbase-ce of the depends. diff --git a/plugins/obproxy/3.1.0/connect.py b/plugins/obproxy/3.1.0/connect.py index 270aba5a1cec8dd048d2c9bd319fd84d22a50cc3..3b6acfbbb3e046ec25dd12ad448da610a8b08797 100644 --- a/plugins/obproxy/3.1.0/connect.py +++ b/plugins/obproxy/3.1.0/connect.py @@ -107,6 +107,10 @@ def connect(plugin_context, target_server=None, sys_root=True, *args, **kwargs): if r_password is None: r_password = '' db, cursor = _connect(server.ip, server_config['listen_port'], user, r_password if count % 2 else '') + if user in ['root', 'root@sys']: + stdio.verbose('execute sql: select * from information_schema.TABLES limit 1') + cursor.execute('select * from information_schema.TABLES limit 1') + stdio.verbose("result: {}".format(cursor.fetchone())) dbs[server] = db cursors[server] = cursor except: diff --git a/plugins/obproxy/3.1.0/restart.py b/plugins/obproxy/3.1.0/restart.py index 3cc35628f66e04eb0fe96011c65a130e174fff79..c53b771f9240357f2ef7571fb68a8aa724dd8e8e 100644 --- a/plugins/obproxy/3.1.0/restart.py +++ b/plugins/obproxy/3.1.0/restart.py @@ -26,7 +26,7 @@ import os class Restart(object): - def __init__(self, plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None): + def __init__(self, plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None, bootstrap_plugin=None): self.local_home_path = local_home_path self.plugin_context = plugin_context self.components = plugin_context.components @@ -38,6 +38,7 @@ class Restart(object): self.reload_plugin = reload_plugin self.connect_plugin = connect_plugin self.display_plugin = display_plugin + self.bootstrap_plugin = bootstrap_plugin self.stop_plugin = stop_plugin self.new_clients = new_clients self.new_cluster_config = new_cluster_config @@ -96,12 +97,17 @@ class Restart(object): cluster_config = self.new_cluster_config if self.new_cluster_config else self.cluster_config self.stdio.verbose('Call %s for %s' % (self.start_plugin, self.repository)) - if not self.start_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, local_home_path=self.local_home_path, repository_dir=self.repository.repository_dir): + need_bootstrap = self.bootstrap_plugin is not None + if not self.start_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, local_home_path=self.local_home_path, repository_dir=self.repository.repository_dir, need_bootstrap=need_bootstrap): self.rollback() self.stdio.stop_loading('stop_loading', 'fail') return False if self.connect(): + if self.bootstrap_plugin: + self.stdio.verbose('Call %s for %s' % (self.bootstrap_plugin, self.repository)) + self.bootstrap_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, cursor=self.cursors) + self.stdio.verbose('Call %s for %s' % (self.display_plugin, self.repository)) ret = self.display_plugin(self.components, clients, cluster_config, self.plugin_context.cmd, self.plugin_context.options, self.sub_io, cursor=self.cursors) if self.new_cluster_config: self.stdio.verbose('Call %s for %s' % (self.reload_plugin, self.repository)) @@ -121,8 +127,8 @@ class Restart(object): new_client.execute_command('sudo chown -R %s: %s' % (client.config.username, home_path)) -def restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None, rollback=False, *args, **kwargs): - task = Restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config, new_clients) +def restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config=None, new_clients=None, rollback=False, bootstrap_plugin=None, *args, **kwargs): + task = Restart(plugin_context, local_home_path, start_plugin, reload_plugin, stop_plugin, connect_plugin, display_plugin, repository, new_cluster_config, new_clients, bootstrap_plugin=bootstrap_plugin) call = task.rollback if rollback else task.restart if call(): plugin_context.return_true() diff --git a/plugins/obproxy/3.1.0/start.py b/plugins/obproxy/3.1.0/start.py index c7b5ccce485007ae3d4d1e5c5f54761ddd2ae0dd..6a2fc6fb1ca491371caf9b7428d0e20a09e864ec 100644 --- a/plugins/obproxy/3.1.0/start.py +++ b/plugins/obproxy/3.1.0/start.py @@ -86,7 +86,7 @@ def obproxyd(home_path, client, ip, port): return False -def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): +def start(plugin_context, local_home_path, repository_dir, need_bootstrap=False, *args, **kwargs): global stdio cluster_config = plugin_context.cluster_config clients = plugin_context.clients @@ -95,7 +95,6 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): clusters_cmd = {} real_cmd = {} pid_path = {} - need_bootstrap = False for comp in ['oceanbase', 'oceanbase-ce']: if comp in cluster_config.depends: @@ -171,7 +170,7 @@ def start(plugin_context, local_home_path, repository_dir, *args, **kwargs): ] start_unuse = ['home_path', 'observer_sys_password', 'obproxy_sys_password', 'observer_root_password'] get_value = lambda key: "'%s'" % server_config[key] if isinstance(server_config[key], str) else server_config[key] - opt_str = ["obproxy_sys_password=''"] + opt_str = ["obproxy_sys_password=''"] if need_bootstrap else [] for key in server_config: if key not in start_unuse and key not in not_opt_str: value = get_value(key) diff --git a/plugins/obproxy/3.1.0/start_check.py b/plugins/obproxy/3.1.0/start_check.py index 18a052f706b13da38e91c3efbbba9a907076d480..68b2189dc1a892b5c4fd2448d642f90e695915e6 100644 --- a/plugins/obproxy/3.1.0/start_check.py +++ b/plugins/obproxy/3.1.0/start_check.py @@ -45,6 +45,13 @@ def start_check(plugin_context, strict_check=False, *args, **kwargs): stdio.error(*arg, **kwargs) else: stdio.warn(*arg, **kwargs) + def error(*arg, **kwargs): + global success + if plugin_context.dev_mode: + stdio.warn(*arg, **kwargs) + else: + success = False + stdio.error(*arg, **kwargs) def critical(*arg, **kwargs): global success success = False diff --git a/plugins/obproxy/3.1.0/upgrade.py b/plugins/obproxy/3.1.0/upgrade.py index cc2dead231b3f986a2e8d5456beb4c8c46ff03c1..bd39b6ddc44ab0015db229eacaedc5c93ac0b277 100644 --- a/plugins/obproxy/3.1.0/upgrade.py +++ b/plugins/obproxy/3.1.0/upgrade.py @@ -52,16 +52,18 @@ def upgrade(plugin_context, search_py_script_plugin, apply_param_plugin, *args, start_plugin = search_py_script_plugin([dest_repository], 'start')[dest_repository] connect_plugin = search_py_script_plugin([dest_repository], 'connect')[dest_repository] display_plugin = search_py_script_plugin([dest_repository], 'display')[dest_repository] + bootstrap_plugin = search_py_script_plugin([dest_repository], 'bootstrap')[dest_repository] apply_param_plugin(cur_repository) if not stop_plugin(components, clients, cluster_config, cmd, options, stdio, *args, **kwargs): return apply_param_plugin(dest_repository) - if not start_plugin(components, clients, cluster_config, cmd, options, stdio, *args, **kwargs): + if not start_plugin(components, clients, cluster_config, cmd, options, stdio, need_bootstrap=True, *args, **kwargs): return ret = connect_plugin(components, clients, cluster_config, cmd, options, stdio, *args, **kwargs) - if ret and display_plugin(components, clients, cluster_config, cmd, options, stdio, ret.get_return('cursor'), *args, **kwargs): - upgrade_ctx['index'] = len(upgrade_repositories) - return plugin_context.return_true() + if ret: + if bootstrap_plugin(components, clients, cluster_config, cmd, options, stdio, ret.get_return('cursor'), *args, **kwargs) and display_plugin(components, clients, cluster_config, cmd, options, stdio, ret.get_return('cursor'), *args, **kwargs): + upgrade_ctx['index'] = len(upgrade_repositories) + return plugin_context.return_true() diff --git a/plugins/oceanbase/3.1.0/create_tenant.py b/plugins/oceanbase/3.1.0/create_tenant.py index 2c5156cda4bc01bc7f5803033e13cc733575c60c..8c00f5e07e7788cd4d714e31818104113b634232 100644 --- a/plugins/oceanbase/3.1.0/create_tenant.py +++ b/plugins/oceanbase/3.1.0/create_tenant.py @@ -281,6 +281,7 @@ def create_tenant(plugin_context, cursor, *args, **kwargs): cursor.execute(sql) except: exception('faild to crate tenant, execute sql exception: %s' % sql) + return stdio.stop_loading('succeed') return plugin_context.return_true() \ No newline at end of file diff --git a/plugins/oceanbase/3.1.0/start_check.py b/plugins/oceanbase/3.1.0/start_check.py index 409d7bd3957ffd0fc5afe2fe1c37cc0388e6822e..403da9c553d3d4c7789247f88c5caff9c77e830d 100644 --- a/plugins/oceanbase/3.1.0/start_check.py +++ b/plugins/oceanbase/3.1.0/start_check.py @@ -79,6 +79,13 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): stdio.error(*arg, **kwargs) else: stdio.warn(*arg, **kwargs) + def error(*arg, **kwargs): + global success + if plugin_context.dev_mode: + stdio.warn(*arg, **kwargs) + else: + success = False + stdio.error(*arg, **kwargs) def critical(*arg, **kwargs): global success success = False @@ -134,13 +141,13 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): try: memory['num'] += parse_size(server_config['memory_limit']) except: - critical('memory_limit must be an integer') + error('memory_limit must be an integer') return elif 'memory_limit_percentage' in server_config: try: memory['percentage'] += int(parse_size(server_config['memory_limit_percentage'])) except: - critical('memory_limit_percentage must be an integer') + error('memory_limit_percentage must be an integer') return else: memory['percentage'] += 80 @@ -216,7 +223,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): free_memory = parse_size(str(v)) total_use = servers_memory[ip]['percentage'] * total_memory / 100 + servers_memory[ip]['num'] if total_use > free_memory: - stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(total_use))) + error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(total_use))) # disk disk = {'/': 0} ret = client.execute_command('df --block-size=1024') @@ -331,4 +338,4 @@ def start_check(plugin_context, strict_check=False, *args, **kwargs): stdio.stop_loading('succeed') plugin_context.return_true() else: - stdio.stop_loading('fail') \ No newline at end of file + stdio.stop_loading('fail') diff --git a/plugins/tpcc/3.1.0/build.py b/plugins/tpcc/3.1.0/build.py index 676bf2e8cb92e5938dda92a1f953c04f7379ae06..7134161b52affad181a5f80c4b3d14f8e3f7a661 100644 --- a/plugins/tpcc/3.1.0/build.py +++ b/plugins/tpcc/3.1.0/build.py @@ -98,7 +98,38 @@ def build(plugin_context, cursor, odp_cursor, *args, **kwargs): stdio.stop_loading('fail') stdio.exception('') return - + stdio.start_loading('Server check') + try: + # check for observer + while True: + sql = "select * from oceanbase.__all_server where status != 'active' or stop_time > 0 or start_service_time = 0" + stdio.verbose('execute sql: %s' % sql) + cursor.execute(sql) + ret = cursor.fetchone() + if ret is None: + break + time.sleep(3) + # check for obproxy + if odp_cursor: + while True: + sql = "show proxycongestion all" + stdio.verbose('execute obproxy sql: %s' % sql) + odp_cursor.execute(sql) + proxy_congestions = odp_cursor.fetchall() + passed = True + for proxy_congestion in proxy_congestions: + if proxy_congestion.get('dead_congested') != 0 or proxy_congestion.get('server_state') != 'ACTIVE': + passed = False + break + if passed: + break + else: + time.sleep(3) + except: + stdio.stop_loading('fail') + stdio.exception('') + return + stdio.stop_loading('succeed') # drop old tables bmsql_sql_path = kwargs.get('bmsql_sql_path', '') run_sql(sql_file=os.path.join(bmsql_sql_path, 'tableDrops.sql'), force=True) diff --git a/profile/obd.sh b/profile/obd.sh index 577e06b06399af77bd97f4791cd873f014322c15..ce7de20c0a58bd69fbf5be8088e67233be47b7a0 100644 --- a/profile/obd.sh +++ b/profile/obd.sh @@ -15,7 +15,7 @@ function _obd_complete_func tenant_cmd="create drop" mirror_cmd="clone create list update enable disable" repo_cmd="list" - test_cmd="mysqltest sysbench tpch tpcc" + test_cmd="mysqltest sysbench tpch" if [ -f "${OBD_HOME:-"$HOME"}/.obd/.dev_mode" ]; then obd_cmd="$obd_cmd devmode" devmode_cmd="enable disable" @@ -38,7 +38,7 @@ function _obd_complete_func -c|--config) filename=${cur##*/} dirname=${cur%*$filename} - res=`ls -a -p $dirname 2>/dev/null | sed "s#^#$dirname#"` + res=`ls -p $dirname 2>/dev/null | sed "s#^#$dirname#"` compopt -o nospace COMPREPLY=( $(compgen -o filenames -W "${res}" -- ${cur}) ) ;; diff --git a/rpm/build.sh b/rpm/build.sh index fe8e6891f429ff9455de0876ce2d1778baa61b9d..56e2cf7f30583b9062cbecd06c15be307dccb1b9 100755 --- a/rpm/build.sh +++ b/rpm/build.sh @@ -2,6 +2,7 @@ python_bin='python' W_DIR=`pwd` +VERSION=${VERSION:-'1.3.1'} function python_version() @@ -59,11 +60,12 @@ function pacakge_obd() DIR=`pwd` RELEASE=${RELEASE:-'1'} export RELEASE=$RELEASE + export VERSION=$VERSION pip install -r ../requirements3.txt rm -fr rpmbuild mkdir -p rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} rpmbuild --define "_topdir $DIR/rpmbuild" -bb ob-deploy.spec - rpms=`find rpmbuild/RPMS/ -name *.rpm` || exit 1 + rpms=`find rpmbuild/RPMS/ -name ob-deploy-\*` || exit 1 for rpm in ${rpms[@]}; do cp $rpm ./ done @@ -104,9 +106,13 @@ function build() cd2workdir DIR=`pwd` cd .. - VERSION=`grep 'Version:' rpm/ob-deploy.spec | head -n1 | awk -F' ' '{print $2}'` - CID=`git log |head -n1 | awk -F' ' '{print $2}'` - BRANCH=`git rev-parse --abbrev-ref HEAD` + if [ `git log |head -n1 | awk -F' ' '{print $2}'` ]; then + CID=`git log |head -n1 | awk -F' ' '{print $2}'` + BRANCH=`git rev-parse --abbrev-ref HEAD` + else + CID='UNKNOWN' + BRANCH='UNKNOWN' + fi DATE=`date '+%b %d %Y %H:%M:%S'` VERSION="$VERSION".`date +%s` BUILD_DIR="$DIR/.build" @@ -129,7 +135,7 @@ function build() cp -fr ./profile/* /etc/profile.d/ mv $BUILD_DIR /usr/obd rm -fr dist - cd $BUILD_DIR/plugins && ln -s oceanbase oceanbase-ce && mkdir -p obproxy-ce && cp -fr obproxy/3.1.0 obproxy-ce/3.1.0 + cd $BUILD_DIR/plugins && ln -s oceanbase oceanbase-ce && mv obproxy obproxy-ce cd $BUILD_DIR/config_parser && ln -s oceanbase oceanbase-ce chmod +x /usr/bin/obd chmod -R 755 /usr/obd/* diff --git a/rpm/ob-deploy.spec b/rpm/ob-deploy.spec index 83c99a649f4745eb2759d8f1f3979d3387af93ca..e2b5afd3fa28379cab747a09325189ef10043fbd 100644 --- a/rpm/ob-deploy.spec +++ b/rpm/ob-deploy.spec @@ -1,5 +1,5 @@ Name: ob-deploy -Version: 1.3.0 +Version: %(echo $VERSION) Release: %(echo $RELEASE)%{?dist} # if you want use the parameter of rpm_create on build time, # uncomment below @@ -42,8 +42,13 @@ rm -fr $SRC_DIR/mirror/remote && mkdir -p $SRC_DIR/mirror/remote && cd $SRC_DIR/ wget https://mirrors.aliyun.com/oceanbase/OceanBase.repo cd $SRC_DIR/ rm -rf build.log build dist obd.spec -CID=`git log |head -n1 | awk -F' ' '{print $2}'` -BRANCH=`git rev-parse --abbrev-ref HEAD` +if [ `git log |head -n1 | awk -F' ' '{print $2}'` ]; then + CID=`git log |head -n1 | awk -F' ' '{print $2}'` + BRANCH=`git rev-parse --abbrev-ref HEAD` +else + CID='UNKNOWN' + BRANCH='UNKNOWN' +fi DATE=`date '+%b %d %Y %H:%M:%S'` VERSION="$RPM_PACKAGE_VERSION" if [ "$OBD_DUBUG" ]; then @@ -74,7 +79,7 @@ mkdir -p ${RPM_BUILD_ROOT}/usr/obd/lib/ \cp -rf $BUILD_DIR/SOURCES/site-packages ${RPM_BUILD_ROOT}/usr/obd/lib/site-packages mkdir -p ${RPM_BUILD_ROOT}/usr/obd/lib/executer \cp -rf ${RPM_DIR}/executer27 ${RPM_BUILD_ROOT}/usr/obd/lib/executer/ -cd ${RPM_BUILD_ROOT}/usr/obd/plugins && ln -s oceanbase oceanbase-ce && mkdir -p obproxy-ce && cp -fr obproxy/3.1.0 obproxy-ce/3.1.0 +cd ${RPM_BUILD_ROOT}/usr/obd/plugins && ln -s oceanbase oceanbase-ce && mv obproxy obproxy-ce cd ${RPM_BUILD_ROOT}/usr/obd/config_parser && ln -s oceanbase oceanbase-ce # package infomation diff --git a/ssh.py b/ssh.py index 42b083b591be5f0c2c84f3c84a1eb23a7e9c4831..479698e19de473901181cd83e9a0edf37156848c 100644 --- a/ssh.py +++ b/ssh.py @@ -306,7 +306,7 @@ class SshClient(object): for path in failed: stdio and getattr(stdio, 'error', print)('send %s to %s@%s failed' % (path, self.config.username, self.config.host)) - return True + return not failed def get_file(self, local_path, remote_path, stdio=None): stdio = stdio if stdio else self.stdio