From ae0f3bcac764d9365a6f1f7db68049edee7074ff Mon Sep 17 00:00:00 2001 From: Rongfeng Fu Date: Fri, 19 May 2023 18:08:18 +0800 Subject: [PATCH] V2.1.0 (#170) * V2.1.0 * V2.1.0 #163 --- .gitignore | 1 + _cmd.py | 370 ++- _environ.py | 6 + _errno.py | 9 +- _plugin.py | 19 +- _stdio.py | 48 +- const.py | 42 + core.py | 249 +- .../1.quickly-start-the-oceanbase-database.md | 2 - .../3.user-guide/3.obd-command/0.obd-demo.md | 2 - .../3.obd-command/1.cluster-command-groups.md | 227 +- ...and-group-for-mirroring-and-warehousing.md | 20 +- .../3.obd-command/3.test-command-group.md | 89 +- .../3.obd-command/4.tools-commands.md | 12 +- .../en-US/4.configuration-file-description.md | 6 +- docs/zh-CN/1.what-is-obd.md | 6 +- .../3.obd-command/1.cluster-command-groups.md | 10 + .../3.obd-command/4.tools-commands.md | 10 + .../3.obd-command/5.obdiag-commands.md | 233 ++ docs/zh-CN/5.faq/1.faq.md | 78 +- ...-to-upgrade-obproxy-to-obproxy-ce-3.2.3.md | 6 +- docs/zh-CN/6.error-messages-in-obd.md | 330 +-- example/all-components-min.yaml | 5 +- example/all-components.yaml | 3 + example/autodeploy/all-components.yaml | 4 + example/autodeploy/default-example.yaml | 4 + ...omponents-with-prometheus-and-grafana.yaml | 24 - ...uted-with-obproxy-and-obagent-example.yaml | 82 +- plugins/general/0.1/install_repo.py | 95 +- plugins/general/0.1/telemetry_info_collect.py | 269 +++ plugins/general/0.1/telemetry_post.py | 39 + plugins/grafana/7.5.17/generate_config.py | 22 +- plugins/obagent/0.1/generate_config.py | 22 +- plugins/obproxy/3.1.0/generate_config.py | 28 +- plugins/obproxy/3.1.0/restart.py | 4 +- .../oceanbase-ce-utils/3.1.0/file_map.yaml | 3 + .../1.0/file_map.yaml | 3 + .../1.0/gather_all.py | 127 ++ .../1.0/gather_clog.py | 115 + .../1.0/gather_log.py | 106 + .../1.0/gather_obproxy_log.py | 106 + .../1.0/gather_perf.py | 85 + .../1.0/gather_plan_monitor.py | 80 + .../1.0/gather_slog.py | 114 + .../1.0/gather_stack.py | 83 + .../1.0/gather_sysstat.py | 78 + .../1.0/generate_config.py | 152 ++ .../1.0/pre_check.py | 95 + plugins/oceanbase/3.1.0/generate_config.py | 38 +- plugins/oceanbase/3.1.0/list_tenant.py | 10 +- plugins/oceanbase/3.1.0/upgrade_route.py | 2 +- plugins/oceanbase/4.0.0.0/generate_config.py | 39 +- plugins/oceanbase/4.0.0.0/list_tenant.py | 19 +- plugins/oceanbase/4.0.0.0/upgrade.py | 54 +- plugins/oceanbase/4.1.0.0/upgrade.py | 573 +++++ plugins/ocp-express/1.0.1/bootstrap.py | 21 + plugins/ocp-express/1.0.1/connect.py | 105 + plugins/ocp-express/1.0.1/display.py | 50 + plugins/ocp-express/1.0.1/generate_config.py | 81 + plugins/ocp-express/1.0.1/parameter.yaml | 349 +++ plugins/ocp-express/1.0.1/start.py | 464 ++++ plugins/ocp-express/1.0.1/start_check.py | 535 +++++ plugins/ocp-express/1.0.1/upgrade.py | 83 + plugins/ocp-express/1.0/display.py | 4 +- plugins/ocp-express/1.0/generate_config.py | 21 +- plugins/prometheus/2.37.1/generate_config.py | 24 +- profile/obd.sh | 5 +- rpm/build.sh | 3 +- rpm/ob-deploy.spec | 24 +- service/api/v1/common.py | 50 + service/api/v1/components.py | 8 +- service/api/v1/deployments.py | 5 +- service/app.py | 4 +- service/handler/comment_handler.py | 52 + service/handler/component_handler.py | 9 +- service/handler/deployment_handler.py | 49 +- service/handler/handler_utils.py | 6 +- ssh.py | 126 +- tool.py | 47 + web/config/config.ts | 7 +- web/mock/createDeploymentConfig.mock.ts | 4 +- web/mock/deployAndStartADeployment.mock.ts | 10 + web/mock/destroyDeployment .mock.ts | 15 + web/mock/exitProcess.mock.ts | 15 + web/mock/getDeployment.mock.ts | 36 + web/mock/getDestroyTaskInfo.mock.ts | 38 +- web/mock/getObdInfo.mock.ts | 8 +- web/mock/listRemoteMirrors.mock.ts | 257 +++ web/mock/preCheckStatus.mock.ts | 88 +- web/mock/precheck.mock.ts | 4 +- web/mock/queryAllComponentVersions.mock.ts | 823 +++---- .../queryComponentByComponentName.mock.ts | 118 +- web/mock/queryComponentParameters.mock.ts | 2030 +++++++++++++---- web/mock/queryConnectionInfo.mock.ts | 73 +- web/mock/queryDeploymentConfig.mock.ts | 1003 ++------ web/mock/queryDeploymentReport.mock.ts | 167 +- web/mock/queryInstallLog.mock.ts | 6 +- web/mock/queryInstallStatus.mock.ts | 92 +- web/mock/recover.mock.ts | 51 +- web/mock/validateOrSetKeepAliveToken.mock.ts | 10 + web/package.json | 2 + web/public/assets/empty2.png | Bin 0 -> 14832 bytes web/public/assets/progress/data.mp4 | Bin 0 -> 566361 bytes web/src/app.ts | 5 + web/src/constants/index.ts | 15 + web/src/global.less | 8 + web/src/i18n/index.ts | 81 + web/src/i18n/strings/en-US.json | 289 +++ web/src/i18n/strings/index.d.ts | 2 + web/src/i18n/strings/index.js | 4 + web/src/i18n/strings/zh-CN.json | 289 +++ web/src/models/global.ts | 18 +- web/src/pages/components/CheckInfo.tsx | 320 ++- web/src/pages/components/ClusterConfig.tsx | 404 +++- .../pages/components/DeleteDeployModal.tsx | 60 +- web/src/pages/components/DeployType.tsx | 36 +- web/src/pages/components/ExitPage.tsx | 44 +- web/src/pages/components/InstallConfig.tsx | 323 ++- web/src/pages/components/InstallFinished.tsx | 375 ++- web/src/pages/components/InstallProcess.tsx | 229 +- web/src/pages/components/NodeConfig.tsx | 516 ++++- web/src/pages/components/Parameter.tsx | 31 +- web/src/pages/components/PreCheckStatus.tsx | 256 ++- web/src/pages/components/ProgressQuit.tsx | 36 + web/src/pages/components/ServerTags.tsx | 8 +- web/src/pages/components/Steps.tsx | 55 +- web/src/pages/components/TooltipInput.tsx | 142 ++ web/src/pages/components/Welcome.tsx | 52 +- web/src/pages/components/indexEn.less | 867 +++++++ web/src/pages/components/indexZh.less | 868 +++++++ web/src/pages/constants/index.ts | 103 +- web/src/pages/index.less | 26 +- web/src/pages/index.tsx | 327 ++- web/src/services/ob-deploy-web/Common.ts | 26 + web/src/services/ob-deploy-web/Components.ts | 6 + web/src/services/ob-deploy-web/Deployments.ts | 28 +- web/src/services/ob-deploy-web/Mirror.ts | 14 - web/src/services/ob-deploy-web/index.ts | 4 +- web/src/services/ob-deploy-web/typings.d.ts | 53 +- web/src/typings/global.d.ts | 5 + web/src/utils/index.tsx | 77 +- web/src/utils/intl.tsx | 23 + web/src/utils/useRequest.ts | 34 +- web/yarn.lock | 135 +- 144 files changed, 13828 insertions(+), 3606 deletions(-) create mode 100644 const.py create mode 100644 docs/zh-CN/3.user-guide/3.obd-command/5.obdiag-commands.md create mode 100644 plugins/general/0.1/telemetry_info_collect.py create mode 100644 plugins/general/0.1/telemetry_post.py create mode 100644 plugins/oceanbase-ce-utils/3.1.0/file_map.yaml create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/file_map.yaml create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_all.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_clog.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_log.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_obproxy_log.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_perf.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_plan_monitor.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_slog.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_stack.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/gather_sysstat.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/generate_config.py create mode 100644 plugins/oceanbase-diagnostic-tool/1.0/pre_check.py create mode 100644 plugins/oceanbase/4.1.0.0/upgrade.py create mode 100644 plugins/ocp-express/1.0.1/bootstrap.py create mode 100644 plugins/ocp-express/1.0.1/connect.py create mode 100644 plugins/ocp-express/1.0.1/display.py create mode 100644 plugins/ocp-express/1.0.1/generate_config.py create mode 100644 plugins/ocp-express/1.0.1/parameter.yaml create mode 100644 plugins/ocp-express/1.0.1/start.py create mode 100644 plugins/ocp-express/1.0.1/start_check.py create mode 100644 plugins/ocp-express/1.0.1/upgrade.py create mode 100644 service/api/v1/common.py create mode 100644 service/handler/comment_handler.py create mode 100644 web/mock/deployAndStartADeployment.mock.ts create mode 100644 web/mock/destroyDeployment .mock.ts create mode 100644 web/mock/exitProcess.mock.ts create mode 100644 web/mock/getDeployment.mock.ts create mode 100644 web/mock/listRemoteMirrors.mock.ts create mode 100644 web/mock/validateOrSetKeepAliveToken.mock.ts create mode 100644 web/public/assets/empty2.png create mode 100644 web/public/assets/progress/data.mp4 create mode 100644 web/src/constants/index.ts create mode 100644 web/src/i18n/index.ts create mode 100644 web/src/i18n/strings/en-US.json create mode 100644 web/src/i18n/strings/index.d.ts create mode 100644 web/src/i18n/strings/index.js create mode 100644 web/src/i18n/strings/zh-CN.json create mode 100644 web/src/pages/components/ProgressQuit.tsx create mode 100644 web/src/pages/components/TooltipInput.tsx create mode 100644 web/src/pages/components/indexEn.less create mode 100644 web/src/pages/components/indexZh.less create mode 100644 web/src/services/ob-deploy-web/Common.ts create mode 100644 web/src/utils/intl.tsx diff --git a/.gitignore b/.gitignore index 87a42d7..b05824c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ tags /web/src/.umi-production /web/src/.umi-test /web/.env.local +/web/.must.config.js diff --git a/_cmd.py b/_cmd.py index 4fe1115..392ad5f 100644 --- a/_cmd.py +++ b/_cmd.py @@ -25,8 +25,9 @@ import os import sys import time import textwrap +import json from uuid import uuid1 as uuid, UUID -from optparse import OptionParser, OptionGroup, BadOptionError, Option, IndentedHelpFormatter +from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter from core import ObdHome from _stdio import IO @@ -34,20 +35,14 @@ from _lock import LockMode from tool import DirectoryUtil, FileUtil, NetUtil, COMMAND_ENV from _errno import DOC_LINK_MSG, LockError import _environ as ENV +from ssh import LocalClient +from const import * ROOT_IO = IO(1) -VERSION = '' -REVISION = '' -BUILD_BRANCH = '' -BUILD_TIME = '' - -CONST_OBD_HOME = "OBD_HOME" -CONST_OBD_INSTALL_PRE = "OBD_INSTALL_PRE" -CONST_OBD_INSTALL_PATH = "OBD_INSTALL_PATH" -FORBIDDEN_VARS = (CONST_OBD_HOME, CONST_OBD_INSTALL_PRE, CONST_OBD_INSTALL_PATH) OBD_HOME_PATH = os.path.join(os.environ.get(CONST_OBD_HOME, os.getenv('HOME')), '.obd') +OBDIAG_HOME_PATH = os.path.join(os.environ.get(CONST_OBD_HOME, os.getenv('HOME')), 'oceanbase-diagnostic-tool') COMMAND_ENV.load(os.path.join(OBD_HOME_PATH, '.obd_environ'), ROOT_IO) @@ -212,6 +207,10 @@ class ObdCommand(BaseCommand): def lock_mode(self): return COMMAND_ENV.get(ENV.ENV_LOCK_MODE) + @property + def enable_log(self): + return True + def parse_command(self): if self.parser.allow_undefine != True: self.parser.allow_undefine = self.dev_mode @@ -226,11 +225,12 @@ class ObdCommand(BaseCommand): log_dir = os.path.join(self.OBD_PATH, 'log') DirectoryUtil.mkdir(log_dir) log_path = os.path.join(log_dir, 'obd') - ROOT_IO.init_trace_logger(log_path, 'obd', trace_id) - obd = ObdHome(home_path=self.OBD_PATH, dev_mode=self.dev_mode, lock_mode=self.lock_mode, stdio=ROOT_IO) + if self.enable_log: + ROOT_IO.init_trace_logger(log_path, 'obd', trace_id) ROOT_IO.track_limit += 1 ROOT_IO.verbose('cmd: %s' % self.cmds) ROOT_IO.verbose('opts: %s' % self.opts) + obd = ObdHome(home_path=self.OBD_PATH, dev_mode=self.dev_mode, lock_mode=self.lock_mode, stdio=ROOT_IO) obd.set_options(self.opts) obd.set_cmds(self.cmds) ret = self._do_command(obd) @@ -419,6 +419,45 @@ class EnvironmentMajorCommand(HiddenMajorCommand): self.register_command(EnvironmentClearCommand()) +class TelemetryPostCommand(HiddenObdCommand): + + def __init__(self): + super(TelemetryPostCommand, self).__init__("post", "Post telemetry data to OceanBase.By default, OBD telemetry is enabled. To disable OBD telemetry, run the `obd env set TELEMETRY_MODE 0` command. To enable OBD telemetry data printing, run `obd env set TELEMETRY_LOG_MODE 1`.") + self.parser.add_option('-d', '--data', type='string', help="post obd data") + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + @property + def enable_log(self): + if COMMAND_ENV.get(ENV.TELEMETRY_LOG_MODE, default='1') == '0': + return False + return True + + def init(self, cmd, args): + super(TelemetryPostCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obd): + return obd.telemetry_post(self.cmds[0]) + + +class TelemetryMajorCommand(HiddenMajorCommand): + + def __init__(self): + super(TelemetryMajorCommand, self).__init__("telemetry", "Telemetry for OB-Deploy.By default, OBD telemetry is enabled. To disable OBD telemetry, run the `obd env set TELEMETRY_MODE 0` command. To enable OBD telemetry data printing, run `obd env set TELEMETRY_LOG_MODE 1`.") + self.register_command(TelemetryPostCommand()) + + def do_command(self): + if COMMAND_ENV.get(ENV.TELEMETRY_MODE, default='1') == '1': + return super(TelemetryMajorCommand, self).do_command() + else: + ROOT_IO.critical('Telemetry is disabled. To enable OBD telemetry, run the `obd env set TELEMETRY_MODE 1` command.') + return False + + class MirrorCloneCommand(ObdCommand): def __init__(self): @@ -609,6 +648,16 @@ class ClusterMirrorCommand(ObdCommand): self.parser.set_usage('%s [options]' % self.prev_cmd) return self + def get_obd_namespaces_data(self, obd): + data = {} + for component, _ in obd.namespaces.items(): + data[component] = _.get_variable('run_result') + return data + + def background_telemetry_task(self, obd): + data = json.dumps(self.get_obd_namespaces_data(obd)) + LocalClient.execute_command_background(f"nohup obd telemetry post {self.cmds[0]} --data='{data}' >/dev/null 2>&1 &") + class ClusterConfigStyleChange(ClusterMirrorCommand): @@ -701,7 +750,9 @@ class ClusterAutoDeployCommand(ClusterMirrorCommand): if obd.genconfig(name): self.opts.config = '' obd.set_cmds(self.cmds[1:]) - return obd.deploy_cluster(name) and obd.start_cluster(name) + res = obd.deploy_cluster(name) and obd.start_cluster(name) + self.background_telemetry_task(obd) + return res return False else: return self._show_help() @@ -723,7 +774,9 @@ class ClusterDeployCommand(ClusterMirrorCommand): if getattr(self.opts, 'force', False) or getattr(self.opts, 'clean', False): setattr(self.opts, 'skip_cluster_status_check', True) obd.set_options(self.opts) - return obd.deploy_cluster(self.cmds[0]) + res = obd.deploy_cluster(self.cmds[0]) + self.background_telemetry_task(obd) + return res else: return self._show_help() @@ -741,7 +794,9 @@ class ClusterStartCommand(ClusterMirrorCommand): def _do_command(self, obd): if self.cmds: obd.set_cmds(self.cmds[1:]) - return obd.start_cluster(self.cmds[0]) + res = obd.start_cluster(self.cmds[0]) + self.background_telemetry_task(obd) + return res else: return self._show_help() @@ -755,7 +810,9 @@ class ClusterStopCommand(ClusterMirrorCommand): def _do_command(self, obd): if self.cmds: - return obd.stop_cluster(self.cmds[0]) + res = obd.stop_cluster(self.cmds[0]) + self.background_telemetry_task(obd) + return res else: return self._show_help() @@ -768,7 +825,8 @@ class ClusterDestroyCommand(ClusterMirrorCommand): def _do_command(self, obd): if self.cmds: - return obd.destroy_cluster(self.cmds[0]) + res = obd.destroy_cluster(self.cmds[0]) + return res else: return self._show_help() @@ -798,7 +856,9 @@ class ClusterRestartCommand(ClusterMirrorCommand): if not getattr(self.opts, 'with_parameter', False): setattr(self.opts, 'without_parameter', True) obd.set_options(self.opts) - return obd.restart_cluster(self.cmds[0]) + res = obd.restart_cluster(self.cmds[0]) + self.background_telemetry_task(obd) + return res else: return self._show_help() @@ -811,7 +871,9 @@ class ClusterRedeployCommand(ClusterMirrorCommand): def _do_command(self, obd): if self.cmds: - return obd.redeploy_cluster(self.cmds[0]) + res = obd.redeploy_cluster(self.cmds[0]) + self.background_telemetry_task(obd) + return res else: return self._show_help() @@ -823,7 +885,9 @@ class ClusterReloadCommand(ClusterMirrorCommand): def _do_command(self, obd): if self.cmds: - return obd.reload_cluster(self.cmds[0]) + res = obd.reload_cluster(self.cmds[0]) + self.background_telemetry_task(obd) + return res else: return self._show_help() @@ -881,10 +945,13 @@ class CLusterUpgradeCommand(ClusterMirrorCommand): self.parser.add_option('--usable', type='string', help="Hash list for priority mirrors, separated with `,`.", default='') self.parser.add_option('--disable', type='string', help="Hash list for disabled mirrors, separated with `,`.", default='') self.parser.add_option('-e', '--executer-path', type='string', help="Executer path.", default=os.path.join(ObdCommand.OBD_INSTALL_PATH, 'lib/executer')) + self.parser.add_option('-t', '--script-query-timeout', type='string', help="The timeout(s) for executing sql in upgrade scripts. Supported since version 4.1.0", default='') def _do_command(self, obd): if self.cmds: - return obd.upgrade_cluster(self.cmds[0]) + res = obd.upgrade_cluster(self.cmds[0]) + self.background_telemetry_task(obd) + return res else: return self._show_help() @@ -1321,28 +1388,251 @@ class DisplayTraceCommand(ObdCommand): def lock_mode(self): return LockMode.NO_LOCK + @property + def enable_log(self): + return False + def _do_command(self, obd): from ssh import LocalClient + if not self.cmds: + return self._show_help() + log_dir = os.path.join(obd.home_path, 'log/obd') + trace_id = self.cmds[0] + ROOT_IO.verbose('Get log by trace_id') + try: + if UUID(trace_id).version != 1: + ROOT_IO.critical('%s is not trace id' % trace_id) + return False + except: + ROOT_IO.print('%s is not trace id' % trace_id) + return False + cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id) + data = LocalClient.execute_command(cmd) + ROOT_IO.print(data.stdout) + return True + + +class ObdiagCommand(MajorCommand): + + def __init__(self): + super(ObdiagCommand, self).__init__('obdiag', 'Oceanbase Diagnostic Tool') + self.register_command(ObdiagDeployCommand()) + self.register_command(ObdiagGatherCommand()) + + +class ObdiagDeployCommand(ObdCommand): + + + def __init__(self): + super(ObdiagDeployCommand, self).__init__('deploy', 'deploy obdiag') + self.parser.allow_undefine = True + self.parser.undefine_warn = False + + def _do_command(self, obd): + obd.set_options(self.opts) + return obd.obdiag_deploy() + + +class ObdiagGatherMirrorCommand(ObdCommand): + + def init(self, cmd, args): + super(ObdiagGatherMirrorCommand, self).init(cmd, args) + self.parser.set_usage('%s [options]' % self.prev_cmd) + return self + + def _do_command(self, obd): if self.cmds: - if obd.stdio.log_path: - log_dir = obd.stdio.log_path - obd.stdio = IO(0, 20) - trace_id = self.cmds[0] - obd._call_stdio('verbose', 'Get log by trace_id') - try: - if UUID(trace_id).version != 1: - obd._call_stdio('critical', '%s is not trace id' % trace_id) - return False - except: - obd._call_stdio('critical', '%s is not trace id' % trace_id) - return False - cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id) - data = LocalClient.execute_command(cmd) - obd.stdio.print(data.stdout) - return True + return obd.obdiag_gather(self.cmds[0], "gather_%s" % self.name, self.opts) else: - self._show_help() - return False + return self._show_help() + + +class ObdiagGatherCommand(MajorCommand): + + def __init__(self): + super(ObdiagGatherCommand, self).__init__('gather', 'Gather oceanbase diagnostic info') + self.register_command(ObdiagGatherAllCommand()) + self.register_command(ObdiagGatherLogCommand()) + self.register_command(ObdiagGatherSysStatCommand()) + self.register_command(ObdiagGatherStackCommand()) + self.register_command(ObdiagGatherPerfCommand()) + self.register_command(ObdiagGatherSlogCommand()) + self.register_command(ObdiagGatherClogCommand()) + self.register_command(ObdiagGatherPlanMonitorCommand()) + self.register_command(ObdiagGatherObproxyLogCommand()) + +class ObdiagGatherAllCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherAllCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherAllCommand, self).__init__('all', 'Gather oceanbase diagnostic info') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.",default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]",default='all') + self.parser.add_option('--grep', type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) + +class ObdiagGatherLogCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherLogCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherLogCommand, self).__init__('log', 'Gather oceanbase logs from oceanbase machines') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.",default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]",default='all') + self.parser.add_option('--grep', type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) + + +class ObdiagGatherSysStatCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherSysStatCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherSysStatCommand, self).__init__('sysstat', 'Gather Host information') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) + + +class ObdiagGatherStackCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherStackCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherStackCommand, self).__init__('stack', 'Gather stack') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) + + +class ObdiagGatherPerfCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherPerfCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherPerfCommand, self).__init__('perf', 'Gather perf') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('--scope', type='string', help="perf type constrains, choices=[sample, flame, pstack, all]",default='all') + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) + + +class ObdiagGatherSlogCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherSlogCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherSlogCommand, self).__init__('slog', 'Gather slog') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.",default='30m') + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) + + +class ObdiagGatherClogCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherClogCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherClogCommand, self).__init__('clog', 'Gather clog') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.",default='30m') + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) + + +class ObdiagGatherPlanMonitorCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherPlanMonitorCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherPlanMonitorCommand, self).__init__('plan_monitor', 'Gather ParalleSQL information') + self.parser.add_option('-c', '--component', type='string', help="Component name to connect.", default='oceanbase-ce') + self.parser.add_option('--trace_id', type='string', help='sql trace id') + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('-u', '--user', type='string', help='The username used by database connection. [root]',default='root') + self.parser.add_option('-p', '--password', type='string', help='The password used by database connection.',default='') + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) + + +class ObdiagGatherObproxyLogCommand(ObdiagGatherMirrorCommand): + + def init(self, cmd, args): + super(ObdiagGatherObproxyLogCommand, self).init(cmd, args) + return self + + @property + def lock_mode(self): + return LockMode.NO_LOCK + + def __init__(self): + super(ObdiagGatherObproxyLogCommand, self).__init__('obproxy_log', 'Gather obproxy log from obproxy machines') + self.parser.add_option('--from', type='string', help="specify the start of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--to', type='string', help="specify the end of the time range. format: yyyy-mm-dd hh:mm:ss") + self.parser.add_option('--since', type='string', help="Specify time range that from 'n' [d]ays, 'n' [h]ours or 'n' [m]inutes. before to now. format: . example: 1h.",default='30m') + self.parser.add_option('--scope', type='string', help="log type constrains, choices=[observer, election, rootservice, all]",default='all') + self.parser.add_option('--grep', type='string', help="specify keywords constrain") + self.parser.add_option('--encrypt', type='string', help="Whether the returned results need to be encrypted, choices=[true, false]", default="false") + self.parser.add_option('--store_dir', type='string', help='the dir to store gather result, current dir by default.', default=os.getcwd()) + self.parser.add_option('--obdiag_dir', type='string', help="obdiag install dir",default=OBDIAG_HOME_PATH) class MainCommand(MajorCommand): @@ -1359,7 +1649,9 @@ class MainCommand(MajorCommand): self.register_command(UpdateCommand()) self.register_command(DisplayTraceCommand()) self.register_command(EnvironmentMajorCommand()) + self.register_command(TelemetryMajorCommand()) self.register_command(ToolCommand()) + self.register_command(ObdiagCommand()) self.parser.version = '''OceanBase Deploy: %s REVISION: %s BUILD_BRANCH: %s diff --git a/_environ.py b/_environ.py index 8d022d0..687dc64 100644 --- a/_environ.py +++ b/_environ.py @@ -36,3 +36,9 @@ ENV_REPO_INSTALL_MODE = "OBD_REPO_INSTALL_MODE" ENV_DISABLE_RSYNC = "OBD_DISABLE_RSYNC" ENV_DISABLE_PARALLER_EXTRACT = "OBD_DISALBE_PARALLER_EXTRACT" + +# telemetry mode. 0 - disable, 1 - enable. +TELEMETRY_MODE = "TELEMETRY_MODE" + +# telemetry log mode. 0 - disable, 1 - enable. +TELEMETRY_LOG_MODE = "TELEMETRY_LOG_MODE" diff --git a/_errno.py b/_errno.py index 4c92c32..5ed9841 100644 --- a/_errno.py +++ b/_errno.py @@ -174,9 +174,15 @@ EC_OCP_EXPRESS_DEPENDS_COMP_VERSION = OBDErrorCodeTemplate(4304, 'OCP express {o EC_OCP_EXPRESS_META_DB_NOT_ENOUGH_LOG_DISK_AVAILABLE = OBDErrorCodeTemplate(4305, 'There is not enough log disk for ocp meta tenant. (Avail: {avail}, Need: {need})') EC_OCP_EXPRESS_META_DB_NOT_ENOUGH_LOG_DISK = OBDErrorCodeTemplate(4305, 'There is not enough log disk for ocp meta tenant.') EC_OCP_EXPRESS_META_DB_NOT_ENOUGH_MEM = OBDErrorCodeTemplate(4305, 'There is not enough memory for ocp meta tenant') +EC_OCP_EXPRESS_ADMIN_PASSWD_ERROR = OBDErrorCodeTemplate(4306, '({ip}) ocp-express admin_passwd invalid.(Current :{current})') # sql EC_SQL_EXECUTE_FAILED = OBDErrorCodeTemplate(5000, "{sql} execute failed") +# obdiag +EC_OBDIAG_NOT_FOUND = OBDErrorCodeTemplate(6000, 'Failed to executable obdiag command, you may not have obdiag installed') +EC_OBDIAG_NOT_CONTAIN_DEPEND_COMPONENT = OBDErrorCodeTemplate(6001, 'obdiag must contain depend components {components}') +EC_OBDIAG_OPTIONS_FORMAT_ERROR = OBDErrorCodeTemplate(6002, 'obdiag options {option} format error, please check the value : {value}') + # WARN CODE WC_ULIMIT_CHECK = OBDErrorCodeTemplate(1007, '({server}) The recommended number of {key} is {need} (Current value: {now})') WC_AIO_NOT_ENOUGH = OBDErrorCodeTemplate(1011, '({ip}) The recommended value of fs.aio-max-nr is 1048576 (Current value: {current})') @@ -221,4 +227,5 @@ SUG_OCP_EXPRESS_REDUCE_MEM = OBDErrorSuggestionTemplate('Please reduce the `memo SUG_OCP_EXPRESS_REDUCE_DISK = OBDErrorSuggestionTemplate('Please reduce the `logging_file_total_size_cap`', fix_eval=[FixEval(FixEval.DEL, 'logging_file_total_size_cap')]) SUG_OCP_EXPRESS_COMP_VERSION = OBDErrorSuggestionTemplate('Please use {comp} with version {version} or above') SUG_OCP_EXPRESS_REDUCE_META_DB_MEM = OBDErrorSuggestionTemplate('Please reduce the `ocp_meta_tenant_memory_size`', fix_eval=[FixEval(FixEval.DEL, 'ocp_meta_tenant_memory_size')]) -SUG_OCP_EXPRESS_REDUCE_META_DB_LOG_DISK = OBDErrorSuggestionTemplate('Please reduce the `ocp_meta_tenant_log_disk_size`', fix_eval=[FixEval(FixEval.DEL, 'ocp_meta_tenant_log_disk_size')]) \ No newline at end of file +SUG_OCP_EXPRESS_REDUCE_META_DB_LOG_DISK = OBDErrorSuggestionTemplate('Please reduce the `ocp_meta_tenant_log_disk_size`', fix_eval=[FixEval(FixEval.DEL, 'ocp_meta_tenant_log_disk_size')]) +SUG_OCP_EXPRESS_EDIT_ADMIN_PASSWD_ERROR = OBDErrorSuggestionTemplate('Please edit the `admin_passwd`, must be 8 to 32 characters in length, and must contain at least two digits, two uppercase letters, two lowercase letters, and two of the following special characters:~!@#%^&*_-+=|(){{}}[]:;,.?/)', fix_eval=[FixEval(FixEval.DEL, 'admin_passwd')], auto_fix=True) \ No newline at end of file diff --git a/_plugin.py b/_plugin.py index 0814944..83f5b7d 100644 --- a/_plugin.py +++ b/_plugin.py @@ -23,6 +23,7 @@ from __future__ import absolute_import, division, print_function import os import re import sys +import time from enum import Enum from glob import glob from copy import deepcopy, copy @@ -78,8 +79,8 @@ class PluginContextNamespace: def variables(self): return self._variables - def get_variable(self, name): - return self._variables.get(name) + def get_variable(self, name, default=None): + return self._variables.get(name, default) def set_variable(self, name, value): self._variables[name] = value @@ -177,12 +178,12 @@ class PluginContext(object): self._return.return_false(*args, **kwargs) self.namespace.set_return(self.plugin_name, self._return) - def get_variable(self, name, spacename=None): + def get_variable(self, name, spacename=None, default=None): if spacename: namespace = self.namespaces.get(spacename) else: namespace = self.namespace - return namespace.get_variable(name) if namespace else None + return namespace.get_variable(name, default) if namespace else None def set_variable(self, name, value): self.namespace.set_variable(name, value) @@ -270,8 +271,11 @@ def pyScriptPluginExec(func): self.before_do(self.name, namespace, namespaces, deploy_name, repositories, components, clients, cluster_config, cmd, options, stdio, *arg, **kwargs) + method_name = self.PLUGIN_NAME + run_result = self.context.get_variable('run_result', default={}) + run_result[method_name] = {'result': True} + start_time = time.time() if self.module: - method_name = func.__name__ method = getattr(self.module, method_name, False) namespace_vars = copy(self.context.namespace.variables) namespace_vars.update(kwargs) @@ -280,10 +284,15 @@ def pyScriptPluginExec(func): try: ret = method(self.context, *arg, **kwargs) if ret is None and self.context and self.context.get_return() is None: + run_result[method_name]['result'] = False self.context.return_false() except Exception as e: + run_result[method_name]['result'] = False self.context.return_false(exception=e) stdio and getattr(stdio, 'exception', print)('%s RuntimeError: %s' % (self, e)) + end_time = time.time() + run_result[method_name]['time'] = end_time - start_time + self.context.set_variable('run_result', run_result) ret = self.context.get_return() if self.context else PluginReturn() self.after_do(stdio, *arg, **kwargs) return ret diff --git a/_stdio.py b/_stdio.py index f3f1942..2e3240b 100644 --- a/_stdio.py +++ b/_stdio.py @@ -379,10 +379,9 @@ class IO(object): ): self.level = level self.msg_lv = msg_lv - self.log_path = None - self.trace_id = None - self.log_name = 'default' - self.log_path = None + self._log_path = None + self._trace_id = None + self._log_name = 'default' self._trace_logger = None self._log_cache = [] if use_cache else None self._root_io = root_io @@ -419,13 +418,18 @@ class IO(object): self._output_is_tty = output_stream.isatty() return True - def init_trace_logger(self, log_path, log_name=None, trace_id=None): - if self._trace_logger is None: - self.log_path = log_path - if trace_id: - self.trace_id = trace_id + def init_trace_logger(self, log_path, log_name=None, trace_id=None, recreate=False): + if self._root_io: + return False + if self._trace_logger is None or recreate: + self._log_path = log_path if log_name: - self.log_name = log_name + self._log_name = log_name + if trace_id: + self._trace_id = trace_id + self._trace_logger = None + return True + return False def __getstate__(self): state = {} @@ -437,6 +441,8 @@ class IO(object): @property def trace_logger(self): + if self._root_io: + return self._root_io.trace_logger if self.log_path and self._trace_logger is None: self._trace_logger = Logger(self.log_name) handler = handlers.TimedRotatingFileHandler(self.log_path, when='midnight', interval=1, backupCount=30) @@ -447,6 +453,24 @@ class IO(object): self._trace_logger.addHandler(handler) return self._trace_logger + @property + def trace_id(self): + if self._root_io: + return self._root_io.trace_id + return self._trace_id + + @property + def log_path(self): + if self._root_io: + return self._root_io.log_path + return self._log_path + + @property + def log_name(self): + if self._root_io: + return self._root_io.log_name + return self._log_name + @property def log_cache(self): if self._root_io: @@ -618,10 +642,6 @@ class IO(object): track_limit=self.track_limit, root_io=self._root_io if self._root_io else self ) - sub_io.log_name = self.log_name - sub_io.log_path = self.log_path - sub_io.trace_id = self.trace_id - sub_io._trace_logger = self.trace_logger self.sub_ios[key] = sub_io return self.sub_ios[key] diff --git a/const.py b/const.py new file mode 100644 index 0000000..92f0c23 --- /dev/null +++ b/const.py @@ -0,0 +1,42 @@ +# coding: utf-8 +# OceanBase Deploy. +# Copyright (C) 2021 OceanBase +# +# This file is part of OceanBase Deploy. +# +# OceanBase Deploy is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OceanBase Deploy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OceanBase Deploy. If not, see . + +# OceanBase official website +OB_OFFICIAL_WEBSITE = 'https://www.oceanbase.com/' + +# post telemetry data to OceanBase official +TELEMETRY_URL = 'http://openwebapi.dev.alipay.net/api/web/oceanbase/report' + +# obdeploy version +VERSION = '' +# obdeploy build commit +REVISION = '' +# obdeploy build branch +BUILD_BRANCH = '' +# obdeploy build time +BUILD_TIME = '' + +# obdeploy home path +CONST_OBD_HOME = "OBD_HOME" +# obdeploy install pre path +CONST_OBD_INSTALL_PRE = "OBD_INSTALL_PRE" +# obdeploy install path +CONST_OBD_INSTALL_PATH = "OBD_INSTALL_PATH" +# obdeploy forbidden variable +FORBIDDEN_VARS = (CONST_OBD_HOME, CONST_OBD_INSTALL_PRE, CONST_OBD_INSTALL_PATH) \ No newline at end of file diff --git a/core.py b/core.py index c3011f5..8d16013 100644 --- a/core.py +++ b/core.py @@ -25,6 +25,7 @@ import os import time from optparse import Values from copy import deepcopy, copy +import requests import tempfile from subprocess import call as subprocess_call @@ -41,6 +42,7 @@ import _errno as err from _lock import LockManager, LockMode from _optimize import OptimizeManager from _environ import ENV_REPO_INSTALL_MODE, ENV_BASE_DIR +from const import OB_OFFICIAL_WEBSITE class ObdHome(object): @@ -108,7 +110,7 @@ class ObdHome(object): self._optimize_manager = OptimizeManager(self.home_path, stdio=self.stdio) return self._optimize_manager - def _obd_update_lock(self): + def _global_ex_lock(self): self.lock_manager.global_ex_lock() def fork(self, deploy=None, repositories=None, cmds=None, options=None, stdio=None): @@ -936,6 +938,7 @@ class ObdHome(object): continue if component_status[repository] != cluster_status[server]: self._call_stdio('verbose', '%s cluster status is inconsistent' % repository) + component_status[repository] = False break else: continue @@ -1252,8 +1255,9 @@ class ObdHome(object): if component_name: components.add(component_name) self.get_namespace(component_name).set_variable('generate_config_mini', True) + self.get_namespace(component_name).set_variable('generate_password', False) self.get_namespace(component_name).set_variable('auto_depend', True) - + if not components: self._call_stdio('error', 'Use `-c/--components` to set in the components to be deployed') return @@ -1404,6 +1408,12 @@ class ObdHome(object): # Check whether the components have the parameter plugins and apply the plugins self.search_param_plugin_and_apply(repositories, deploy_config) + # Generate password when password is None + gen_config_plugins = self.search_py_script_plugin(repositories, 'generate_config') + for repository in repositories: + if repository in gen_config_plugins: + self.call_plugin(gen_config_plugins[repository], repository, only_generate_password=True) + # Parameter check self._call_stdio('verbose', 'Cluster param configuration check') errors = self.deploy_param_check(repositories, deploy_config) @@ -2605,6 +2615,19 @@ class ObdHome(object): ssh_clients = self.get_clients(deploy_config, [current_repository]) cluster_config = deploy_config.components[current_repository.name] + # Check the status for the deployed cluster + component_status = {} + cluster_status = self.cluster_status_check(repositories, component_status) + if cluster_status is False or cluster_status == 0: + if self.stdio: + self._call_stdio('error', err.EC_SOME_SERVER_STOPED) + for repository in component_status: + cluster_status = component_status[repository] + for server in cluster_status: + if cluster_status[server] == 0: + self._call_stdio('print', '%s %s is stopped' % (server, repository.name)) + return False + route = [] use_images = [] upgrade_route_plugins = self.search_py_script_plugin([current_repository], 'upgrade_route', no_found_act='warn') @@ -2722,6 +2745,7 @@ class ObdHome(object): if not self.install_repositories_to_servers(deploy_config, upgrade_repositories[1:], install_plugins, ssh_clients, self.options): return False + script_query_timeout = getattr(self.options, 'script_query_timeout', '') n = len(upgrade_repositories) while upgrade_ctx['index'] < n: repository = upgrade_repositories[upgrade_ctx['index']] @@ -2737,7 +2761,8 @@ class ObdHome(object): apply_param_plugin=lambda repository: self.search_param_plugin_and_apply([repository], deploy_config), upgrade_ctx=upgrade_ctx, install_repository_to_servers=self.install_repository_to_servers, - unuse_lib_repository=deploy_config.unuse_lib_repository + unuse_lib_repository=deploy_config.unuse_lib_repository, + script_query_timeout=script_query_timeout ) deploy.update_upgrade_ctx(**upgrade_ctx) if not ret: @@ -3466,7 +3491,7 @@ class ObdHome(object): connect_plugin=connect_plugin, optimize_envs=kwargs, operation='recover') def update_obd(self, version, install_prefix='/'): - self._obd_update_lock() + self._global_ex_lock() component_name = 'ob-deploy' plugin = self.plugin_manager.get_best_plugin(PluginType.INSTALL, component_name, '1.0.0') if not plugin: @@ -3920,3 +3945,219 @@ class ObdHome(object): self.call_plugin(sync_config_plugin, target_repository) dooba_plugin = self.plugin_manager.get_best_py_script_plugin('run', 'dooba', plugin_version) return self.call_plugin(dooba_plugin, target_repository) + + def telemetry_post(self, name): + self._call_stdio('verbose', 'Get Deploy by name') + deploy = self.deploy_manager.get_deploy_config(name) + self.set_deploy(deploy) + if not deploy: + self._call_stdio('error', 'No such deploy: %s.' % name) + return False + + deploy_info = deploy.deploy_info + if deploy_info.status in (DeployStatus.STATUS_DESTROYED, DeployStatus.STATUS_CONFIGURED): + self._call_stdio('print', 'Deploy "%s" is %s' % (name, deploy_info.status.value)) + return False + + repositories = self.load_local_repositories(deploy_info) + if repositories == []: + return + self.set_repositories(repositories) + target_repository = None + for repository in repositories: + if repository.name in ['oceanbase', 'oceanbase-ce']: + target_repository = repository + break + else: + target_repository = repository + telemetry_info_collect_plugin = self.plugin_manager.get_best_py_script_plugin('telemetry_info_collect', 'general', '0.1') + ret = self.call_plugin(telemetry_info_collect_plugin, target_repository, target_repository=target_repository) + if ret: + post_data = ret.get_return('post_data') + self._call_stdio('verbose', 'telemetry_data: %s' % post_data) + + telemetry_post_plugin = self.plugin_manager.get_best_py_script_plugin('telemetry_post', 'general', '0.1') + return self.call_plugin(telemetry_post_plugin, target_repository, data=post_data) + + + def obdiag_gather(self, name, gather_type, opts): + self._global_ex_lock() + self._call_stdio('verbose', 'Get Deploy by name') + deploy = self.deploy_manager.get_deploy_config(name, read_only=True) + if not deploy: + self._call_stdio('error', 'No such deploy: %s.' % name) + return False + self.set_deploy(deploy) + self._call_stdio('verbose', 'Get deploy configuration') + deploy_config = deploy.deploy_config + deploy_info = deploy.deploy_info + + if deploy_info.status in (DeployStatus.STATUS_DESTROYED, DeployStatus.STATUS_CONFIGURED): + self._call_stdio('print', 'Deploy "%s" is %s' % (name, deploy_info.status.value)) + return False + + allow_components = [] + if gather_type.startswith("gather_obproxy"): + allow_components = ['obproxy-ce', 'obproxy'] + else: + allow_components = ['oceanbase-ce', 'oceanbase'] + + component_name = "" + for component in deploy_config.components: + if component in allow_components: + component_name = component + break + if component_name == "": + self._call_stdio('error', err.EC_OBDIAG_NOT_CONTAIN_DEPEND_COMPONENT.format(components=allow_components)) + return False + + cluster_config = deploy_config.components[component_name] + if not cluster_config.servers: + self._call_stdio('error', '%s server list is empty' % allow_components[0]) + return False + self._call_stdio('start_loading', 'Get local repositories and plugins') + # Get the repository + repositories = self.load_local_repositories(deploy_info) + self.set_repositories(repositories) + self._call_stdio('stop_loading', 'succeed') + target_repository = None + for repository in repositories: + if repository.name == allow_components[0]: + target_repository = repository + if gather_type in ['gather_plan_monitor']: + setattr(opts, 'connect_cluster', True) + obdiag_path = getattr(opts, 'obdiag_dir', None) + + diagnostic_component_name = 'oceanbase-diagnostic-tool' + obdiag_version = '1.0' + pre_check_plugin = self.plugin_manager.get_best_py_script_plugin('pre_check', diagnostic_component_name, obdiag_version) + check_pass = self.call_plugin(pre_check_plugin, + target_repository, + gather_type = gather_type, + obdiag_path = obdiag_path, + version_check = True, + utils_work_dir_check = True) + if not check_pass: + # obdiag checker return False + if not check_pass.get_return('obdiag_found'): + if not self._call_stdio('confirm', 'Could not find the obdiag, please confirm whether to install it' ): + return False + self.obdiag_deploy(auto_deploy=True, install_prefix=obdiag_path) + # utils checker return False + if not check_pass.get_return('utils_status'): + repositories_utils_map = self.get_repositories_utils(repositories) + if repositories_utils_map is False: + self._call_stdio('error', 'Failed to get utils package') + else: + if not self._call_stdio('confirm', 'obdiag gather clog/slog need to install ob_admin\nDo you want to install ob_admin?'): + if not check_pass.get_return('skip'): + return False + else: + self._call_stdio('warn', 'Just skip gather clog/slog') + else: + if not self.install_utils_to_servers(repositories, repositories_utils_map): + self._call_stdio('error', 'Failed to install utils to servers') + obdiag_version = check_pass.get_return('obdiag_version') + generate_config_plugin = self.plugin_manager.get_best_py_script_plugin('generate_config', diagnostic_component_name, obdiag_version) + self.call_plugin(generate_config_plugin, target_repository, deploy_config=deploy_config) + self._call_stdio('generate_config', 'succeed') + obdiag_plugin = self.plugin_manager.get_best_py_script_plugin(gather_type, diagnostic_component_name, obdiag_version) + return self.call_plugin(obdiag_plugin, target_repository) + + + def obdiag_deploy(self, auto_deploy=False, install_prefix=None): + self._global_ex_lock() + component_name = 'oceanbase-diagnostic-tool' + if install_prefix is None: + install_prefix = os.path.join(os.getenv('HOME'), component_name) + pkg = self.mirror_manager.get_best_pkg(name=component_name) + if not pkg: + self._call_stdio('critical', '%s package not found' % component_name) + return False + plugin = self.plugin_manager.get_best_plugin(PluginType.INSTALL, component_name, pkg.version) + self._call_stdio('print', 'obdiag plugin : %s' % plugin) + + repository = self.repository_manager.create_instance_repository(pkg.name, pkg.version, pkg.md5) + check_plugin = self.plugin_manager.get_best_py_script_plugin('pre_check', component_name, pkg.version) + if not auto_deploy: + ret = self.call_plugin(check_plugin, + repository, + clients={}, + obdiag_path = install_prefix, + obdiag_new_version = pkg.version, + version_check = True) + if not ret and ret.get_return('obdiag_found'): + self._call_stdio('print', 'No updates detected. obdiag is already up to date.') + return False + if not self._call_stdio('confirm', 'Found a higher version\n%s\nDo you want to use it?' % pkg): + return False + self._call_stdio('start_loading', 'Get local repositories and plugins') + repository.load_pkg(pkg, plugin) + src_path = os.path.join(repository.repository_dir, component_name) + if FileUtil.symlink(src_path, install_prefix, self.stdio): + self._call_stdio('stop_loading', 'succeed') + self._call_stdio('print', 'Deploy obdiag successful.\nCurrent version : %s. \nPath of obdiag : %s' % (pkg.version, install_prefix)) + return True + + + def get_repositories_utils(self, repositories): + all_data = [] + data = {} + temp_map = {} + need_install_repositories = ['oceanbase-ce'] + for repository in repositories: + utils_name = '%s-utils' % repository.name + if (utils_name in data) or (repository.name not in need_install_repositories): + continue + data[utils_name] = {'version': repository.version} + temp_map[utils_name] = repository + all_data.append((data, temp_map)) + try: + repositories_utils_map = {} + for data, temp_map in all_data: + with tempfile.NamedTemporaryFile(suffix=".yaml", mode='w') as tf: + yaml_loader = YamlLoader(self.stdio) + yaml_loader.dump(data, tf) + deploy_config = DeployConfig(tf.name, yaml_loader=yaml_loader, config_parser_manager=self.deploy_manager.config_parser_manager) + self._call_stdio('verbose', 'Search best suitable repository utils') + pkgs, utils_repositories, errors = self.search_components_from_mirrors(deploy_config, only_info=False) + if errors: + self._call_stdio('error', '\n'.join(errors)) + return False + + # Get the installation plugin and install + install_plugins = self.get_install_plugin_and_install(utils_repositories, pkgs) + if not install_plugins: + return False + for utils_repository in utils_repositories: + repository = temp_map[utils_repository.name] + install_plugin = install_plugins[utils_repository] + repositories_utils_map[repository] = { + 'repositories': utils_repository, + 'install_plugin': install_plugin + } + return repositories_utils_map + except: + self._call_stdio('exception', 'Failed to create utils-repo config file') + pass + return False + + + def install_utils_to_servers(self, repositories, repositories_utils_map, unuse_utils_repository=True): + install_repo_plugin = self.plugin_manager.get_best_py_script_plugin('install_repo', 'general', '0.1') + check_file_maps = {} + need_install_repositories = ['oceanbase-ce'] + for repository in repositories: + if (repository.name not in need_install_repositories): + continue + temp_repository = deepcopy(repository) + temp_repository.name = '%s-utils' % repository.name + utils_repository = repositories_utils_map[temp_repository]['repositories'] + install_plugin = repositories_utils_map[temp_repository]['install_plugin'] + check_file_map = check_file_maps[repository] = install_plugin.file_map(repository) + ret = self.call_plugin(install_repo_plugin, repository, obd_home=self.home_path, install_repository=utils_repository, + install_plugin=install_plugin, check_repository=repository, check_file_map=check_file_map, + msg_lv='error' if unuse_utils_repository else 'warn') + if not ret: + return False + return True \ No newline at end of file diff --git a/docs/en-US/3.user-guide/1.quickly-start-the-oceanbase-database.md b/docs/en-US/3.user-guide/1.quickly-start-the-oceanbase-database.md index 6ad3119..33e4024 100644 --- a/docs/en-US/3.user-guide/1.quickly-start-the-oceanbase-database.md +++ b/docs/en-US/3.user-guide/1.quickly-start-the-oceanbase-database.md @@ -10,8 +10,6 @@ After you deploy OceanBase Deployer (OBD), you can run the `obd demo` command to - At least 54 GB of disk space is available on the server. -- Your server can be connected to the network, or there are installation packages required for deployment. - > **Note** > > If the foregoing prerequisites are not met, see [Use OBD to start an OceanBase cluster](../3.user-guide/2.start-the-oceanbase-cluster-by-using-obd.md). diff --git a/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md b/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md index bafdc9f..cd8f123 100644 --- a/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md +++ b/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md @@ -36,8 +36,6 @@ obd demo -c oceanbase-ce,obproxy-ce --obproxy-ce.home_path=/data/demo/ obd demo --oceanbase-ce.mysql_port=3881 ``` -For more information about the relevant configuration items in the configuration file, refer to [Configuration file description](../../4.configuration-file-description.md). - > **Notice** > > This command supports only level-1 configurations under global that are specified by using options. diff --git a/docs/en-US/3.user-guide/3.obd-command/1.cluster-command-groups.md b/docs/en-US/3.user-guide/3.obd-command/1.cluster-command-groups.md index 18cad7c..0ee395e 100644 --- a/docs/en-US/3.user-guide/3.obd-command/1.cluster-command-groups.md +++ b/docs/en-US/3.user-guide/3.obd-command/1.cluster-command-groups.md @@ -1,49 +1,42 @@ # Cluster commands -OBD provides multiple-level commands. You can use the `-h/--help` option to view the help information of sub-commands. Similarly, you can also use `-v/--verbose` to view the detailed execution process of commands when the execution of sub commands reports an error. +OBD provides multiple-level commands. You can use the`-h/--help` option to view the help information of sub-commands. A deployment configuration is the minimum unit for OBD cluster commands. A deployment configuration is a `yaml` file. It contains all configuration information of a deployment, including the server login information, component information, component configuration information, and component server list. To start a cluster by using OBD, you must register the deployment configuration of your cluster to OBD. You can run the `obd cluster edit-config` command to create an empty deployment configuration or run the `obd cluster deploy -c config` command to import a deployment configuration. -## obd cluster autodeploy +## `obd cluster autodeploy` When you pass a simple configuration file to OBD, OBD will automatically generate a complete configuration file with the maximum specifications based on the resources of the target server, and then deploy and start a cluster on the target server. ```shell obd cluster autodeploy -c [-f] [-U] [-A] [-s] - -# example -obd cluster autodeploy test -c all-components.yaml ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. The following table describes the corresponding options. | Option | Required | Data type | Default value | Description | -|----|-----|-----|----|----| +--- | --- | --- |--- |--- | -c/--config | Yes | string | None | Specifies the yaml file used for deployment and registers the deployment configuration to OBD.
When the `deploy name` already exists, OBD will check the status of the existing deployment configuration. If the existing deployment configuration has not been applied, it will be overwritten. If the existing deployment configuration is in use, an error will be returned. | | -f/--force | No | bool | false | Specifies whether to forcibly clear the working directory.
When the component requires an empty working directory but this option is disabled, an error will be returned if the working directory is not empty. | -| -C/--clean | No | bool | false | Specifies whether to clear the working directory. When the working directory (`home_path`) belongs to the current operating user and this option is true, the working directory will be cleared. | | -U/--ulp/--unuselibrepo | No | bool | false | Specifies whether to prevent OBD from automatically taking actions when dependencies are missing. If this option is disabled and OBD detects that some dependencies are missing, OBD will automatically search for the corresponding libs mirrors and install them. If this option is enabled, the **unuse_lib_repository: true** field will be added to the corresponding configuration file. You can also add the **unuse_lib_repository: true** field to the configuration file to enable this option. | | -A/--act/--auto-create-tenant | No | bool | false | Specifies whether to enable OBD to create the `test` tenant during the bootstrap by using all available resources of the cluster. If this option is enabled, the **auto_create_tenant: true** field will be added to the corresponding configuration file. You can also add the **auto_create_tenant: true** field to the configuration file to enable this option. | | -s/--strict-check | No | bool | false | Some components will do relevant checks before starting. It will issue an alarm when the check fails, but it will not force the process to stop. Using this option can return an error and directly exit the process when the component pre-check fails. We recommend that you enable this option to avoid startup failures due to insufficient resources. | -## obd cluster edit-config +## `obd cluster edit-config` Modifies a deployment configuration or creates one when the specified deployment configuration does not exist. ```shell obd cluster edit-config - -# example -obd cluster edit-config test ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name for the deployment configuration file. -## obd cluster deploy +## `obd cluster deploy` Deploys a cluster based on the deployment configuration file. Based on the deployment configuration file, this command finds the matching mirror, then installs the mirror in a local repository. This process is called local installation. Then, OBD distributes the components of the required version in the local repository to the target server. This process is called remote installation. @@ -52,46 +45,39 @@ This command allows you to deploy a cluster based on a deployment configuration ```shell obd cluster deploy [-c ] [-f] [-U] [-A] - -# example -obd cluster deploy test -c all-components-min.yaml ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. The following table describes the corresponding options. | Option | Required | Data type | Default value | Description | -|----|-----|-----|----|----| +--- | --- | --- |--- |--- | -c/--config | No | string | None | Specifies the yaml file used for deployment and registers the deployment configuration to OBD.
If this option is enabled and a deployment configuration of the specified `deploy name` already exists, the existing deployment configuration will be overwritten.
If this option is not enabled, OBD will search for the registered deployment configuration of the specified `deploy name`. | | -f/--force | No | bool | false | Specifies whether to forcibly clear the working directory.
When the component requires an empty working directory but this option is disabled, an error will be returned if the working directory is not empty. | -| -C/--clean | No | bool | false | Specifies whether to clear the working directory. When the working directory (`home_path`) belongs to the current operating user and this option is true, the working directory will be cleared. | | -U/--ulp/--unuselibrepo | No | bool | false | Specifies whether to prevent OBD from automatically taking actions when dependencies are missing. If this option is disabled and OBD detects that some dependencies are missing, OBD will automatically search for the corresponding libs mirrors and install them. If this option is enabled, the **unuse_lib_repository: true** field will be added to the corresponding configuration file. You can also add the **unuse_lib_repository: true** field to the configuration file to enable this option. | | -A/--act/--auto-create-tenant | No | bool | false | Specifies whether to enable OBD to create the `test` tenant during the bootstrap by using all available resources of the cluster. If this option is enabled, the **auto_create_tenant: true** field will be added to the corresponding configuration file. You can also add the **auto_create_tenant: true** field to the configuration file to enable this option. | -## obd cluster start +## `obd cluster start` Starts a deployed cluster. If the cluster is started, OBD will return its status. ```shell obd cluster start [flags] - -# example -obd cluster start test -S ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. This table describes the corresponding options. | Option | Required | Data type | Default value | Description | -|----|-----|-----|----|----| -| -s/--servers | No | string | Empty | A list of machines, followed by the `name` value corresponding to `servers` in the `yaml` file, separated by `,`. If the `name` value is not configured after `servers`, the `ip` value is used. Be used for specifying the machines need to be started. If this option is disabled, all machines under the component will start without executing bootstrap. | -| -c/--components | No | string | Empty | A list of components, separated by `,`. Be used for specifying the components need to be started. If this option is disabled, all machines under the component will start without entering the running state. | +--- | --- | --- |--- | --- +| -s/--servers | No | string | | A list of machines, followed by the `name` value corresponding to `servers` in the `yaml` file, separated by `,`. Be used for specifying the start-up machines. If this option is disabled, all machines under the component will start without executing bootstrap. | +| -c/--components | No | string | | A list of components, separated by `,`. Be used for specifying the start-up components. If this option is disabled, all machines under the component will start without entering the running state. | | --wop/--without-parameter | No | bool | false | Start without parameters. The node does not respond to this option when this node is starting for the first time. | | -S/--strict-check | No | bool | false | Some components will do relevant checks before starting. OBD will throw an error when the check fails, but OBD will not force the process to stop. Using this option can return an error and directly exit the process when the component pre-check fails. We recommend that you enable this option to avoid startup failures due to insufficient resources. | -## obd cluster list +## `obd cluster list` Shows the status of all clusters that have been registered to OBD. The cluster names are specified by the deploy name parameter. @@ -99,140 +85,107 @@ Shows the status of all clusters that have been registered to OBD. The cluster n obd cluster list ``` -## obd cluster display +## `obd cluster display` Shows the status of the specified cluster. ```shell obd cluster display - -# example -obd cluster display test ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. -## obd cluster reload +## `obd cluster reload` Reloads a running cluster. After you modify the configuration information of a running cluster by using the `edit-config` command, you can run the `reload` command to let your modification take effect. -> **NOTE:** -> -> Some configuration items may not take effect after you run the `reload` command. You need to restart or even redeploy the cluster for these configuration items to take effect. Do operations based on the result returned by the `edit-config` command. +> **NOTE:** Some configuration items may not take effect after you run the `reload` command. You need to restart or even redeploy the cluster for these configuration items to take effect. Do operations based on the result returned by the `edit-config` command. ```shell obd cluster reload - -# example -obd cluster reload test ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. -## obd cluster restart +## `obd cluster restart` Restarts a running cluster. By default, OBD restarts without any parameters. After you run the edit-config command to modify the configuration information of a running cluster, you can run the `restart` command for the modification to take effect. -> **NOTE:** -> -> Some configuration items may not take effect after you run the `restart` command. You even need to redeploy the cluster for some configuration items to take effect. Perform operations based on the result returned by the edit-config command. +> **NOTE:** Some configuration items may not take effect after you run the `restart` command. You even need to redeploy the cluster for some configuration items to take effect. Perform operations based on the result returned by the edit-config command. ```shell obd cluster restart - -# example -obd cluster restart test -c obproxy-ce --wp ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. This table describes the corresponding options. | Option | Required | Data type | Default value | Description | -|----|-----|-----|----|----| -| -s/--servers | No | string | Empty | A list of machines, followed by the `name` value corresponding to `servers` in the `yaml` file, separated by `,`. If the `name` value is not configured after `servers`, the `ip` value is used. Be used to specify the machines need to be restarted. | -| -c/--components | No | string | Empty | A list of components, separated by `,`. Be used to specify the components need to be restarted. If this option is disabled, all machines under the component will start without entering the running state. | +--- | --- | --- |--- | --- +| -s/--servers | No | string | | A list of machines, followed by the `name` value corresponding to `servers` in the `yaml` file, separated by `,`. | +| -c/--components | No | string | | A list of components, separated by `,`. Be used for specifying the start-up components. If this option is disabled, all machines under the component will start without entering the running state. | | --wp/--with-parameter | No | bool | false | Restarts OBD with parameters. This option makes the parameters valid when you restart OBD. | -## obd cluster redeploy +## `obd cluster redeploy` Redeploys a running cluster. After you run the `edit-config` command to modify the configuration information of a running cluster, you can run the `redeploy` command to let your modification take effect. -> **NOTE:** -> -> This command destroys the cluster and redeploys it. Data in the cluster will be lost. Please back up the data before you run this command. +> **NOTE:** This command destroys the cluster and redeploys it. Data in the cluster will be lost. Please back up the data before you run this command. ```shell -obd cluster redeploy [-f] - -# example -obd cluster redeploy test -f +obd cluster redeploy ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. - -Before OBD redeploys the cluster, it will check for running processes. These processes may result from the failure of the `obd cluster start` command. They may also belong to other clusters when configurations of this cluster overlap with those of other clusters. If an ongoing process is found in the working directory, OBD will stop the `obd cluster redeploy` command. - -`-f` is `--force-kill`. This option specifies whether to forcibly stop running processes in the working directory. If this option is enabled, OBD will forcibly stop the ongoing processes and run the `obd cluster redeploy` command. `-f` is optional. Its data type is `bool`. This option is disabled by default. +`deploy name` specifies the name of the deployment configuration file. -## obd cluster stop +## `obd cluster stop` Stops a running cluster. ```shell obd cluster stop - -# example -obd cluster stop test -s server1 ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. This table describes the corresponding options. | Option | Required | Data type | Default value | Description | -|----|-----|-----|----|----| -| -s/--servers | No | string | Empty | A list of machines, followed by the `name` value corresponding to `servers` in the `yaml` file, separated by `,`. If the `name` value is not configured after `servers`, the `ip` value is used. Be used to specify the machines need to be stopped | -| -c/--components | No | string | Empty | A list of components, separated by `,`. Be used to specify the components need to be stopped. If not all components under the configuration start, this configuration will not enter the stopped state. | +--- | --- | --- |--- | --- +| -s/--servers | No | string | | A list of machines, followed by the `name` value corresponding to `servers` in the `yaml` file, separated by `,`. Be used for specifying the start-up machines. | +| -c/--components | No | string | | A list of components, separated by `,`. Be used for specifying the start-up components. If not all components under the configuration start, this configuration will not enter the stopped state. | -## obd cluster destroy +## `obd cluster destroy` Destroys a deployed cluster. If the cluster is running state, this command will first try to execute `stop` and then `destroy` after success. ```shell obd cluster destroy [-f] - -# example -obd cluster destroy test -f ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. - -Before OBD destroys the cluster, it will check for running processes. These processes may result from the failure of the `obd cluster start` command. They may also belong to other clusters when configurations of this cluster overlap with those of other clusters. If an ongoing process is found in the working directory, OBD will stop the `obd cluster destroy` command. +`deploy name` specifies the name of the deployment configuration file. -`-f` is `--force-kill`. This option specifies whether to forcibly stop running processes in the working directory. If this option is enabled, OBD will forcibly stop the ongoing processes and run the `obd cluster destroy` command. `-f` is optional. Its data type is `bool`. This option is disabled by default. +`-f` is `--force-kill`. This option specifies whether to forcibly stop running processes in the working directory. Before OBD destroys the cluster, it will check for running processes. These processes may result from the failure of the **start** command. They may also belong to other clusters when configurations of this cluster overlap with those of other clusters. If an ongoing process is found in the working directory, OBD will stop the **destroy** command. However, if this option is enabled, OBD will forcibly stop the ongoing processes and run the **destroy** command. `-f` is optional. Its data type is `bool`. This option is disabled by default. -## obd cluster upgrade +## `obd cluster upgrade` Update a running component. ```shell obd cluster upgrade -c -V [tags] - -# example -obd cluster upgrade test -c oceanbase-ce -V 4.0.0.0 --usable=c63bb73384b17d74299b34fe3aceb0ae310fd319d2ccdb1acd39f31ba6673198 ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. | Option | Required | Data type | Default value | Description | -|----|-----|-----|----|----| -| -c/--component | Yes | string | Empty | The component name you want to upgrade. | -| -V/--version | Yes | string | Empty | The target upgrade version number. | -| --skip-check | No | bool | false | Skip check. | -| --usable | No | string | Empty | The hash list for the mirrors that you use during upgrade. Separated with `,`. | -| --disable | No | string | Empty | The hash list for the mirrors that you disable during upgrade. Separated with `,`. | -| -e/--executer-path | No | string | /usr/obd/lib/executer | The executer path for the upgrade script. | +--- | --- | --- |--- |--- +-c/--component | Yes | string | empty | The component name you want to upgrade. +-V/--version | Yes | string | empty | The target upgrade version number. +--skip-check | No | bool | false | Skip check. +--usable | No | string | empty | The hash list for the mirrors that you use during upgrade. Separated with `,`. +--disable | No | string | empty | The hash list for the mirrors that you disable during upgrade. Separated with `,`. +-e/--executer-path | No | string | /usr/obd/lib/executer | The executer path for the upgrade script. ## obd cluster reinstall @@ -240,25 +193,6 @@ You can run this command to reinstall the repository of a deployed component. Th ```bash obd cluster reinstall -c --hash [-f/--force] - -# example -[admin@test ~]$ obd mirror list -+------------------------------------------------------------------+ -| Mirror Repository List | -+----------------------------+--------+---------+------------------+ -| SectionName | Type | Enabled | Update Time | -+----------------------------+--------+---------+------------------+ -| oceanbase.community.stable | remote | True | 2022-12-16 10:41 | -| oceanbase.development-kit | remote | True | 2022-12-16 10:41 | -| local | local | - | 2022-12-16 10:52 | -+----------------------------+--------+---------+------------------+ -Use `obd mirror list
` for more details -[admin@test ~]$ obd mirror list oceanbase.community.stable | grep -e " oceanbase-ce " | grep -e " 4.0.0.0 " -| oceanbase-ce | 4.0.0.0 | 100000272022110114.el7 | x86_64 | 759074414c7b7b723013855353f62a7ba0aae0f493216ef2511825850ce77b51 | -| oceanbase-ce | 4.0.0.0 | 100000282022112511.el7 | x86_64 | debb18ab3c0b3d16f145c41cd21c30686863580b721d45ddaa068e6309e03b64 | -| oceanbase-ce | 4.0.0.0 | 102000032022120718.el7 | x86_64 | c63bb73384b17d74299b34fe3aceb0ae310fd319d2ccdb1acd39f31ba6673198 | - -[admin@test ~]$ obd cluster reinstall test -c oceanbase-ce --hash=c63bb73384b17d74299b34fe3aceb0ae310fd319d2ccdb1acd39f31ba6673198 ``` The `deploy name` parameter indicates the name of the deployed cluster, which is also the alias of the configuration file. @@ -266,64 +200,65 @@ The `deploy name` parameter indicates the name of the deployed cluster, which is | Option name | Required | Data type | Default value | Description | |---------|----------|-------------|-------------|--------------| | -c/--component | Yes | string | Null | The name of the component whose repository is to be replaced. | -|--hash | Yes | string | Null | The hash value of the target repository. The target repository must be of the same version as the current repository. | +|--hash | Yes | string | Null | The target repository. It must be of the same version as the current repository. | | -f/--force | No | Bool | false | Specifies whether to enable forced replacement even if the restart fails. | -## obd cluster tenant create +## `obd cluster tenant create` Creates a tenant. This command applies only to an OceanBase cluster. This command automatically creates resource units and resource pools. ```shell obd cluster tenant create [-n ] [flags] - -# example -obd cluster tenant create test -n obmysql --max-cpu=2 --memory-size=2G --log-disk-size=3G --max-iops=10000 --iops-weight=2 --unit-num=1 --charset=utf8 ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. This table describes the corresponding options. | Option | Required | Data type | Default value | Description | -|----|-----|-----|----|----| -| -t/-n/--tenant-name | No | string | test | The tenant name. OBD will automatically generate resource units and resource pools with unique names based on the tenant name. | -| --max-cpu | No | float | 0 | The maximum number of CPU cores available for the tenant. When this option is set to 0, all available CPU cores of the cluster can be used by the tenant. | -| --min-cpu | No | float | 0 | The minimum number of CPU cores available for the tenant. When this option is set to 0, the minimum number of CPU cores is the same as the maximum number of CPU cores. | -| --max-memory | No | int | 0 | The maximum memory capacity available for the tenant. When this option is set to 0, all available memory capacity of the cluster can be used by the tenant. When the actual value is less than 1 GB, an error is returned.
Not supported after the OceanBase database V4.0.0.0. You can use `--memory-size` instead.
| -| --min-memory | No | int | 0 | The minimum memory capacity available for the tenant. When this option is set to 0, the minimum memory capacity is the same as the maximum memory capacity.
Not supported after the OceanBase database V4.0.0.0. You can use `--memory-size` instead.
| -| --memory-size | No | int | 0 | The available memory unit size of the tenant. Supported since the OceanBase database V4.0.0.0. | -| --max-disk-size | No | int | 0 | The maximum disk space available for the tenant. When this option is set to 0, all available disk space of the cluster can be used by the tenant. If the actual value is less than 512 MB, an error is returned.
Not supported after the OceanBase database V4.0.0.0.
| -| --log-disk-size | No | int | 0 | Specifies the tenant's Unit log disk size. The default value is 3 times the memory specification value. The minimum value is `2G`. | -| --max-iops | No | int |
  • The default value is 128 when the OceanBase database version is lower than V4.0.0.0.
  • The default value is 1024 after the OceanBase database V4.0.0.0.
| The maximum IOPS for the tenant. The value range of this parameter can be divided into the following two cases according to the OceanBase version.
  • Value range is [128,+∞) when the OceanBase database version is lower than V4.0.0.0.
  • Value range is [1024,+∞) after the OceanBase database V4.0.0.0.
| -| --min-iops | No | int | 0 | The minimum IOPS for the tenant. The value range is the same as `--max-iops`. When this option is set to 0, the minimum IOPS is the same as the maximum IOPS. | -| --iops-weight | No | int | 0 | Specifies the weight of the tenant's IOPS. Supported since the OceanBase database V4.0.0.0. | -| --max-session-num | No | int | 64 | The maximum number of sessions allowed for the tenant. Value range: [64, +∞).
Not supported after the OceanBase database V4.0.0.0.
| +--- | --- | --- |--- | --- +| -n/--tenant-name | No | string | test | The tenant name. OBD will automatically generate resource units and resource pools with unique names based on the tenant name. | +| --max-cpu | No | float | 0 | The maximum number of CPU cores available for the tenant. When this option is set to 0, all available CPU cores of the cluster can be used by the tenant. | +| --min-cpu | No | float | 0 | The minimum number of CPU cores available for the tenant. When this option is set to 0, the minimum number of CPU cores is the same as the maximum number of CPU cores. | +| --max-memory | No | int | 0 | The maximum memory capacity available for the tenant. When this option is set to 0, all available memory capacity of the cluster can be used by the tenant. When the actual value is less than 1 GB, an error is returned. | +| --min-memory | No | int | 0 | The minimum memory capacity available for the tenant. When this option is set to 0, the minimum memory capacity is the same as the maximum memory capacity. | +| --max-disk-size | No | int | 0 | The maximum disk space available for the tenant. When this option is set to 0, all available disk space of the cluster can be used by the tenant. If the actual value is less than 512 MB, an error is returned. | +| --max-iops | No | int | 128 | The maximum IOPS for the tenant. Value range: [128, +∞). | +| --min-iops | No | int | 0 | The minimum IOPS for the tenant. Value range: [128, +∞). When this option is set to 0, the minimum IOPS is the same as the maximum IOPS. | +| --max-session-num | No | int | 64 | The maximum number of sessions allowed for the tenant. Value range: [64, +∞). | | --unit-num | No | int | 0 | The number of units to be created in a zone. It must be less than the number of OBServers in the zone. When this option is set to 0, the maximum value is used. | -| -z/--zone-list | No | string | Empty | Specifies the list of zones of the tenant. Separate multiple zones with commas (,). If this option is not specified, all zones of the cluster are included. | +| -z/--zone-list | No | string | | Specifies the list of zones of the tenant. Separate multiple zones with commas (,). If this option is not specified, all zones of the cluster are included. | | --primary-zone | No | string | RANDOM | The primary zone of the tenant. | -| --charset | No | string | Empty | The character set of the tenant. | -| --collate | No | string | Empty | The collation of the tenant. | +| --charset | No | string | | The character set of the tenant. | +| --collate | No | string | | The collation of the tenant. | | --replica-num | No | int | 0 | The number of replicas of the tenant. When this option is set to 0, the number of replicas is the same as that of zones. | | --logonly-replica-num | No | string | 0 | The number of log replicas of the tenant. When this option is set to 0, the number of log replicas is the same as that of replicas. | -| --tablegroup | No | string | Empty | The default table group of the tenant. | -| --locality | No | string | Empty | The distribution status of replicas across zones. For example, F@z1,F@z2,F@z3,R@z4 means that z1, z2, and z3 are full-featured replicas and z4 is a read-only replica. | +| --tablegroup | No | string | | The default table group of the tenant. | +| --locality | No | string | | The distribution status of replicas across zones. For example, F@z1,F@z2,F@z3,R@z4 means that z1, z2, and z3 are full-featured replicas and z4 is a read-only replica. | | -s/--variables | No | string | ob_tcp_invited_nodes='%' | The system variables of the tenant. | -## obd cluster tenant drop +## `obd cluster tenant drop` Deletes a tenant. This command applies only to an OceanBase cluster. This command automatically deletes the corresponding resource units and resource pools. ```shell obd cluster tenant drop [-n ] - -# example -obd cluster tenant drop test -n obmysql ``` -The `deploy name` parameter specifies the name of the deployed cluster. You can consider it as an alias for the configuration file. +`deploy name` specifies the name of the deployment configuration file. `-n` is `--tenant-name`. This option specifies the name of the tenant to be deleted. This option is required. +## `obd cluster tenant list` + +list all tenant. This command applies only to an OceanBase cluster. + +```shell +obd cluster tenant list +``` + +`deploy name` specifies the name of the deployment configuration file. + ## obd cluster chst You can run this command to change the configuration style. @@ -331,8 +266,6 @@ You can run this command to change the configuration style. ```shell obd cluster chst --style