From 5a5bf043dcb864d6cd706a1804c8d20785157cc9 Mon Sep 17 00:00:00 2001 From: Rongfeng Fu Date: Thu, 24 Nov 2022 10:53:30 +0800 Subject: [PATCH] v1.6.1 (#146) --- _cmd.py | 5 +- _deploy.py | 2 + _errno.py | 2 + _repository.py | 14 ++++-- core.py | 8 ++-- .../3.user-guide/3.obd-command/0.obd-demo.md | 41 ++++++++++++++++ optimize/obproxy/4.0.0/sysbench.yaml | 24 ++++++++++ optimize/obproxy/4.0.0/tpcc.yaml | 24 ++++++++++ .../7.5.17/oceanbase-metrics_rev1.json | 10 ++-- plugins/mysqltest/3.1.0/init.py | 3 +- plugins/mysqltest/4.0.0.0/init.py | 2 +- plugins/obproxy/3.1.0/display.py | 4 +- plugins/obproxy/3.1.0/file_map.yaml | 2 +- plugins/obproxy/3.1.0/generate_config.py | 2 +- plugins/oceanbase/3.1.0/create_tenant.py | 2 +- plugins/oceanbase/3.1.0/display.py | 2 +- plugins/oceanbase/3.1.0/generate_config.py | 35 ++++++++++---- plugins/oceanbase/3.1.0/reload.py | 5 +- plugins/oceanbase/3.1.0/start_check.py | 47 ++++++++++++++----- plugins/oceanbase/4.0.0.0/create_tenant.py | 2 +- plugins/oceanbase/4.0.0.0/generate_config.py | 45 ++++++++++++------ plugins/oceanbase/4.0.0.0/start_check.py | 45 +++++++++++++----- plugins/sysbench/3.1.0/pre_test.py | 2 +- plugins/sysbench/4.0.0.0/pre_test.py | 2 +- plugins/tpch/3.1.0/run_test.py | 2 +- plugins/tpch/4.0.0.0/run_test.py | 2 +- ssh.py | 8 ++-- 27 files changed, 261 insertions(+), 81 deletions(-) create mode 100644 docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md create mode 100644 optimize/obproxy/4.0.0/sysbench.yaml create mode 100644 optimize/obproxy/4.0.0/tpcc.yaml diff --git a/_cmd.py b/_cmd.py index ae249e5..6ad96f0 100644 --- a/_cmd.py +++ b/_cmd.py @@ -854,6 +854,7 @@ class ClusterTenantCreateCommand(ClusterMirrorCommand): self.parser.add_option('--max-session-num', type='int', help="Max session unit number. Not supported after version 4.0") self.parser.add_option('--unit-num', type='int', help="Pool unit number.") self.parser.add_option('-z', '--zone-list', type='string', help="Tenant zone list.") + self.parser.add_option('--mode', type='string', help='Tenant compatibility mode. {mysql,oracle} [mysql]', default='mysql') self.parser.add_option('--charset', type='string', help="Tenant charset.") self.parser.add_option('--collate', type='string', help="Tenant COLLATE.") self.parser.add_option('--replica-num', type='int', help="Tenant replica number.") @@ -925,6 +926,8 @@ class MySQLTestCommand(TestMirrorCommand): def __init__(self): super(MySQLTestCommand, self).__init__('mysqltest', 'Run a mysqltest for a deployment.') + self.parser.add_option('--mode', type='string', help='Test mode. Available values are mysql, oracle, and both.', default='both') + # self.parser.add_option('--case-mode', type='string', help='case run mode [mysql,oracle]', default='mysql') self.parser.add_option('--component', type='string', help='Components for mysqltest.') self.parser.add_option('--test-server', type='string', help='The server for mysqltest. By default, the first root server in the component is the mysqltest server.') self.parser.add_option('--user', type='string', help='Username for a test. [admin]', default='admin') @@ -1042,7 +1045,7 @@ class TPCHCommand(TestMirrorCommand): self.parser.add_option('--remote-tbl-dir', type='string', help='Directory for the tbl on target observers. Make sure that you have read and write access to the directory when you start observer.') self.parser.add_option('--disable-transfer', '--dt', action='store_true', help='Disable the transfer. When enabled, OBD will use the tbl files under remote-tbl-dir instead of transferring local tbl files to remote remote-tbl-dir.') self.parser.add_option('--dss-config', type='string', help='Directory for dists.dss. [/usr/tpc-h-tools/tpc-h-tools]', default='/usr/tpc-h-tools/tpc-h-tools/') - self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1}. [1]', default=1) + self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1/2}. [1] 0 - No optimization. 1 - Optimize some of the parameters which do not need to restart servers. 2 - Optimize all the parameters and maybe RESTART SERVERS for better performance.', default=1) self.parser.add_option('--test-only', action='store_true', help='Only testing SQLs are executed. No initialization is executed.') self.parser.add_option('-S', '--skip-cluster-status-check', action='store_true', help='Skip cluster status check', default=False) diff --git a/_deploy.py b/_deploy.py index cb32fa3..7f51587 100644 --- a/_deploy.py +++ b/_deploy.py @@ -625,6 +625,8 @@ class ClusterConfig(object): return error def set_global_conf(self, conf): + if not isinstance(conf, dict): + raise Exception('%s global config is not a dictionary. Please check the syntax of your configuration file.\n See https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md' % self.name) self._original_global_conf = deepcopy(conf) self._global_conf = None self._clear_cache_server() diff --git a/_errno.py b/_errno.py index cf22afe..b1c09e2 100644 --- a/_errno.py +++ b/_errno.py @@ -62,6 +62,8 @@ EC_FAIL_TO_CONNECT = OBDErrorCode(1006, 'Failed to connect to {component}') EC_ULIMIT_CHECK = OBDErrorCode(1007, '({server}) {key} must not be less than {need} (Current value: {now})') EC_OBSERVER_NOT_ENOUGH_MEMORY = OBDErrorCode(2000, '({ip}) not enough memory. (Free: {free}, Need: {need})') +EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE = OBDErrorCode(2000, '({ip}) not enough memory. (Available: {available}, Need: {need})') +EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED = OBDErrorCode(2000, '({ip}) not enough memory. (Free: {free}, Buff/Cache: {cached}, Need: {need})') EC_OBSERVER_CAN_NOT_MIGRATE_IN = OBDErrorCode(2001, 'server can not migrate in') EC_OBSERVER_FAIL_TO_START = OBDErrorCode(2002, 'Failed to start {server} observer') EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG = OBDErrorCode(2003, '({ip}) {path} not enough disk space for clog. Use redo_dir to set other disk for clog, or reduce the value of datafile_size') diff --git a/_repository.py b/_repository.py index 6004717..7830a4b 100644 --- a/_repository.py +++ b/_repository.py @@ -172,7 +172,9 @@ class ParallerExtractWorker(object): class ParallerExtractor(object): - MAX_PARALLER = cpu_count() if cpu_count() else 8 + MAX_PARALLER = cpu_count() * 2 if cpu_count() else 8 + MAX_SIZE = 100 + MIN_SIZE = 20 def __init__(self, pkg, files, stdio=None): self.pkg = pkg @@ -180,11 +182,13 @@ class ParallerExtractor(object): self.stdio = stdio def extract(self): + if not self.files: + return workers = [] file_num = len(self.files) - paraler = int(min(self.MAX_PARALLER, file_num)) - size = min(100, int(file_num / paraler)) - size = int(max(10, size)) + paraller = int(min(self.MAX_PARALLER, file_num)) + size = min(self.MAX_SIZE, int(file_num / paraller)) # + size = int(max(self.MIN_SIZE, size)) index = 0 while index < file_num: p_index = index + size @@ -195,7 +199,7 @@ class ParallerExtractor(object): )) index = p_index - pool = Pool(processes=paraler) + pool = Pool(processes=paraller) try: results = pool.map(ParallerExtractWorker.extract, workers) for r in results: diff --git a/core.py b/core.py index d25f9a3..4bcc5d3 100644 --- a/core.py +++ b/core.py @@ -890,7 +890,7 @@ class ObdHome(object): self._call_stdio('verbose', 'Get deploy configuration') deploy_config = deploy.deploy_config if not deploy_config: - self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.') + self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md') return False # Check the best suitable mirror for the components and installation plugins. Install locally @@ -1061,7 +1061,7 @@ class ObdHome(object): return False deploy_config = deploy.deploy_config if not deploy_config: - self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.') + self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md') return False style = getattr(options, 'style', '') @@ -1211,11 +1211,11 @@ class ObdHome(object): self._call_stdio('verbose', 'Get deploy configuration') deploy_config = deploy.deploy_config if not deploy_config: - self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.') + self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md') return False if not deploy_config.components: - self._call_stdio('error', 'Components not detected.\nPlease check the syntax of your configuration file.') + self._call_stdio('error', 'Components not detected.\nPlease check the syntax of your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md') return False for component_name in deploy_config.components: diff --git a/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md b/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md new file mode 100644 index 0000000..cd8f123 --- /dev/null +++ b/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md @@ -0,0 +1,41 @@ +# Quick deployment command + +## obd demo + +You can use this command to deploy and start the specified component on the local server without loading the configuration file. The fixed name of the cluster deployed is `demo`. After the deployment, you can run the `obd cluster list` command to view the cluster in the cluster list. You can also run other cluster commands, such as `obd cluster display demo`, to manage the cluster. + +```bash +obd demo [-c/--components] +``` + +The following table describes the parameters. + +| Parameter | Required | Data type | Default value | Description | +|------------------|---------|------------|----------|--------------------------------------------------------------------| +| -c/--components | No | String | oceanbase-ce,obproxy-ce,obagent,prometheus,grafana | The list of components that are separated with commas (`,`). You can use this parameter to specify the components to be deployed. | + +By default, this command deploys the minimum specifications in the home directory of the current user, and the latest versions are deployed by default. You can use this command to deploy OceanBase Community Edition, OBProxy Community Edition, OBAgent, Grafana, and Prometheus. + +You can select the version and specify the configurations of a component to be deployed. + +```bash +# Deploy components of the specified version. +obd demo -c oceanbase-ce,obproxy-ce --oceanbase-ce.version=3.1.3 +# Specify the components to be deployed and the package hash of OceanBase Community Edition. +obd demo -c oceanbase-ce,obproxy-ce --oceanbase-ce.package_hash=f38723204d49057d3e062ffad778edc1552a7c114622bf2a86fea769fbd202ea +# Specify the installation path for all components to be deployed. +## Deploy OceanBase Community Edition and OBProxy Community Edition in the /data/demo directory and create corresponding working directories for them. +obd demo -c oceanbase-ce,obproxy-ce --home_path=/data/demo +# Specify the installation path for all components to be deployed. +obd demo --home_path=/path +# Specify the installation path for a specific component to be deployed. +## Deploy OceanBase Community Edition in the home directory and create a working directory for it, and deploy OBProxy Community Edition in the /data/playground/obproxy-ce directory. +obd demo -c oceanbase-ce,obproxy-ce --obproxy-ce.home_path=/data/demo/ +# Specify the configurations of a component to be deployed. +## Specify the mysql_port parameter of OceanBase Community Edition. +obd demo --oceanbase-ce.mysql_port=3881 +``` + +> **Notice** +> +> This command supports only level-1 configurations under global that are specified by using options. diff --git a/optimize/obproxy/4.0.0/sysbench.yaml b/optimize/obproxy/4.0.0/sysbench.yaml new file mode 100644 index 0000000..0ada4cc --- /dev/null +++ b/optimize/obproxy/4.0.0/sysbench.yaml @@ -0,0 +1,24 @@ +test: + system_config: + - name: proxy_mem_limited + value: 2G + - name: slow_proxy_process_time_threshold + value: 500ms + - name: syslog_level + value: error + query_key: level + - name: enable_prometheus + value: false + - name: enable_compression_protocol + value: false + need_restart: true + value_type: BOOL + - name: enable_ob_protocol_v2 + value: false + need_restart: true + value_type: BOOL + - name: work_thread_num + value: 128 + need_restart: true + - name: enable_async_log + value: true \ No newline at end of file diff --git a/optimize/obproxy/4.0.0/tpcc.yaml b/optimize/obproxy/4.0.0/tpcc.yaml new file mode 100644 index 0000000..782bbd0 --- /dev/null +++ b/optimize/obproxy/4.0.0/tpcc.yaml @@ -0,0 +1,24 @@ +build: + system_config: + - name: proxy_mem_limited + value: 2G + - name: slow_proxy_process_time_threshold + value: 500ms + - name: syslog_level + value: error + query_key: level + - name: enable_prometheus + value: false + - name: enable_compression_protocol + value: false + need_restart: true + value_type: BOOL + - name: enable_ob_protocol_v2 + value: false + need_restart: true + value_type: BOOL + - name: work_thread_num + value: 128 + need_restart: true + - name: enable_async_log + value: true \ No newline at end of file diff --git a/plugins/grafana/7.5.17/oceanbase-metrics_rev1.json b/plugins/grafana/7.5.17/oceanbase-metrics_rev1.json index 81ba4bd..15cd3a2 100644 --- a/plugins/grafana/7.5.17/oceanbase-metrics_rev1.json +++ b/plugins/grafana/7.5.17/oceanbase-metrics_rev1.json @@ -218,11 +218,11 @@ "exemplar": true, "expr": "(sum(rate(ob_sysstat{stat_id=\"40003\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40009\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40009\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40001\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group))\n/\n(sum(rate(ob_sysstat{stat_id=\"40002\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40004\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40008\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40000\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group))", "interval": "", - "legendFormat": "qps rt {{$group}}", + "legendFormat": "sql latency {{$group}}", "refId": "A" } ], - "title": "QPS rt", + "title": "Sql Latency", "type": "timeseries" }, { @@ -2129,7 +2129,7 @@ "exemplar": true, "expr": "(sum(rate(ob_sysstat{stat_id=\"10005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"10006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)) / sum(rate(ob_sysstat{stat_id=\"10000\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)", "interval": "", - "legendFormat": "roc in rt {{$group}}", + "legendFormat": "rpc in Latency {{$group}}", "refId": "A" }, { @@ -2137,11 +2137,11 @@ "expr": "(sum(rate(ob_sysstat{stat_id=\"10005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"10006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)) / sum(rate(ob_sysstat{stat_id=\"10002\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)", "hide": false, "interval": "", - "legendFormat": "rpc out rt {{$group}}", + "legendFormat": "rpc out latency {{$group}}", "refId": "B" } ], - "title": "Rpc rt", + "title": "Rpc Latency", "type": "timeseries" }, { diff --git a/plugins/mysqltest/3.1.0/init.py b/plugins/mysqltest/3.1.0/init.py index 081f7ea..0ab0593 100644 --- a/plugins/mysqltest/3.1.0/init.py +++ b/plugins/mysqltest/3.1.0/init.py @@ -36,7 +36,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -127,6 +127,7 @@ def init(plugin_context, env, *args, **kwargs): exec_init = 'init.sql' exec_mini_init = 'init_mini.sql' exec_init_user = 'init_user.sql|root@mysql|test' + exec_init_user_for_oracle = 'init_user_oracle.sql|SYS@oracle|SYS' client = plugin_context.clients[server] memory_limit = get_memory_limit(cursor, client) is_mini = memory_limit and parse_size(memory_limit) < (16<<30) diff --git a/plugins/mysqltest/4.0.0.0/init.py b/plugins/mysqltest/4.0.0.0/init.py index cd8f04a..ff25721 100644 --- a/plugins/mysqltest/4.0.0.0/init.py +++ b/plugins/mysqltest/4.0.0.0/init.py @@ -36,7 +36,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/obproxy/3.1.0/display.py b/plugins/obproxy/3.1.0/display.py index e6fb096..2310ca7 100644 --- a/plugins/obproxy/3.1.0/display.py +++ b/plugins/obproxy/3.1.0/display.py @@ -56,12 +56,12 @@ def display(plugin_context, cursor, *args, **kwargs): continue password = ob_config.get('root_password', '') with_observer = True - cmd = 'obclient -h%s -P%s -uroot %s-Doceanbase' % (server.ip, server_config['listen_port'], '-p%s ' % password if password else '') + cmd = 'obclient -h%s -P%s -uroot %s-Doceanbase -A' % (server.ip, server_config['listen_port'], '-p%s ' % password if password else '') break if not with_observer: password = server_config.get('obproxy_sys_password', '') - cmd = 'obclient -h%s -P%s -uroot@proxysys %s-Doceanbase' % (server.ip, server_config['listen_port'], '-p%s ' % password if password else '') + cmd = 'obclient -h%s -P%s -uroot@proxysys %s-Doceanbase -A' % (server.ip, server_config['listen_port'], '-p%s ' % password if password else '') stdio.print(cmd) diff --git a/plugins/obproxy/3.1.0/file_map.yaml b/plugins/obproxy/3.1.0/file_map.yaml index b7a1393..34ec912 100644 --- a/plugins/obproxy/3.1.0/file_map.yaml +++ b/plugins/obproxy/3.1.0/file_map.yaml @@ -1,4 +1,4 @@ -- src_path: ./home/admin/obproxy-$version/bin/obproxy +- src_path: ./opt/taobao/install/obproxy-$version/bin/obproxy target_path: bin/obproxy type: bin mode: 755 \ No newline at end of file diff --git a/plugins/obproxy/3.1.0/generate_config.py b/plugins/obproxy/3.1.0/generate_config.py index 73bc2c5..08def59 100644 --- a/plugins/obproxy/3.1.0/generate_config.py +++ b/plugins/obproxy/3.1.0/generate_config.py @@ -47,7 +47,7 @@ def generate_config(plugin_context, deploy_config, auto_depend=False, *args, **k if getattr(plugin_context.options, 'mini', False): if 'proxy_mem_limited' not in global_config: - cluster_config.update_global_conf('proxy_mem_limited', '200M', False) + cluster_config.update_global_conf('proxy_mem_limited', '500M', False) ob_comps = ['oceanbase', 'oceanbase-ce'] ob_cluster_config = None diff --git a/plugins/oceanbase/3.1.0/create_tenant.py b/plugins/oceanbase/3.1.0/create_tenant.py index 9ce6ba8..f3e16d4 100644 --- a/plugins/oceanbase/3.1.0/create_tenant.py +++ b/plugins/oceanbase/3.1.0/create_tenant.py @@ -35,7 +35,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'^([1-9][0-9]*)\s*([B,K,M,G,T])$', size.upper()) + match = re.match(r'^(0|[1-9][0-9]*)\s*([B,K,M,G,T])$', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/oceanbase/3.1.0/display.py b/plugins/oceanbase/3.1.0/display.py index 3d7cc24..a2a12db 100644 --- a/plugins/oceanbase/3.1.0/display.py +++ b/plugins/oceanbase/3.1.0/display.py @@ -36,7 +36,7 @@ def display(plugin_context, cursor, *args, **kwargs): stdio.print_list(servers, ['ip', 'version', 'port', 'zone', 'status'], lambda x: [x['svr_ip'], x['build_version'].split('_')[0], x['inner_port'], x['zone'], x['status']], title='observer') password = cluster_config.get_global_conf().get('root_password', '') - cmd = 'obclient -h%s -P%s -uroot %s-Doceanbase' % (servers[0]['svr_ip'], servers[0]['inner_port'], '-p%s ' % password if password else '') + cmd = 'obclient -h%s -P%s -uroot %s-Doceanbase -A' % (servers[0]['svr_ip'], servers[0]['inner_port'], '-p%s ' % password if password else '') stdio.print(cmd) stdio.stop_loading('succeed') return plugin_context.return_true() diff --git a/plugins/oceanbase/3.1.0/generate_config.py b/plugins/oceanbase/3.1.0/generate_config.py index d3c29a5..5e453f7 100644 --- a/plugins/oceanbase/3.1.0/generate_config.py +++ b/plugins/oceanbase/3.1.0/generate_config.py @@ -23,7 +23,7 @@ from __future__ import absolute_import, division, print_function import re, os -from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY +from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE, EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED def parse_size(size): @@ -32,7 +32,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -90,6 +90,7 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): MIN_MEMORY = 8 << 30 MIN_CPU_COUNT = 16 + START_NEED_MEMORY = 3 << 30 clog_disk_utilization_threshold_max = 95 clog_disk_usage_limit_percentage_max = 98 global_config = cluster_config.get_original_global_conf() @@ -158,16 +159,32 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): if not server_config.get('memory_limit'): ret = client.execute_command('cat /proc/meminfo') if ret: - free_memory = 0 + server_memory_stats = {} + memory_key_map = { + 'MemTotal': 'total', + 'MemFree': 'free', + 'MemAvailable': 'available', + 'Buffers': 'buffers', + 'Cached': 'cached' + } + for key in memory_key_map: + server_memory_stats[memory_key_map[key]] = 0 for k, v in re.findall('(\w+)\s*:\s*(\d+\s*\w+)', ret.stdout): - if k == 'MemAvailable': - free_memory = parse_size(str(v)) - memory_limit = free_memory - if memory_limit < MIN_MEMORY: - stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(MIN_MEMORY))) + if k in memory_key_map: + key = memory_key_map[k] + server_memory_stats[key] = parse_size(str(v)) + + if server_memory_stats['available'] < START_NEED_MEMORY: + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE.format(ip=ip, available=format_size(server_memory_stats['available']), need=format_size(START_NEED_MEMORY))) success = False continue - memory_limit = max(MIN_MEMORY, memory_limit * 0.9) + + if server_memory_stats['free'] + server_memory_stats['buffers'] + server_memory_stats['cached'] < MIN_MEMORY: + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED.format(ip=ip, free=format_size(server_memory_stats['free']), cached=format_size(server_memory_stats['buffers'] + server_memory_stats['cached']), need=format_size(MIN_MEMORY))) + success = False + continue + + memory_limit = max(MIN_MEMORY, server_memory_stats['available'] * 0.9) server_config['memory_limit'] = format_size(memory_limit, 0) cluster_config.update_server_conf(server, 'memory_limit', server_config['memory_limit'], False) else: diff --git a/plugins/oceanbase/3.1.0/reload.py b/plugins/oceanbase/3.1.0/reload.py index 0fb6bd9..d0021af 100644 --- a/plugins/oceanbase/3.1.0/reload.py +++ b/plugins/oceanbase/3.1.0/reload.py @@ -115,10 +115,11 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): for server in servers: if key not in change_conf[server]: continue + value = change_conf[server][key] msg = sql = 'alter system set %s = %%s server=%%s' % key stdio.verbose('execute sql: %s' % msg) - cursor.execute(sql, [change_conf[server][key], cluster_server[server]]) - cluster_config.update_server_conf(server,key, value, False) + cursor.execute(sql, [value, cluster_server[server]]) + cluster_config.update_server_conf(server, key, value, False) except: global_ret = False stdio.exception('execute sql exception: %s' % msg) diff --git a/plugins/oceanbase/3.1.0/start_check.py b/plugins/oceanbase/3.1.0/start_check.py index 884695b..93d35f2 100644 --- a/plugins/oceanbase/3.1.0/start_check.py +++ b/plugins/oceanbase/3.1.0/start_check.py @@ -24,7 +24,11 @@ import os import re import time -from _errno import EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, EC_OBSERVER_NOT_ENOUGH_MEMORY, EC_ULIMIT_CHECK, WC_ULIMIT_CHECK +from _errno import ( + EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, + EC_OBSERVER_NOT_ENOUGH_MEMORY, EC_ULIMIT_CHECK, WC_ULIMIT_CHECK, + EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE, EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED +) stdio = None @@ -46,8 +50,9 @@ def parse_size(size): if not isinstance(size, str) or size.isdigit(): _bytes = int(size) else: + print (size) units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -90,6 +95,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): global success success = False stdio.error(*arg, **kwargs) + global stdio cluster_config = plugin_context.cluster_config clients = plugin_context.clients @@ -101,6 +107,8 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): servers_clog_mount = {} servers_net_inferface = {} server_num = len(cluster_config.servers) + START_NEED_MEMORY = 3 << 30 + stdio.start_loading('Check before start observer') for server in cluster_config.servers: ip = server.ip @@ -119,7 +127,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): servers_port[ip] = {} servers_clog_mount[ip] = {} servers_net_inferface[ip] = {} - servers_memory[ip] = {'num': 0, 'percentage': 0} + servers_memory[ip] = {'num': 0, 'percentage': 0, 'server_num': 0} memory = servers_memory[ip] ports = servers_port[ip] disk = servers_disk[ip] @@ -137,6 +145,8 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): } if get_port_socket_inode(client, port): critical('%s:%s port is already used' % (ip, port)) + + memory['server_num'] += 1 if 'memory_limit' in server_config: try: memory['num'] += parse_size(server_config['memory_limit']) @@ -234,16 +244,29 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): # memory ret = client.execute_command('cat /proc/meminfo') if ret: - total_memory = 0 - free_memory = 0 + server_memory_stats = {} + memory_key_map = { + 'MemTotal': 'total', + 'MemFree': 'free', + 'MemAvailable': 'available', + 'Buffers': 'buffers', + 'Cached': 'cached' + } + for key in memory_key_map: + server_memory_stats[memory_key_map[key]] = 0 for k, v in re.findall('(\w+)\s*:\s*(\d+\s*\w+)', ret.stdout): - if k == 'MemTotal': - total_memory = parse_size(str(v)) - elif k == 'MemAvailable': - free_memory = parse_size(str(v)) - total_use = servers_memory[ip]['percentage'] * total_memory / 100 + servers_memory[ip]['num'] - if total_use > free_memory: - error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(total_use))) + if k in memory_key_map: + key = memory_key_map[k] + server_memory_stats[key] = parse_size(str(v)) + + min_start_need = servers_memory[ip]['server_num'] * START_NEED_MEMORY + total_use = servers_memory[ip]['percentage'] * server_memory_stats['total'] / 100 + servers_memory[ip]['num'] + if min_start_need > server_memory_stats['available']: + error(EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE.format(ip=ip, available=format_size(server_memory_stats['available']), need=format_size(min_start_need))) + elif total_use > server_memory_stats['free'] + server_memory_stats['buffers'] + server_memory_stats['cached']: + error(EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED.format(ip=ip, free=format_size(server_memory_stats['free']), cached=format_size(server_memory_stats['buffers'] + server_memory_stats['cached']), need=format_size(min_start_need))) + elif total_use > server_memory_stats['free']: + alert(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(server_memory_stats['free']), need=format_size(min_start_need))) # disk disk = {'/': 0} ret = client.execute_command('df --block-size=1024') diff --git a/plugins/oceanbase/4.0.0.0/create_tenant.py b/plugins/oceanbase/4.0.0.0/create_tenant.py index 7999072..6315ccd 100644 --- a/plugins/oceanbase/4.0.0.0/create_tenant.py +++ b/plugins/oceanbase/4.0.0.0/create_tenant.py @@ -35,7 +35,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'^([1-9][0-9]*)\s*([B,K,M,G,T])$', size.upper()) + match = re.match(r'^(0|[1-9][0-9]*)\s*([B,K,M,G,T])$', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/oceanbase/4.0.0.0/generate_config.py b/plugins/oceanbase/4.0.0.0/generate_config.py index 15fb2ad..03271cc 100644 --- a/plugins/oceanbase/4.0.0.0/generate_config.py +++ b/plugins/oceanbase/4.0.0.0/generate_config.py @@ -23,7 +23,7 @@ from __future__ import absolute_import, division, print_function import re, os -from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY +from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE, EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED def parse_size(size): @@ -32,7 +32,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -106,6 +106,7 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): PRO_MEMORY_MIN = 16 << 30 SLOG_SIZE = 10 << 30 MIN_CPU_COUNT = 16 + START_NEED_MEMORY = 3 << 30 if getattr(plugin_context.options, 'mini', False): if not global_config.get('memory_limit_percentage') and not global_config.get('memory_limit'): cluster_config.update_global_conf('memory_limit', format_size(MIN_MEMORY, 0), False) @@ -161,18 +162,35 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): if not server_config.get('memory_limit'): ret = client.execute_command('cat /proc/meminfo') if ret: - free_memory = 0 + server_memory_stats = {} + memory_key_map = { + 'MemTotal': 'total', + 'MemFree': 'free', + 'MemAvailable': 'available', + 'Buffers': 'buffers', + 'Cached': 'cached' + } + for key in memory_key_map: + server_memory_stats[memory_key_map[key]] = 0 for k, v in re.findall('(\w+)\s*:\s*(\d+\s*\w+)', ret.stdout): - if k == 'MemAvailable': - free_memory = parse_size(str(v)) - memory_limit = free_memory - if memory_limit < min_memory: - stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(min_memory))) + if k in memory_key_map: + key = memory_key_map[k] + server_memory_stats[key] = parse_size(str(v)) + + if server_memory_stats['available'] < START_NEED_MEMORY: + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE.format(ip=ip, available=format_size(server_memory_stats['available']), need=format_size(START_NEED_MEMORY))) + success = False + continue + + if server_memory_stats['free'] + server_memory_stats['buffers'] + server_memory_stats['cached'] < MIN_MEMORY: + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED.format(ip=ip, free=format_size(server_memory_stats['free']), cached=format_size(server_memory_stats['buffers'] + server_memory_stats['cached']), need=format_size(MIN_MEMORY))) success = False continue - memory_limit = max(min_memory, memory_limit * 0.9) + + memory_limit = max(MIN_MEMORY, server_memory_stats['available'] * 0.9) server_config['memory_limit'] = format_size(memory_limit, 0) cluster_config.update_server_conf(server, 'memory_limit', server_config['memory_limit'], False) + auto_set_memory = True else: stdio.error("%s: fail to get memory info.\nPlease configure 'memory_limit' manually in configuration file") success = False @@ -180,7 +198,6 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): else: try: memory_limit = parse_size(server_config.get('memory_limit')) - auto_set_memory = True except: stdio.error('memory_limit must be an integer') return @@ -300,14 +317,14 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): continue disk_free = disk_free - log_size - SLOG_SIZE - memory_factor = 0 + memory_factor = 6 if auto_set_datafile_size is False: disk_free -= min_datafile_size - memory_factor += 3 + memory_factor -= 3 if auto_set_log_disk_size is False: disk_free -= min_log_disk_size - memory_factor += 3 - memory_limit = format_size(disk_free / memory_factor, 0) + memory_factor -= 3 + memory_limit = format_size(disk_free / max(1, memory_factor), 0) cluster_config.update_server_conf(server, 'memory_limit', memory_limit, False) memory_limit = parse_size(memory_limit) if auto_set_system_memory: diff --git a/plugins/oceanbase/4.0.0.0/start_check.py b/plugins/oceanbase/4.0.0.0/start_check.py index 20a705e..dd08d81 100644 --- a/plugins/oceanbase/4.0.0.0/start_check.py +++ b/plugins/oceanbase/4.0.0.0/start_check.py @@ -24,8 +24,11 @@ import os import re import time -from _errno import EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, EC_OBSERVER_NOT_ENOUGH_MEMORY, EC_ULIMIT_CHECK, WC_ULIMIT_CHECK - +from _errno import ( + EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, + EC_OBSERVER_NOT_ENOUGH_MEMORY, EC_ULIMIT_CHECK, WC_ULIMIT_CHECK, + EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE, EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED +) stdio = None success = True @@ -47,7 +50,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -113,6 +116,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): PRO_MEMORY_MIN = 16 << 30 PRO_POOL_MEM_MIN = 2147483648 + START_NEED_MEMORY = 3 << 30 stdio.start_loading('Check before start observer') for server in cluster_config.servers: ip = server.ip @@ -131,7 +135,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): servers_port[ip] = {} servers_clog_mount[ip] = {} servers_net_inferface[ip] = {} - servers_memory[ip] = {'num': 0, 'percentage': 0} + servers_memory[ip] = {'num': 0, 'percentage': 0, 'server_num': 0} memory = servers_memory[ip] ports = servers_port[ip] disk = servers_disk[ip] @@ -154,6 +158,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): if server_config.get('production_mode') and __min_full_resource_pool_memory < PRO_POOL_MEM_MIN: error('(%s): when production_mode is True, __min_full_resource_pool_memory can not be less then %s' % (server, PRO_POOL_MEM_MIN)) + memory['server_num'] += 1 if 'memory_limit' in server_config: try: memory_limit = parse_size(server_config['memory_limit']) @@ -171,6 +176,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): return else: memory['percentage'] += 80 + data_path = server_config['data_dir'] if server_config.get('data_dir') else os.path.join(server_config['home_path'], 'store') redo_dir = server_config['redo_dir'] if server_config.get('redo_dir') else data_path clog_dir = server_config['clog_dir'] if server_config.get('clog_dir') else os.path.join(redo_dir, 'clog') @@ -261,16 +267,29 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): # memory ret = client.execute_command('cat /proc/meminfo') if ret: - total_memory = 0 - free_memory = 0 + server_memory_stats = {} + memory_key_map = { + 'MemTotal': 'total', + 'MemFree': 'free', + 'MemAvailable': 'available', + 'Buffers': 'buffers', + 'Cached': 'cached' + } + for key in memory_key_map: + server_memory_stats[memory_key_map[key]] = 0 for k, v in re.findall('(\w+)\s*:\s*(\d+\s*\w+)', ret.stdout): - if k == 'MemTotal': - total_memory = parse_size(str(v)) - elif k == 'MemAvailable': - free_memory = parse_size(str(v)) - total_use = servers_memory[ip]['percentage'] * total_memory / 100 + servers_memory[ip]['num'] - if total_use > free_memory: - error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(total_use))) + if k in memory_key_map: + key = memory_key_map[k] + server_memory_stats[key] = parse_size(str(v)) + + min_start_need = servers_memory[ip]['server_num'] * START_NEED_MEMORY + total_use = servers_memory[ip]['percentage'] * server_memory_stats['total'] / 100 + servers_memory[ip]['num'] + if min_start_need > server_memory_stats['available']: + error(EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE.format(ip=ip, available=format_size(server_memory_stats['available']), need=format_size(min_start_need))) + elif total_use > server_memory_stats['free'] + server_memory_stats['buffers'] + server_memory_stats['cached']: + error(EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED.format(ip=ip, free=format_size(server_memory_stats['free']), cached=format_size(server_memory_stats['buffers'] + server_memory_stats['cached']), need=format_size(min_start_need))) + elif total_use > server_memory_stats['free']: + alert(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(server_memory_stats['free']), need=format_size(min_start_need))) # disk disk = {'/': 0} ret = client.execute_command('df --block-size=1024') diff --git a/plugins/sysbench/3.1.0/pre_test.py b/plugins/sysbench/3.1.0/pre_test.py index 13a19fc..a7a7832 100644 --- a/plugins/sysbench/3.1.0/pre_test.py +++ b/plugins/sysbench/3.1.0/pre_test.py @@ -35,7 +35,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/sysbench/4.0.0.0/pre_test.py b/plugins/sysbench/4.0.0.0/pre_test.py index 9520d7e..5f03528 100644 --- a/plugins/sysbench/4.0.0.0/pre_test.py +++ b/plugins/sysbench/4.0.0.0/pre_test.py @@ -35,7 +35,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/tpch/3.1.0/run_test.py b/plugins/tpch/3.1.0/run_test.py index 1a33681..c6782c5 100644 --- a/plugins/tpch/3.1.0/run_test.py +++ b/plugins/tpch/3.1.0/run_test.py @@ -40,7 +40,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/tpch/4.0.0.0/run_test.py b/plugins/tpch/4.0.0.0/run_test.py index da774ad..37fec05 100644 --- a/plugins/tpch/4.0.0.0/run_test.py +++ b/plugins/tpch/4.0.0.0/run_test.py @@ -41,7 +41,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/ssh.py b/ssh.py index b6028e9..a076b5b 100644 --- a/ssh.py +++ b/ssh.py @@ -368,7 +368,7 @@ class SshClient(SafeStdio): _transporter = RemoteTransporter.CLIENT if not self._is_local() and self._remote_transporter is None: if not self.config.password and not self.disable_rsync: - ret = LocalClient.execute_command('rsync -h', stdio=self.stdio) + ret = LocalClient.execute_command('rsync -h', stdio=self.stdio) and self.execute_command('rsync -h', stdio=self.stdio) if ret: _transporter = RemoteTransporter.RSYNC self._remote_transporter = _transporter @@ -418,8 +418,10 @@ class SshClient(SafeStdio): def _rsync(self, source, target, stdio=None): identity_option = "" if self.config.key_filename: - identity_option += '-e "ssh -i {key_filename} "'.format(key_filename=self.config.key_filename) - cmd = 'rsync -a -W {identity_option} {source} {target}'.format( + identity_option += '-i {key_filename} '.format(key_filename=self.config.key_filename) + if self.config.port: + identity_option += '-p {}'.format(self.config.port) + cmd = 'rsync -a -W -e "ssh {identity_option}" {source} {target}'.format( identity_option=identity_option, source=source, target=target -- GitLab