diff --git a/_cmd.py b/_cmd.py index ae249e52232664e1b75b2ca90cabf3590cf6c283..6ad96f0cc351909997a2fb2cd67eecfa5dd8234e 100644 --- a/_cmd.py +++ b/_cmd.py @@ -854,6 +854,7 @@ class ClusterTenantCreateCommand(ClusterMirrorCommand): self.parser.add_option('--max-session-num', type='int', help="Max session unit number. Not supported after version 4.0") self.parser.add_option('--unit-num', type='int', help="Pool unit number.") self.parser.add_option('-z', '--zone-list', type='string', help="Tenant zone list.") + self.parser.add_option('--mode', type='string', help='Tenant compatibility mode. {mysql,oracle} [mysql]', default='mysql') self.parser.add_option('--charset', type='string', help="Tenant charset.") self.parser.add_option('--collate', type='string', help="Tenant COLLATE.") self.parser.add_option('--replica-num', type='int', help="Tenant replica number.") @@ -925,6 +926,8 @@ class MySQLTestCommand(TestMirrorCommand): def __init__(self): super(MySQLTestCommand, self).__init__('mysqltest', 'Run a mysqltest for a deployment.') + self.parser.add_option('--mode', type='string', help='Test mode. Available values are mysql, oracle, and both.', default='both') + # self.parser.add_option('--case-mode', type='string', help='case run mode [mysql,oracle]', default='mysql') self.parser.add_option('--component', type='string', help='Components for mysqltest.') self.parser.add_option('--test-server', type='string', help='The server for mysqltest. By default, the first root server in the component is the mysqltest server.') self.parser.add_option('--user', type='string', help='Username for a test. [admin]', default='admin') @@ -1042,7 +1045,7 @@ class TPCHCommand(TestMirrorCommand): self.parser.add_option('--remote-tbl-dir', type='string', help='Directory for the tbl on target observers. Make sure that you have read and write access to the directory when you start observer.') self.parser.add_option('--disable-transfer', '--dt', action='store_true', help='Disable the transfer. When enabled, OBD will use the tbl files under remote-tbl-dir instead of transferring local tbl files to remote remote-tbl-dir.') self.parser.add_option('--dss-config', type='string', help='Directory for dists.dss. [/usr/tpc-h-tools/tpc-h-tools]', default='/usr/tpc-h-tools/tpc-h-tools/') - self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1}. [1]', default=1) + self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1/2}. [1] 0 - No optimization. 1 - Optimize some of the parameters which do not need to restart servers. 2 - Optimize all the parameters and maybe RESTART SERVERS for better performance.', default=1) self.parser.add_option('--test-only', action='store_true', help='Only testing SQLs are executed. No initialization is executed.') self.parser.add_option('-S', '--skip-cluster-status-check', action='store_true', help='Skip cluster status check', default=False) diff --git a/_deploy.py b/_deploy.py index cb32fa37dbefaf6e88cc0a2d89f9f81a9c039c60..7f5158754b0c18534202eb116e33d768dadcd809 100644 --- a/_deploy.py +++ b/_deploy.py @@ -625,6 +625,8 @@ class ClusterConfig(object): return error def set_global_conf(self, conf): + if not isinstance(conf, dict): + raise Exception('%s global config is not a dictionary. Please check the syntax of your configuration file.\n See https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md' % self.name) self._original_global_conf = deepcopy(conf) self._global_conf = None self._clear_cache_server() diff --git a/_errno.py b/_errno.py index cf22afedb711470073ac63da04a03f5957fb9b34..b1c09e2622ec398128bb78ea24df74ada34b0e48 100644 --- a/_errno.py +++ b/_errno.py @@ -62,6 +62,8 @@ EC_FAIL_TO_CONNECT = OBDErrorCode(1006, 'Failed to connect to {component}') EC_ULIMIT_CHECK = OBDErrorCode(1007, '({server}) {key} must not be less than {need} (Current value: {now})') EC_OBSERVER_NOT_ENOUGH_MEMORY = OBDErrorCode(2000, '({ip}) not enough memory. (Free: {free}, Need: {need})') +EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE = OBDErrorCode(2000, '({ip}) not enough memory. (Available: {available}, Need: {need})') +EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED = OBDErrorCode(2000, '({ip}) not enough memory. (Free: {free}, Buff/Cache: {cached}, Need: {need})') EC_OBSERVER_CAN_NOT_MIGRATE_IN = OBDErrorCode(2001, 'server can not migrate in') EC_OBSERVER_FAIL_TO_START = OBDErrorCode(2002, 'Failed to start {server} observer') EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG = OBDErrorCode(2003, '({ip}) {path} not enough disk space for clog. Use redo_dir to set other disk for clog, or reduce the value of datafile_size') diff --git a/_repository.py b/_repository.py index 600471757ee4dc7f1492ceda00b3af374beda1c6..7830a4bcfba24d2cf5c7f3365473b7adda79ff15 100644 --- a/_repository.py +++ b/_repository.py @@ -172,7 +172,9 @@ class ParallerExtractWorker(object): class ParallerExtractor(object): - MAX_PARALLER = cpu_count() if cpu_count() else 8 + MAX_PARALLER = cpu_count() * 2 if cpu_count() else 8 + MAX_SIZE = 100 + MIN_SIZE = 20 def __init__(self, pkg, files, stdio=None): self.pkg = pkg @@ -180,11 +182,13 @@ class ParallerExtractor(object): self.stdio = stdio def extract(self): + if not self.files: + return workers = [] file_num = len(self.files) - paraler = int(min(self.MAX_PARALLER, file_num)) - size = min(100, int(file_num / paraler)) - size = int(max(10, size)) + paraller = int(min(self.MAX_PARALLER, file_num)) + size = min(self.MAX_SIZE, int(file_num / paraller)) # + size = int(max(self.MIN_SIZE, size)) index = 0 while index < file_num: p_index = index + size @@ -195,7 +199,7 @@ class ParallerExtractor(object): )) index = p_index - pool = Pool(processes=paraler) + pool = Pool(processes=paraller) try: results = pool.map(ParallerExtractWorker.extract, workers) for r in results: diff --git a/core.py b/core.py index d25f9a354e49b30cc5e2b6bde7febc61dc7bb0a1..4bcc5d3a2ed015c0a3d73de2dd3ea675d5cc245a 100644 --- a/core.py +++ b/core.py @@ -890,7 +890,7 @@ class ObdHome(object): self._call_stdio('verbose', 'Get deploy configuration') deploy_config = deploy.deploy_config if not deploy_config: - self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.') + self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md') return False # Check the best suitable mirror for the components and installation plugins. Install locally @@ -1061,7 +1061,7 @@ class ObdHome(object): return False deploy_config = deploy.deploy_config if not deploy_config: - self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.') + self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md') return False style = getattr(options, 'style', '') @@ -1211,11 +1211,11 @@ class ObdHome(object): self._call_stdio('verbose', 'Get deploy configuration') deploy_config = deploy.deploy_config if not deploy_config: - self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.') + self._call_stdio('error', 'Deploy configuration is empty.\nIt may be caused by a failure to resolve the configuration.\nPlease check your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md') return False if not deploy_config.components: - self._call_stdio('error', 'Components not detected.\nPlease check the syntax of your configuration file.') + self._call_stdio('error', 'Components not detected.\nPlease check the syntax of your configuration file.\nSee https://github.com/oceanbase/obdeploy/blob/master/docs/zh-CN/4.configuration-file-description.md') return False for component_name in deploy_config.components: diff --git a/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md b/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md new file mode 100644 index 0000000000000000000000000000000000000000..cd8f1233f92e84151a0e4bb632ecc2b9404bf4fd --- /dev/null +++ b/docs/en-US/3.user-guide/3.obd-command/0.obd-demo.md @@ -0,0 +1,41 @@ +# Quick deployment command + +## obd demo + +You can use this command to deploy and start the specified component on the local server without loading the configuration file. The fixed name of the cluster deployed is `demo`. After the deployment, you can run the `obd cluster list` command to view the cluster in the cluster list. You can also run other cluster commands, such as `obd cluster display demo`, to manage the cluster. + +```bash +obd demo [-c/--components] +``` + +The following table describes the parameters. + +| Parameter | Required | Data type | Default value | Description | +|------------------|---------|------------|----------|--------------------------------------------------------------------| +| -c/--components | No | String | oceanbase-ce,obproxy-ce,obagent,prometheus,grafana | The list of components that are separated with commas (`,`). You can use this parameter to specify the components to be deployed. | + +By default, this command deploys the minimum specifications in the home directory of the current user, and the latest versions are deployed by default. You can use this command to deploy OceanBase Community Edition, OBProxy Community Edition, OBAgent, Grafana, and Prometheus. + +You can select the version and specify the configurations of a component to be deployed. + +```bash +# Deploy components of the specified version. +obd demo -c oceanbase-ce,obproxy-ce --oceanbase-ce.version=3.1.3 +# Specify the components to be deployed and the package hash of OceanBase Community Edition. +obd demo -c oceanbase-ce,obproxy-ce --oceanbase-ce.package_hash=f38723204d49057d3e062ffad778edc1552a7c114622bf2a86fea769fbd202ea +# Specify the installation path for all components to be deployed. +## Deploy OceanBase Community Edition and OBProxy Community Edition in the /data/demo directory and create corresponding working directories for them. +obd demo -c oceanbase-ce,obproxy-ce --home_path=/data/demo +# Specify the installation path for all components to be deployed. +obd demo --home_path=/path +# Specify the installation path for a specific component to be deployed. +## Deploy OceanBase Community Edition in the home directory and create a working directory for it, and deploy OBProxy Community Edition in the /data/playground/obproxy-ce directory. +obd demo -c oceanbase-ce,obproxy-ce --obproxy-ce.home_path=/data/demo/ +# Specify the configurations of a component to be deployed. +## Specify the mysql_port parameter of OceanBase Community Edition. +obd demo --oceanbase-ce.mysql_port=3881 +``` + +> **Notice** +> +> This command supports only level-1 configurations under global that are specified by using options. diff --git a/optimize/obproxy/4.0.0/sysbench.yaml b/optimize/obproxy/4.0.0/sysbench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ada4cc734a0a9b0047c3b791d59e4375c798faf --- /dev/null +++ b/optimize/obproxy/4.0.0/sysbench.yaml @@ -0,0 +1,24 @@ +test: + system_config: + - name: proxy_mem_limited + value: 2G + - name: slow_proxy_process_time_threshold + value: 500ms + - name: syslog_level + value: error + query_key: level + - name: enable_prometheus + value: false + - name: enable_compression_protocol + value: false + need_restart: true + value_type: BOOL + - name: enable_ob_protocol_v2 + value: false + need_restart: true + value_type: BOOL + - name: work_thread_num + value: 128 + need_restart: true + - name: enable_async_log + value: true \ No newline at end of file diff --git a/optimize/obproxy/4.0.0/tpcc.yaml b/optimize/obproxy/4.0.0/tpcc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..782bbd0f61c697650115c8838593c955d7b23c93 --- /dev/null +++ b/optimize/obproxy/4.0.0/tpcc.yaml @@ -0,0 +1,24 @@ +build: + system_config: + - name: proxy_mem_limited + value: 2G + - name: slow_proxy_process_time_threshold + value: 500ms + - name: syslog_level + value: error + query_key: level + - name: enable_prometheus + value: false + - name: enable_compression_protocol + value: false + need_restart: true + value_type: BOOL + - name: enable_ob_protocol_v2 + value: false + need_restart: true + value_type: BOOL + - name: work_thread_num + value: 128 + need_restart: true + - name: enable_async_log + value: true \ No newline at end of file diff --git a/plugins/grafana/7.5.17/oceanbase-metrics_rev1.json b/plugins/grafana/7.5.17/oceanbase-metrics_rev1.json index 81ba4bda0a667f738155058d7d65a72b019c5227..15cd3a21a802908e14d35f37f312a7c584862cbc 100644 --- a/plugins/grafana/7.5.17/oceanbase-metrics_rev1.json +++ b/plugins/grafana/7.5.17/oceanbase-metrics_rev1.json @@ -218,11 +218,11 @@ "exemplar": true, "expr": "(sum(rate(ob_sysstat{stat_id=\"40003\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40009\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40009\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40001\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group))\n/\n(sum(rate(ob_sysstat{stat_id=\"40002\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40004\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40008\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"40000\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group))", "interval": "", - "legendFormat": "qps rt {{$group}}", + "legendFormat": "sql latency {{$group}}", "refId": "A" } ], - "title": "QPS rt", + "title": "Sql Latency", "type": "timeseries" }, { @@ -2129,7 +2129,7 @@ "exemplar": true, "expr": "(sum(rate(ob_sysstat{stat_id=\"10005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"10006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)) / sum(rate(ob_sysstat{stat_id=\"10000\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)", "interval": "", - "legendFormat": "roc in rt {{$group}}", + "legendFormat": "rpc in Latency {{$group}}", "refId": "A" }, { @@ -2137,11 +2137,11 @@ "expr": "(sum(rate(ob_sysstat{stat_id=\"10005\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group) + sum(rate(ob_sysstat{stat_id=\"10006\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)) / sum(rate(ob_sysstat{stat_id=\"10002\",ob_cluster_name=~\"$obcluster\",obzone=~\"$obzone\",svr_ip=~\"$observer\",tenant_name=~\"$tenant_name\"}[$__rate_interval])) by ($group)", "hide": false, "interval": "", - "legendFormat": "rpc out rt {{$group}}", + "legendFormat": "rpc out latency {{$group}}", "refId": "B" } ], - "title": "Rpc rt", + "title": "Rpc Latency", "type": "timeseries" }, { diff --git a/plugins/mysqltest/3.1.0/init.py b/plugins/mysqltest/3.1.0/init.py index 081f7ea1dafb41ca64c883bd231c528a0ee32fd3..0ab059312f79ecbc6e40c96c6fffa73e404bc021 100644 --- a/plugins/mysqltest/3.1.0/init.py +++ b/plugins/mysqltest/3.1.0/init.py @@ -36,7 +36,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -127,6 +127,7 @@ def init(plugin_context, env, *args, **kwargs): exec_init = 'init.sql' exec_mini_init = 'init_mini.sql' exec_init_user = 'init_user.sql|root@mysql|test' + exec_init_user_for_oracle = 'init_user_oracle.sql|SYS@oracle|SYS' client = plugin_context.clients[server] memory_limit = get_memory_limit(cursor, client) is_mini = memory_limit and parse_size(memory_limit) < (16<<30) diff --git a/plugins/mysqltest/4.0.0.0/init.py b/plugins/mysqltest/4.0.0.0/init.py index cd8f04adc6c50db3499e98b90b17c6974171b0e3..ff25721f967aa6f71b27dc0d21c8ee971ed0fe89 100644 --- a/plugins/mysqltest/4.0.0.0/init.py +++ b/plugins/mysqltest/4.0.0.0/init.py @@ -36,7 +36,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/obproxy/3.1.0/display.py b/plugins/obproxy/3.1.0/display.py index e6fb09679ba19d8307dc46c98e08535062972336..2310ca7a25cbee8a6c636ec1b47b5f41712c8809 100644 --- a/plugins/obproxy/3.1.0/display.py +++ b/plugins/obproxy/3.1.0/display.py @@ -56,12 +56,12 @@ def display(plugin_context, cursor, *args, **kwargs): continue password = ob_config.get('root_password', '') with_observer = True - cmd = 'obclient -h%s -P%s -uroot %s-Doceanbase' % (server.ip, server_config['listen_port'], '-p%s ' % password if password else '') + cmd = 'obclient -h%s -P%s -uroot %s-Doceanbase -A' % (server.ip, server_config['listen_port'], '-p%s ' % password if password else '') break if not with_observer: password = server_config.get('obproxy_sys_password', '') - cmd = 'obclient -h%s -P%s -uroot@proxysys %s-Doceanbase' % (server.ip, server_config['listen_port'], '-p%s ' % password if password else '') + cmd = 'obclient -h%s -P%s -uroot@proxysys %s-Doceanbase -A' % (server.ip, server_config['listen_port'], '-p%s ' % password if password else '') stdio.print(cmd) diff --git a/plugins/obproxy/3.1.0/file_map.yaml b/plugins/obproxy/3.1.0/file_map.yaml index b7a139391972be06ab730273a942bd8779dbcd4a..34ec9123ad2c7c5c581c782e09d6997d159c9d7a 100644 --- a/plugins/obproxy/3.1.0/file_map.yaml +++ b/plugins/obproxy/3.1.0/file_map.yaml @@ -1,4 +1,4 @@ -- src_path: ./home/admin/obproxy-$version/bin/obproxy +- src_path: ./opt/taobao/install/obproxy-$version/bin/obproxy target_path: bin/obproxy type: bin mode: 755 \ No newline at end of file diff --git a/plugins/obproxy/3.1.0/generate_config.py b/plugins/obproxy/3.1.0/generate_config.py index 73bc2c5e07b3a21eac07f77de3a145d4d6d21b1e..08def5951a57f8c027a80d69c55ce97765da4368 100644 --- a/plugins/obproxy/3.1.0/generate_config.py +++ b/plugins/obproxy/3.1.0/generate_config.py @@ -47,7 +47,7 @@ def generate_config(plugin_context, deploy_config, auto_depend=False, *args, **k if getattr(plugin_context.options, 'mini', False): if 'proxy_mem_limited' not in global_config: - cluster_config.update_global_conf('proxy_mem_limited', '200M', False) + cluster_config.update_global_conf('proxy_mem_limited', '500M', False) ob_comps = ['oceanbase', 'oceanbase-ce'] ob_cluster_config = None diff --git a/plugins/oceanbase/3.1.0/create_tenant.py b/plugins/oceanbase/3.1.0/create_tenant.py index 9ce6ba823abc267c0b5609175555ce3d604efafa..f3e16d4a1a8098a942afc8a40e331391fa7831ad 100644 --- a/plugins/oceanbase/3.1.0/create_tenant.py +++ b/plugins/oceanbase/3.1.0/create_tenant.py @@ -35,7 +35,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'^([1-9][0-9]*)\s*([B,K,M,G,T])$', size.upper()) + match = re.match(r'^(0|[1-9][0-9]*)\s*([B,K,M,G,T])$', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/oceanbase/3.1.0/display.py b/plugins/oceanbase/3.1.0/display.py index 3d7cc24ef083edb4a4991caeb48912daa29c4280..a2a12dbd5d396ea6b9ef42b60f4560f8cdb24add 100644 --- a/plugins/oceanbase/3.1.0/display.py +++ b/plugins/oceanbase/3.1.0/display.py @@ -36,7 +36,7 @@ def display(plugin_context, cursor, *args, **kwargs): stdio.print_list(servers, ['ip', 'version', 'port', 'zone', 'status'], lambda x: [x['svr_ip'], x['build_version'].split('_')[0], x['inner_port'], x['zone'], x['status']], title='observer') password = cluster_config.get_global_conf().get('root_password', '') - cmd = 'obclient -h%s -P%s -uroot %s-Doceanbase' % (servers[0]['svr_ip'], servers[0]['inner_port'], '-p%s ' % password if password else '') + cmd = 'obclient -h%s -P%s -uroot %s-Doceanbase -A' % (servers[0]['svr_ip'], servers[0]['inner_port'], '-p%s ' % password if password else '') stdio.print(cmd) stdio.stop_loading('succeed') return plugin_context.return_true() diff --git a/plugins/oceanbase/3.1.0/generate_config.py b/plugins/oceanbase/3.1.0/generate_config.py index d3c29a5b051a9e0c91b5ea07209dcb327deb7e3d..5e453f75df0fbede47e4eeb88d7043ccad377740 100644 --- a/plugins/oceanbase/3.1.0/generate_config.py +++ b/plugins/oceanbase/3.1.0/generate_config.py @@ -23,7 +23,7 @@ from __future__ import absolute_import, division, print_function import re, os -from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY +from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE, EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED def parse_size(size): @@ -32,7 +32,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -90,6 +90,7 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): MIN_MEMORY = 8 << 30 MIN_CPU_COUNT = 16 + START_NEED_MEMORY = 3 << 30 clog_disk_utilization_threshold_max = 95 clog_disk_usage_limit_percentage_max = 98 global_config = cluster_config.get_original_global_conf() @@ -158,16 +159,32 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): if not server_config.get('memory_limit'): ret = client.execute_command('cat /proc/meminfo') if ret: - free_memory = 0 + server_memory_stats = {} + memory_key_map = { + 'MemTotal': 'total', + 'MemFree': 'free', + 'MemAvailable': 'available', + 'Buffers': 'buffers', + 'Cached': 'cached' + } + for key in memory_key_map: + server_memory_stats[memory_key_map[key]] = 0 for k, v in re.findall('(\w+)\s*:\s*(\d+\s*\w+)', ret.stdout): - if k == 'MemAvailable': - free_memory = parse_size(str(v)) - memory_limit = free_memory - if memory_limit < MIN_MEMORY: - stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(MIN_MEMORY))) + if k in memory_key_map: + key = memory_key_map[k] + server_memory_stats[key] = parse_size(str(v)) + + if server_memory_stats['available'] < START_NEED_MEMORY: + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE.format(ip=ip, available=format_size(server_memory_stats['available']), need=format_size(START_NEED_MEMORY))) success = False continue - memory_limit = max(MIN_MEMORY, memory_limit * 0.9) + + if server_memory_stats['free'] + server_memory_stats['buffers'] + server_memory_stats['cached'] < MIN_MEMORY: + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED.format(ip=ip, free=format_size(server_memory_stats['free']), cached=format_size(server_memory_stats['buffers'] + server_memory_stats['cached']), need=format_size(MIN_MEMORY))) + success = False + continue + + memory_limit = max(MIN_MEMORY, server_memory_stats['available'] * 0.9) server_config['memory_limit'] = format_size(memory_limit, 0) cluster_config.update_server_conf(server, 'memory_limit', server_config['memory_limit'], False) else: diff --git a/plugins/oceanbase/3.1.0/reload.py b/plugins/oceanbase/3.1.0/reload.py index 0fb6bd95bef46310962c8e0db1a8f1a07d513efa..d0021af6efb27cbf3f830ac3eadaeb128544d4f7 100644 --- a/plugins/oceanbase/3.1.0/reload.py +++ b/plugins/oceanbase/3.1.0/reload.py @@ -115,10 +115,11 @@ def reload(plugin_context, cursor, new_cluster_config, *args, **kwargs): for server in servers: if key not in change_conf[server]: continue + value = change_conf[server][key] msg = sql = 'alter system set %s = %%s server=%%s' % key stdio.verbose('execute sql: %s' % msg) - cursor.execute(sql, [change_conf[server][key], cluster_server[server]]) - cluster_config.update_server_conf(server,key, value, False) + cursor.execute(sql, [value, cluster_server[server]]) + cluster_config.update_server_conf(server, key, value, False) except: global_ret = False stdio.exception('execute sql exception: %s' % msg) diff --git a/plugins/oceanbase/3.1.0/start_check.py b/plugins/oceanbase/3.1.0/start_check.py index 884695b3852d52ed5564d60b397696b2fc260440..93d35f2862425e7039f92ed4ffc6b5447ab55a73 100644 --- a/plugins/oceanbase/3.1.0/start_check.py +++ b/plugins/oceanbase/3.1.0/start_check.py @@ -24,7 +24,11 @@ import os import re import time -from _errno import EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, EC_OBSERVER_NOT_ENOUGH_MEMORY, EC_ULIMIT_CHECK, WC_ULIMIT_CHECK +from _errno import ( + EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, + EC_OBSERVER_NOT_ENOUGH_MEMORY, EC_ULIMIT_CHECK, WC_ULIMIT_CHECK, + EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE, EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED +) stdio = None @@ -46,8 +50,9 @@ def parse_size(size): if not isinstance(size, str) or size.isdigit(): _bytes = int(size) else: + print (size) units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -90,6 +95,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): global success success = False stdio.error(*arg, **kwargs) + global stdio cluster_config = plugin_context.cluster_config clients = plugin_context.clients @@ -101,6 +107,8 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): servers_clog_mount = {} servers_net_inferface = {} server_num = len(cluster_config.servers) + START_NEED_MEMORY = 3 << 30 + stdio.start_loading('Check before start observer') for server in cluster_config.servers: ip = server.ip @@ -119,7 +127,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): servers_port[ip] = {} servers_clog_mount[ip] = {} servers_net_inferface[ip] = {} - servers_memory[ip] = {'num': 0, 'percentage': 0} + servers_memory[ip] = {'num': 0, 'percentage': 0, 'server_num': 0} memory = servers_memory[ip] ports = servers_port[ip] disk = servers_disk[ip] @@ -137,6 +145,8 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): } if get_port_socket_inode(client, port): critical('%s:%s port is already used' % (ip, port)) + + memory['server_num'] += 1 if 'memory_limit' in server_config: try: memory['num'] += parse_size(server_config['memory_limit']) @@ -234,16 +244,29 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): # memory ret = client.execute_command('cat /proc/meminfo') if ret: - total_memory = 0 - free_memory = 0 + server_memory_stats = {} + memory_key_map = { + 'MemTotal': 'total', + 'MemFree': 'free', + 'MemAvailable': 'available', + 'Buffers': 'buffers', + 'Cached': 'cached' + } + for key in memory_key_map: + server_memory_stats[memory_key_map[key]] = 0 for k, v in re.findall('(\w+)\s*:\s*(\d+\s*\w+)', ret.stdout): - if k == 'MemTotal': - total_memory = parse_size(str(v)) - elif k == 'MemAvailable': - free_memory = parse_size(str(v)) - total_use = servers_memory[ip]['percentage'] * total_memory / 100 + servers_memory[ip]['num'] - if total_use > free_memory: - error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(total_use))) + if k in memory_key_map: + key = memory_key_map[k] + server_memory_stats[key] = parse_size(str(v)) + + min_start_need = servers_memory[ip]['server_num'] * START_NEED_MEMORY + total_use = servers_memory[ip]['percentage'] * server_memory_stats['total'] / 100 + servers_memory[ip]['num'] + if min_start_need > server_memory_stats['available']: + error(EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE.format(ip=ip, available=format_size(server_memory_stats['available']), need=format_size(min_start_need))) + elif total_use > server_memory_stats['free'] + server_memory_stats['buffers'] + server_memory_stats['cached']: + error(EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED.format(ip=ip, free=format_size(server_memory_stats['free']), cached=format_size(server_memory_stats['buffers'] + server_memory_stats['cached']), need=format_size(min_start_need))) + elif total_use > server_memory_stats['free']: + alert(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(server_memory_stats['free']), need=format_size(min_start_need))) # disk disk = {'/': 0} ret = client.execute_command('df --block-size=1024') diff --git a/plugins/oceanbase/4.0.0.0/create_tenant.py b/plugins/oceanbase/4.0.0.0/create_tenant.py index 7999072d06f680421cad007bb17680e2710aa133..6315ccd3ab201ac65a2e8d46a443e7bd8f5a1efd 100644 --- a/plugins/oceanbase/4.0.0.0/create_tenant.py +++ b/plugins/oceanbase/4.0.0.0/create_tenant.py @@ -35,7 +35,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'^([1-9][0-9]*)\s*([B,K,M,G,T])$', size.upper()) + match = re.match(r'^(0|[1-9][0-9]*)\s*([B,K,M,G,T])$', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/oceanbase/4.0.0.0/generate_config.py b/plugins/oceanbase/4.0.0.0/generate_config.py index 15fb2adf93951988820919ea0b4e4a73226b570b..03271cca94a9241bf951a62a5d626f4bafce12f7 100644 --- a/plugins/oceanbase/4.0.0.0/generate_config.py +++ b/plugins/oceanbase/4.0.0.0/generate_config.py @@ -23,7 +23,7 @@ from __future__ import absolute_import, division, print_function import re, os -from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY +from _errno import EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE, EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED def parse_size(size): @@ -32,7 +32,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -106,6 +106,7 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): PRO_MEMORY_MIN = 16 << 30 SLOG_SIZE = 10 << 30 MIN_CPU_COUNT = 16 + START_NEED_MEMORY = 3 << 30 if getattr(plugin_context.options, 'mini', False): if not global_config.get('memory_limit_percentage') and not global_config.get('memory_limit'): cluster_config.update_global_conf('memory_limit', format_size(MIN_MEMORY, 0), False) @@ -161,18 +162,35 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): if not server_config.get('memory_limit'): ret = client.execute_command('cat /proc/meminfo') if ret: - free_memory = 0 + server_memory_stats = {} + memory_key_map = { + 'MemTotal': 'total', + 'MemFree': 'free', + 'MemAvailable': 'available', + 'Buffers': 'buffers', + 'Cached': 'cached' + } + for key in memory_key_map: + server_memory_stats[memory_key_map[key]] = 0 for k, v in re.findall('(\w+)\s*:\s*(\d+\s*\w+)', ret.stdout): - if k == 'MemAvailable': - free_memory = parse_size(str(v)) - memory_limit = free_memory - if memory_limit < min_memory: - stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(min_memory))) + if k in memory_key_map: + key = memory_key_map[k] + server_memory_stats[key] = parse_size(str(v)) + + if server_memory_stats['available'] < START_NEED_MEMORY: + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE.format(ip=ip, available=format_size(server_memory_stats['available']), need=format_size(START_NEED_MEMORY))) + success = False + continue + + if server_memory_stats['free'] + server_memory_stats['buffers'] + server_memory_stats['cached'] < MIN_MEMORY: + stdio.error(EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED.format(ip=ip, free=format_size(server_memory_stats['free']), cached=format_size(server_memory_stats['buffers'] + server_memory_stats['cached']), need=format_size(MIN_MEMORY))) success = False continue - memory_limit = max(min_memory, memory_limit * 0.9) + + memory_limit = max(MIN_MEMORY, server_memory_stats['available'] * 0.9) server_config['memory_limit'] = format_size(memory_limit, 0) cluster_config.update_server_conf(server, 'memory_limit', server_config['memory_limit'], False) + auto_set_memory = True else: stdio.error("%s: fail to get memory info.\nPlease configure 'memory_limit' manually in configuration file") success = False @@ -180,7 +198,6 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): else: try: memory_limit = parse_size(server_config.get('memory_limit')) - auto_set_memory = True except: stdio.error('memory_limit must be an integer') return @@ -300,14 +317,14 @@ def generate_config(plugin_context, deploy_config, *args, **kwargs): continue disk_free = disk_free - log_size - SLOG_SIZE - memory_factor = 0 + memory_factor = 6 if auto_set_datafile_size is False: disk_free -= min_datafile_size - memory_factor += 3 + memory_factor -= 3 if auto_set_log_disk_size is False: disk_free -= min_log_disk_size - memory_factor += 3 - memory_limit = format_size(disk_free / memory_factor, 0) + memory_factor -= 3 + memory_limit = format_size(disk_free / max(1, memory_factor), 0) cluster_config.update_server_conf(server, 'memory_limit', memory_limit, False) memory_limit = parse_size(memory_limit) if auto_set_system_memory: diff --git a/plugins/oceanbase/4.0.0.0/start_check.py b/plugins/oceanbase/4.0.0.0/start_check.py index 20a705e9ca6080a91b19727f9f9d719a1a608fcd..dd08d81ab80ecf0e8fa0dc2d694f05ff3b9d907a 100644 --- a/plugins/oceanbase/4.0.0.0/start_check.py +++ b/plugins/oceanbase/4.0.0.0/start_check.py @@ -24,8 +24,11 @@ import os import re import time -from _errno import EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, EC_OBSERVER_NOT_ENOUGH_MEMORY, EC_ULIMIT_CHECK, WC_ULIMIT_CHECK - +from _errno import ( + EC_OBSERVER_NOT_ENOUGH_DISK_4_CLOG, EC_CONFIG_CONFLICT_PORT, + EC_OBSERVER_NOT_ENOUGH_MEMORY, EC_ULIMIT_CHECK, WC_ULIMIT_CHECK, + EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE, EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED +) stdio = None success = True @@ -47,7 +50,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes @@ -113,6 +116,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): PRO_MEMORY_MIN = 16 << 30 PRO_POOL_MEM_MIN = 2147483648 + START_NEED_MEMORY = 3 << 30 stdio.start_loading('Check before start observer') for server in cluster_config.servers: ip = server.ip @@ -131,7 +135,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): servers_port[ip] = {} servers_clog_mount[ip] = {} servers_net_inferface[ip] = {} - servers_memory[ip] = {'num': 0, 'percentage': 0} + servers_memory[ip] = {'num': 0, 'percentage': 0, 'server_num': 0} memory = servers_memory[ip] ports = servers_port[ip] disk = servers_disk[ip] @@ -154,6 +158,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): if server_config.get('production_mode') and __min_full_resource_pool_memory < PRO_POOL_MEM_MIN: error('(%s): when production_mode is True, __min_full_resource_pool_memory can not be less then %s' % (server, PRO_POOL_MEM_MIN)) + memory['server_num'] += 1 if 'memory_limit' in server_config: try: memory_limit = parse_size(server_config['memory_limit']) @@ -171,6 +176,7 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): return else: memory['percentage'] += 80 + data_path = server_config['data_dir'] if server_config.get('data_dir') else os.path.join(server_config['home_path'], 'store') redo_dir = server_config['redo_dir'] if server_config.get('redo_dir') else data_path clog_dir = server_config['clog_dir'] if server_config.get('clog_dir') else os.path.join(redo_dir, 'clog') @@ -261,16 +267,29 @@ def _start_check(plugin_context, strict_check=False, *args, **kwargs): # memory ret = client.execute_command('cat /proc/meminfo') if ret: - total_memory = 0 - free_memory = 0 + server_memory_stats = {} + memory_key_map = { + 'MemTotal': 'total', + 'MemFree': 'free', + 'MemAvailable': 'available', + 'Buffers': 'buffers', + 'Cached': 'cached' + } + for key in memory_key_map: + server_memory_stats[memory_key_map[key]] = 0 for k, v in re.findall('(\w+)\s*:\s*(\d+\s*\w+)', ret.stdout): - if k == 'MemTotal': - total_memory = parse_size(str(v)) - elif k == 'MemAvailable': - free_memory = parse_size(str(v)) - total_use = servers_memory[ip]['percentage'] * total_memory / 100 + servers_memory[ip]['num'] - if total_use > free_memory: - error(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(free_memory), need=format_size(total_use))) + if k in memory_key_map: + key = memory_key_map[k] + server_memory_stats[key] = parse_size(str(v)) + + min_start_need = servers_memory[ip]['server_num'] * START_NEED_MEMORY + total_use = servers_memory[ip]['percentage'] * server_memory_stats['total'] / 100 + servers_memory[ip]['num'] + if min_start_need > server_memory_stats['available']: + error(EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE.format(ip=ip, available=format_size(server_memory_stats['available']), need=format_size(min_start_need))) + elif total_use > server_memory_stats['free'] + server_memory_stats['buffers'] + server_memory_stats['cached']: + error(EC_OBSERVER_NOT_ENOUGH_MEMORY_CACHED.format(ip=ip, free=format_size(server_memory_stats['free']), cached=format_size(server_memory_stats['buffers'] + server_memory_stats['cached']), need=format_size(min_start_need))) + elif total_use > server_memory_stats['free']: + alert(EC_OBSERVER_NOT_ENOUGH_MEMORY.format(ip=ip, free=format_size(server_memory_stats['free']), need=format_size(min_start_need))) # disk disk = {'/': 0} ret = client.execute_command('df --block-size=1024') diff --git a/plugins/sysbench/3.1.0/pre_test.py b/plugins/sysbench/3.1.0/pre_test.py index 13a19fc8bed702cb2a341b6c6118e5807b9494ca..a7a7832692b32d39cb2e2e890371158b9f4d3b3e 100644 --- a/plugins/sysbench/3.1.0/pre_test.py +++ b/plugins/sysbench/3.1.0/pre_test.py @@ -35,7 +35,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/sysbench/4.0.0.0/pre_test.py b/plugins/sysbench/4.0.0.0/pre_test.py index 9520d7e3ef50ab3aca0ebb194beebd90003e3f4d..5f03528a2efeef32f2b4280cb8e605ecd4c48559 100644 --- a/plugins/sysbench/4.0.0.0/pre_test.py +++ b/plugins/sysbench/4.0.0.0/pre_test.py @@ -35,7 +35,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/tpch/3.1.0/run_test.py b/plugins/tpch/3.1.0/run_test.py index 1a33681adfdc41443857820f573ec37cd1c403b4..c6782c578543b60f4aa12855042ad688292cdace 100644 --- a/plugins/tpch/3.1.0/run_test.py +++ b/plugins/tpch/3.1.0/run_test.py @@ -40,7 +40,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/plugins/tpch/4.0.0.0/run_test.py b/plugins/tpch/4.0.0.0/run_test.py index da774ad7fd70fd372748329c8a2f4212e8306da7..37fec0513d8f297359f8b1303018e7d2b519de56 100644 --- a/plugins/tpch/4.0.0.0/run_test.py +++ b/plugins/tpch/4.0.0.0/run_test.py @@ -41,7 +41,7 @@ def parse_size(size): _bytes = int(size) else: units = {"B": 1, "K": 1<<10, "M": 1<<20, "G": 1<<30, "T": 1<<40} - match = re.match(r'([1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) + match = re.match(r'(0|[1-9][0-9]*)\s*([B,K,M,G,T])', size.upper()) _bytes = int(match.group(1)) * units[match.group(2)] return _bytes diff --git a/ssh.py b/ssh.py index b6028e9a73af31a4f5e0b8a650ca0dec847ae474..a076b5b50d593ae4f8e951bb0369b2cbd50d1443 100644 --- a/ssh.py +++ b/ssh.py @@ -368,7 +368,7 @@ class SshClient(SafeStdio): _transporter = RemoteTransporter.CLIENT if not self._is_local() and self._remote_transporter is None: if not self.config.password and not self.disable_rsync: - ret = LocalClient.execute_command('rsync -h', stdio=self.stdio) + ret = LocalClient.execute_command('rsync -h', stdio=self.stdio) and self.execute_command('rsync -h', stdio=self.stdio) if ret: _transporter = RemoteTransporter.RSYNC self._remote_transporter = _transporter @@ -418,8 +418,10 @@ class SshClient(SafeStdio): def _rsync(self, source, target, stdio=None): identity_option = "" if self.config.key_filename: - identity_option += '-e "ssh -i {key_filename} "'.format(key_filename=self.config.key_filename) - cmd = 'rsync -a -W {identity_option} {source} {target}'.format( + identity_option += '-i {key_filename} '.format(key_filename=self.config.key_filename) + if self.config.port: + identity_option += '-p {}'.format(self.config.port) + cmd = 'rsync -a -W -e "ssh {identity_option}" {source} {target}'.format( identity_option=identity_option, source=source, target=target