From 9813bb0b6f975e2086e8ae9be5d51d9664927b97 Mon Sep 17 00:00:00 2001 From: lta Date: Wed, 17 Jul 2019 16:51:45 +0800 Subject: [PATCH] add sync CHN docu and memory tool docu --- .../UserGuideV0.7.0/8-System Tools/1-Sync.md | 280 ++++++++++++++++++ .../2-Memory Estimation Tool.md | 90 ++++++ .../{7-Tools-Cli.md => 8-Tools-Cli.md} | 0 ...{7-Tools-Grafana.md => 8-Tools-Grafana.md} | 0 .../{7-Tools-Hadoop.md => 8-Tools-Hadoop.md} | 0 .../{7-Tools-spark.md => 8-Tools-spark.md} | 0 .../UserGuideV0.7.0/0-Content.md | 5 +- .../1-Sync.md} | 34 ++- .../2-Memory Estimation Tool.md | 81 +++++ .../{7-Tools-Cli.md => 8-Tools-Cli.md} | 0 ...{7-Tools-Grafana.md => 8-Tools-Grafana.md} | 0 .../{7-Tools-Hadoop.md => 8-Tools-Hadoop.md} | 0 .../{7-Tools-spark.md => 8-Tools-spark.md} | 0 iotdb/iotdb/bin/memory-tool.sh | 1 - .../adapter/IoTDBConfigDynamicAdapter.java | 11 +- .../db/engine/memtable/ChunkBufferPool.java | 6 +- .../apache/iotdb/db/metadata/MManager.java | 6 +- .../iotdb/db/tools/MemEst/MemEstToolCmd.java | 29 +- 18 files changed, 506 insertions(+), 37 deletions(-) create mode 100644 docs/Documentation-CHN/UserGuideV0.7.0/8-System Tools/1-Sync.md create mode 100644 docs/Documentation-CHN/UserGuideV0.7.0/8-System Tools/2-Memory Estimation Tool.md rename docs/Documentation-CHN/UserGuideV0.7.0/{7-Tools-Cli.md => 8-Tools-Cli.md} (100%) rename docs/Documentation-CHN/UserGuideV0.7.0/{7-Tools-Grafana.md => 8-Tools-Grafana.md} (100%) rename docs/Documentation-CHN/UserGuideV0.7.0/{7-Tools-Hadoop.md => 8-Tools-Hadoop.md} (100%) rename docs/Documentation-CHN/UserGuideV0.7.0/{7-Tools-spark.md => 8-Tools-spark.md} (100%) rename docs/Documentation/UserGuideV0.7.0/{7-Tools-Sync.md => 8-System Tools/1-Sync.md} (90%) create mode 100644 docs/Documentation/UserGuideV0.7.0/8-System Tools/2-Memory Estimation Tool.md rename docs/Documentation/UserGuideV0.7.0/{7-Tools-Cli.md => 8-Tools-Cli.md} (100%) rename docs/Documentation/UserGuideV0.7.0/{7-Tools-Grafana.md => 8-Tools-Grafana.md} (100%) rename docs/Documentation/UserGuideV0.7.0/{7-Tools-Hadoop.md => 8-Tools-Hadoop.md} (100%) rename docs/Documentation/UserGuideV0.7.0/{7-Tools-spark.md => 8-Tools-spark.md} (100%) diff --git a/docs/Documentation-CHN/UserGuideV0.7.0/8-System Tools/1-Sync.md b/docs/Documentation-CHN/UserGuideV0.7.0/8-System Tools/1-Sync.md new file mode 100644 index 0000000000..d9315f523b --- /dev/null +++ b/docs/Documentation-CHN/UserGuideV0.7.0/8-System Tools/1-Sync.md @@ -0,0 +1,280 @@ + + +# 同步工具 + + +- [同步工具](#同步工具) +- [介绍](#介绍) +- [配置参数](#配置参数) + - [同步工具接收端](#同步工具接收端) + - [同步工具发送端](#同步工具发送端) +- [使用方式](#使用方式) + - [启动同步功能接收端](#启动同步功能接收端) + - [关闭同步功能接收端](#关闭同步功能接收端) + - [启动同步功能发送端](#启动同步功能发送端) + - [关闭同步功能发送端](#关闭同步功能发送端) + + +# 介绍 +同步工具是定期将本地磁盘中和新增的已持久化的tsfile文件上传至云端并加载的IoTDB套件工具。 + +在同步工具的发送端，同步模块是一个独立的进程，独立于本地的IoTDB。通过独立的脚本进行启动和关闭(详见章节`使用方式`)，同步的频率周期可由用户设置。 + +在同步工具的的接收端，同步模块内嵌于IoTDB的引擎，和IoTDB处于同一个进程中。同步模块监听一个独立的端口，该端口可由用户设置(详见章节`配置参数`)。用户使用前，需要在同步接收端设置同步白名单，以网段形式表示，接收端的同步模块只接受位于白名单网段中的发送端同步的数据. + +同步工具具有多对一的发送-接受模式，即一个同步接收端可以同时接受多个同步发送端传输的数据，一个同步发送端只能向一个同步接收端发送数据 + +> 注意：在使用同步工具前，同步工具的接收端和发送端需要单独配置。 +# 配置参数 +## 同步工具接收端 +同步工具接收端的参数配置位于IoTDB的配置文件iotdb-engine.properties中，其安装目录为$IOTDB_HOME/conf/iotdb-engine.properties。在该配置文件中，有四个参数和同步接收端有关，配置说明如下: + + + + + + + + + + + + + + + + + + + + + +
参数名: is_sync_enable
描述同步功能开关，配置为true表示接收端允许接收同步的数据并加载，设置为false的时候表示接收端不允许接收同步的数据
类型Boolean
默认值false
改后生效方式重启服务器生效
+ + + + + + + + + + + + + + + + + + + + + +
参数名: IP_white_list
描述设置同步功能发送端IP地址的白名单，以网段的形式表示，多个网段之间用逗号分隔。发送端向接收端同步数据时，只有当该发送端IP地址处于该白名单设置的网段范围内，接收端才允许同步操作。如果白名单为空，则接收端不允许任何发送端同步数据。默认接收端接受全部IP的同步请求。
类型String
默认值0.0.0.0/0
改后生效方式重启服务器生效
+ + + + + + + + + + + + + + + + + + + + + +
参数名: update_historical_data_possibility
描述同步服务端在合并同步的数据时选择的处理策略。如果同步的数据对历史数据(相比本地该存储组数据的最新时间戳)更新占比超过50%，则建议选择策略1，将参数设置为true，使用该策略对IoTDB系统的写入性能产生较大影响，对机器的CPU占用较小；如果同步的数据对历史数据更新占比少于50%，则建议选择策略2，将参数设置为false，使用该策略对IoTDB系统的写入性能产生较小影响，对机器CPU的占用较大。
+
类型Boolean
默认值false
改后生效方式重启服务器生效
+ + + + + + + + + + + + + + + + + + + + + +
参数名: sync_server_port
描述同步接收端服务器监听接口，请确认该端口不是系统保留端口并且未被占用。参数is_sync_enable设置为true时有效，参数is_sync_enable设置为false时无效
类型Short Int : [0,65535]
默认值5555
改后生效方式重启服务器生效
+ +## 同步工具发送端 +同步功能发送端的参数配置在一个单独的配置文件中，其安装目录为```$IOTDB_HOME/conf/iotdb-sync-client.properties```。在该配置文件中，有五个参数和同步发送端有关，配置说明如下： + + + + + + + + + + + + + + + + + + + + +
参数名: server_ip
描述同步接收端的IP地址
类型String
默认值127.0.0.1
改后生效方式重启同步功能发送端生效
+ + + + + + + + + + + + + + + + + + + + + +
参数名: server_port
描述同步接收端服务器监听端口，需要保证该端口和同步接收端配置的监听端口一致
类型Short Int : [0,65535]
默认值5555
改后生效方式重启同步功能发送端生效
+ + + + + + + + + + + + + + + + + + + + + +
参数名: sync_period_in_second
描述同步周期，两次同步任务开始时间的间隔，单位为秒(s)
类型Int : [0,2147483647]
默认值600
改后生效方式重启同步功能发送端生效
+ + + + + + + + + + + + + + + + + +
参数名: iotdb_schema_directory
描述同步发送端的IoTDB schema文件的绝对路径，例如$IOTDB_HOME /data/system/schema/mlog.txt(若用户未手动设置schema元数据的路径，则该路径为默认路径)，该参数默认不生效，用户有需求时进行手动设置
类型String
改后生效方式重启同步功能发送端生效
+ + + + + + + + + + + + + + + + + +
参数名: iotdb_bufferWrite_directory
描述同步发送端的IoTDB 的bufferWrite数据(tsfile文件)目录的绝对路径，定位至bufferWrite目录下，例如： $IOTDB_HOME /data/data/settled(若用户未手动设置数据路径，则该路径为默认路径)，该参数默认不生效，用户有需求时进行手动设置。该参数需要保证和参数iotdb_schema_directory属于同一个IoTDB
类型String
改后生效方式重启同步功能发送端生效
+ +# 使用方式 +## 启动同步功能接收端 +1. 配置接收端的参数，例如： + +2. 启动IoTDB引擎，同步功能接收端会同时启动，启动时LOG日志会出现`IoTDB: start SYNC ServerService successfully`字样，表示同步接收端启动成功，如图所示： + + +## 关闭同步功能接收端 +关闭IoTDB，同步功能接收端会同时关闭。 + +## 启动同步功能发送端 +1. 配置发送端的参数， 如图所示: + +2. 启动同步功能发送端 + +用户可以使用```$IOTDB_HOME/bin```文件夹下的脚本启动同步功能的发送端 +Linux系统与MacOS系统启动命令如下： +``` + Shell >$IOTDB_HOME/bin/start-sync-client.sh +``` +Windows系统启动命令如下： +``` + Shell >$IOTDB_HOME/bin/start-sync-client.bat +``` + + +## 关闭同步功能发送端 + +用户可以使用```$IOTDB_HOME/bin```文件夹下的脚本关闭同步功能的发送端。 +Linux系统与MacOS系统停止命令如下： +``` + Shell >$IOTDB_HOME/bin/stop-sync-client.sh +``` +Windows系统停止命令如下： +``` + Shell >$IOTDB_HOME/bin/stop-sync-client.bat +``` \ No newline at end of file diff --git a/docs/Documentation-CHN/UserGuideV0.7.0/8-System Tools/2-Memory Estimation Tool.md b/docs/Documentation-CHN/UserGuideV0.7.0/8-System Tools/2-Memory Estimation Tool.md new file mode 100644 index 0000000000..b8b2464367 --- /dev/null +++ b/docs/Documentation-CHN/UserGuideV0.7.0/8-System Tools/2-Memory Estimation Tool.md @@ -0,0 +1,90 @@ + + +# 内存预估工具 + + + +- [内存预估工具](#内存预估工具) +- [介绍](#介绍) +- [输入参数](#输入参数) +- [使用方式](#使用方式) + + + +# 介绍 +本工具通过用户输入的若干参数，计算出IoTDB运行此负载的最小写内存。(IoTDB中的内存分为三部分：写内存，读内存，预留内存。写内存是用于数据写入分配的内存，三者的比例可在配置文件中设置)，结果以GB为单位。 + +# 输入参数 +本工具使用时，需要输入的参数如下: + + + + + + + + + + + + + + + + + + + + + + + + + + +
参数名参数说明示例是否必需
-sg | --storagegroup <storage group number>存储组数量-sg 20是
-ts | --timeseries <total timeseries number>总时间序列数量-ts 10000是
-mts | --maxtimeseries <max timeseries>存储组中的最大时间序列的数量，如果时间序列均匀分配在存储组中，本参数可以不设置-mts 10000否
+ +在内存预估时，若工具计算需要较长的时间，则会在下方显示出运行进度，便于用户掌握进度。 + +# 使用方式 + +用户可以使用```$IOTDB_HOME/bin```文件夹下的脚本使用该工具 +Linux系统与MacOS系统启动命令如下： +* 以20个存储组，共10w条时间序列，时间序列在存储组中均分为例： +``` + Shell >$IOTDB_HOME/bin/memory-tool.sh calmem -sg 20 -ts 100000 +``` +* 以20个存储组，共10w条时间序列，存储组中最大时间序列数为50000为例： +``` + Shell >$IOTDB_HOME/bin/memory-tool.sh calmem -sg 20 -ts 100000 -tsm -50000 +``` + +Windows系统启动命令如下： +* 以20个存储组，共10w条时间序列，时间序列在存储组中均分为例： +``` + Shell >$IOTDB_HOME/bin/memory-tool.bat calmem -sg 20 -ts 100000 +``` +* 以20个存储组，共10w条时间序列，存储组中最大时间序列数为50000为例： +``` + Shell >$IOTDB_HOME/bin/memory-tool.bat calmem -sg 20 -ts 100000 -tsm -50000 +``` + diff --git a/docs/Documentation-CHN/UserGuideV0.7.0/7-Tools-Cli.md b/docs/Documentation-CHN/UserGuideV0.7.0/8-Tools-Cli.md similarity index 100% rename from docs/Documentation-CHN/UserGuideV0.7.0/7-Tools-Cli.md rename to docs/Documentation-CHN/UserGuideV0.7.0/8-Tools-Cli.md diff --git a/docs/Documentation-CHN/UserGuideV0.7.0/7-Tools-Grafana.md b/docs/Documentation-CHN/UserGuideV0.7.0/8-Tools-Grafana.md similarity index 100% rename from docs/Documentation-CHN/UserGuideV0.7.0/7-Tools-Grafana.md rename to docs/Documentation-CHN/UserGuideV0.7.0/8-Tools-Grafana.md diff --git a/docs/Documentation-CHN/UserGuideV0.7.0/7-Tools-Hadoop.md b/docs/Documentation-CHN/UserGuideV0.7.0/8-Tools-Hadoop.md similarity index 100% rename from docs/Documentation-CHN/UserGuideV0.7.0/7-Tools-Hadoop.md rename to docs/Documentation-CHN/UserGuideV0.7.0/8-Tools-Hadoop.md diff --git a/docs/Documentation-CHN/UserGuideV0.7.0/7-Tools-spark.md b/docs/Documentation-CHN/UserGuideV0.7.0/8-Tools-spark.md similarity index 100% rename from docs/Documentation-CHN/UserGuideV0.7.0/7-Tools-spark.md rename to docs/Documentation-CHN/UserGuideV0.7.0/8-Tools-spark.md diff --git a/docs/Documentation/UserGuideV0.7.0/0-Content.md b/docs/Documentation/UserGuideV0.7.0/0-Content.md index a741e154de..48016aeaa3 100644 --- a/docs/Documentation/UserGuideV0.7.0/0-Content.md +++ b/docs/Documentation/UserGuideV0.7.0/0-Content.md @@ -47,4 +47,7 @@ * 1-IoTDB Query Statement * 2-Reference # Chapter 6: JDBC API -* 1-JDBC API \ No newline at end of file +* 1-JDBC API +# Chapter 8: System Tools +* 1-Sync.md +* 2-Memory estimation tool.md \ No newline at end of file diff --git a/docs/Documentation/UserGuideV0.7.0/7-Tools-Sync.md b/docs/Documentation/UserGuideV0.7.0/8-System Tools/1-Sync.md similarity index 90% rename from docs/Documentation/UserGuideV0.7.0/7-Tools-Sync.md rename to docs/Documentation/UserGuideV0.7.0/8-System Tools/1-Sync.md index 155d85ec7f..01a4d88508 100644 --- a/docs/Documentation/UserGuideV0.7.0/7-Tools-Sync.md +++ b/docs/Documentation/UserGuideV0.7.0/8-System Tools/1-Sync.md @@ -19,19 +19,23 @@ --> - +# Chapter 7: System Tools + +## Data Import -## Outline + -- Introduction -- Configuration - - Sync Receiver - - Sync Sender -- Usage - - Start Sync Receiver - - Stop Sync Receiver - - Start Sync Sender - - Stop Sync Sender +- [Chapter 7: System Tools](#chapter-7-system-tools) + - [Data Import](#data-import) +- [Introduction](#introduction) +- [Configuration](#configuration) + - [Sync Receiver](#sync-receiver) + - [Sync Sender](#sync-sender) +- [Usage](#usage) + - [Start Sync Receiver](#start-sync-receiver) + - [Stop Sync Receiver](#stop-sync-receiver) + - [Start Sync Sender](#start-sync-sender) + - [Stop Sync Sender](#stop-sync-sender) # Introduction @@ -54,7 +58,7 @@ The parameter configuration of the sync receiver is located in the configuration Description - Sync function switch, which is configured as true to indicate that the receiver is allowed to receive the data from sender and load it. When set to false, it means that the receiver is not allowed to receive the data from any sender. + Sync function switch, which is configured as true to indicate that the receiver is allowed to receive the data from the sender and load it. When set to false, it means that the receiver is not allowed to receive the data from any sender. Type @@ -140,7 +144,7 @@ The parameter configuration of the sync receiver is located in the configuration ## Sync Sender -The parameters of the sync sender are configured in a separate configuration file iotdb-postbackClient.pro-perties with the installation directory of ```$IOTDB_HOME/conf/iotdb-sync-client.properties```. In this configuration file, there are five parameters related to the sync sender. The configuration instructions are as follows: +The parameters of the sync sender are configured in a separate configuration file iotdb-sync-client.properties with the installation directory of ```$IOTDB_HOME/conf/iotdb-sync-client.properties```. In this configuration file, there are five parameters related to the sync sender. The configuration instructions are as follows: @@ -257,7 +261,7 @@ Stop IoTDB and the sync receiver will be closed at the same time. 1. Set up parameters of sync sender. For example: 2. Start sync sender -Users can use the scripts under the $IOTDB_HOME/bin folder to start the sync sender. +Users can use the scripts under the ```$IOTDB_HOME/bin``` folder to start the sync sender. For Linux and Mac OS X users: ``` Shell >$IOTDB_HOME/bin/start-sync-client.sh @@ -269,7 +273,7 @@ For Windows users: ## Stop Sync Sender -Users can use the scripts under the $IOTDB_HOME/bin folder to stop the sync sender. +Users can use the scripts under the ```$IOTDB_HOME/bin``` folder to stop the sync sender. For Linux and Mac OS X users: ``` Shell >$IOTDB_HOME/bin/stop-sync-client.sh diff --git a/docs/Documentation/UserGuideV0.7.0/8-System Tools/2-Memory Estimation Tool.md b/docs/Documentation/UserGuideV0.7.0/8-System Tools/2-Memory Estimation Tool.md new file mode 100644 index 0000000000..9b9a1cea10 --- /dev/null +++ b/docs/Documentation/UserGuideV0.7.0/8-System Tools/2-Memory Estimation Tool.md @@ -0,0 +1,81 @@ + + +# Memory Estimation Tool + +# Introduction +This tool calculates the minimum memory for writing to meet specific workload through a number of parameters input by users. (Memory in IoTDB is divided into three parts: write memory, read memory and reserve memory. Write memory is used for data write allocation. The ratio of the three can be set in the configuration file) The unit of result is in GB. + +# Input parameters +When using this tool, the parameters needed to be input are as follows: +
parameter: server_ip
+ + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterParameter DescriptionExampleNecessary
-sg | --storagegroup <storage group number>storage group number-sg 20true
-ts | --timeseries <total timeseries number>total timeseries number-ts 10000true
-mts | --maxtimeseries <max timeseries>maximum number of timeseries among storage groups.If the time series are evenly distributed in the storage group, this parameter may not be set.-mts 10000false
+ +In memory estimation, if the calculation takes a long time, the tool will show the running progress below, which is convenient for users to master the progress. + +# Usage + +Users can use the tool using scripts under the ``IOTDB_HOME/bin`folder. +For Linux and Mac OS X users: +* Assume that there are 20 storage groups, 10w timeseries and timeseries are evenly distributed in the storage groups: +``` + Shell >$IOTDB_HOME/bin/memory-tool.sh calmem -sg 20 -ts 100000 +``` +* Assume that there are 20 storage groups, 10w timeseries and maximum timeseries number among storage groups is 50000: +``` + Shell >$IOTDB_HOME/bin/memory-tool.sh calmem -sg 20 -ts 100000 -tsm -50000 +``` + +For Windows users: +* Assume that there are 20 storage groups, 10w timeseries and timeseries are evenly distributed in the storage groups: +``` + Shell >$IOTDB_HOME/bin/memory-tool.bat calmem -sg 20 -ts 100000 +``` +* Assume that there are 20 storage groups, 10w timeseries and maximum timeseries number among storage groups is 50000: +``` + Shell >$IOTDB_HOME/bin/memory-tool.bat calmem -sg 20 -ts 100000 -tsm -50000 +``` + diff --git a/docs/Documentation/UserGuideV0.7.0/7-Tools-Cli.md b/docs/Documentation/UserGuideV0.7.0/8-Tools-Cli.md similarity index 100% rename from docs/Documentation/UserGuideV0.7.0/7-Tools-Cli.md rename to docs/Documentation/UserGuideV0.7.0/8-Tools-Cli.md diff --git a/docs/Documentation/UserGuideV0.7.0/7-Tools-Grafana.md b/docs/Documentation/UserGuideV0.7.0/8-Tools-Grafana.md similarity index 100% rename from docs/Documentation/UserGuideV0.7.0/7-Tools-Grafana.md rename to docs/Documentation/UserGuideV0.7.0/8-Tools-Grafana.md diff --git a/docs/Documentation/UserGuideV0.7.0/7-Tools-Hadoop.md b/docs/Documentation/UserGuideV0.7.0/8-Tools-Hadoop.md similarity index 100% rename from docs/Documentation/UserGuideV0.7.0/7-Tools-Hadoop.md rename to docs/Documentation/UserGuideV0.7.0/8-Tools-Hadoop.md diff --git a/docs/Documentation/UserGuideV0.7.0/7-Tools-spark.md b/docs/Documentation/UserGuideV0.7.0/8-Tools-spark.md similarity index 100% rename from docs/Documentation/UserGuideV0.7.0/7-Tools-spark.md rename to docs/Documentation/UserGuideV0.7.0/8-Tools-spark.md diff --git a/iotdb/iotdb/bin/memory-tool.sh b/iotdb/iotdb/bin/memory-tool.sh index 5c2ad0d428..77e3876e42 100755 --- a/iotdb/iotdb/bin/memory-tool.sh +++ b/iotdb/iotdb/bin/memory-tool.sh @@ -23,7 +23,6 @@ if [ -z "${IOTDB_HOME}" ]; then fi IOTDB_CONF=${IOTDB_HOME}/conf -# IOTDB_LOGS=${IOTDB_HOME}/logs CLASSPATH="" for f in ${IOTDB_HOME}/lib/*.jar; do diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java index 3e809e3817..0a24371572 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java @@ -129,8 +129,13 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter { LOGGER.debug("memtableSizeInByte {} is smaller than memTableSizeFloorThreshold {}", memtableSizeInByte, memTableSizeFloorThreshold); tsFileSize = calcTsFileSize(memTableSizeFloorThreshold); - memtableSizeInByte = memTableSizeFloorThreshold + ((tsFileSize - memTableSizeFloorThreshold) >> 1); - if (tsFileSize < memTableSizeFloorThreshold) { + if(tsFileSize > memTableSizeFloorThreshold) { + memtableSizeInByte = + memTableSizeFloorThreshold + ((tsFileSize - memTableSizeFloorThreshold) >> 1); + } else { + memtableSizeInByte = memTableSizeFloorThreshold; + } + if (tsFileSize * CompressionRatio.getInstance().getRatio() < memTableSizeFloorThreshold) { canAdjust = false; } } @@ -192,7 +197,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter { * occupied by each value is 8 bytes. The reason for multiplying 2 is that the timestamp also * takes 8 bytes. */ - private int getMemTableSizeFloorThreshold() { + private long getMemTableSizeFloorThreshold() { return MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups() * PrimitiveArrayPool.ARRAY_SIZE * Long.BYTES * 2; } diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java index af5a6995ac..c4f642f0e2 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java @@ -37,7 +37,7 @@ public class ChunkBufferPool { private static final Deque availableChunkBuffer = new ArrayDeque<>(); - private int size = 0; + private long size = 0; private static final int WAIT_TIME = 2000; @@ -52,7 +52,7 @@ public class ChunkBufferPool { synchronized (availableChunkBuffer) { //we use the memtable number * maximal series number in one StroageGroup * 2 as the capacity - int capacity = + long capacity = 2 * MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups() * IoTDBDescriptor .getInstance().getConfig().getMaxMemtableNumber() + 100000; if (availableChunkBuffer.isEmpty() && size < capacity) { @@ -91,7 +91,7 @@ public class ChunkBufferPool { synchronized (availableChunkBuffer) { chunkBuffer.reset(); //we use the memtable number * maximal series number in one StroageGroup as the capacity - int capacity = + long capacity = MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups() * IoTDBDescriptor .getInstance().getConfig().getMaxMemtableNumber(); if (size > capacity) { diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java index 6459b2b273..b23253e9f2 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java @@ -74,7 +74,7 @@ public class MManager { private RandomDeleteCache mNodeCache; private Map seriesNumberInStorageGroups = new HashMap<>(); - private int maxSeriesNumberAmongStorageGroup; + private long maxSeriesNumberAmongStorageGroup; private boolean initialized; private MManager() { @@ -1201,11 +1201,11 @@ public class MManager { /** * Only for test */ - public void setMaxSeriesNumberAmongStorageGroup(int maxSeriesNumberAmongStorageGroup) { + public void setMaxSeriesNumberAmongStorageGroup(long maxSeriesNumberAmongStorageGroup) { this.maxSeriesNumberAmongStorageGroup = maxSeriesNumberAmongStorageGroup; } - public int getMaximalSeriesNumberAmongStorageGroups() { + public long getMaximalSeriesNumberAmongStorageGroups() { return maxSeriesNumberAmongStorageGroup; } diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java index ff3a8b9c7a..c2ebff7822 100644 --- a/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java +++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java @@ -31,16 +31,16 @@ import org.apache.iotdb.db.metadata.MManager; @Command(name = "calmem", description = "calculate minimum memory required for writing based on the number of storage groups and timeseries") public class MemEstToolCmd implements Runnable { - @Option(type = OptionType.GLOBAL, title = "storage group number", name = {"-sg", + @Option(title = "storage group number", name = {"-sg", "--storagegroup"}, description = "Storage group number") private String sgNumString = "10"; - @Option(type = OptionType.GLOBAL, title = "total timeseries number", name = {"-ts", - "--tsNum"}, description = "Total timeseries number") + @Option(title = "total timeseries number", name = {"-ts", + "--timeseries"}, description = "Total timeseries number") private String tsNumString = "1000"; @Option(title = "max timeseries", name = {"-mts", - "--mtsNum"}, description = "Maximum timeseries number among storage groups, make sure that it's smaller than total timeseries number") + "--maxtimeseries"}, description = "Maximum timeseries number among storage groups, make sure that it's smaller than total timeseries number") private String maxTsNumString = "0"; @Override @@ -50,9 +50,10 @@ public class MemEstToolCmd implements Runnable { int maxMemtableNumber = config.getMaxMemtableNumber(); long tsFileSize = config.getTsFileSizeThreshold(); long memory = IoTDBConstant.GB; - int sgNum = Integer.parseInt(sgNumString); - int tsNum = Integer.parseInt(tsNumString); - int maxTsNum = Integer.parseInt(maxTsNumString); + long sgNum = Long.parseLong(sgNumString); + long tsNum = Long.parseLong(tsNumString); + long maxTsNum = Long.parseLong(maxTsNumString); + long maxTsNumValid = maxTsNum; while (true) { // init parameter config.setAllocateMemoryForWrite(memory); @@ -71,8 +72,13 @@ public class MemEstToolCmd implements Runnable { } for (; tsCnt <= tsNum; tsCnt++) { IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1); - MManager.getInstance().setMaxSeriesNumberAmongStorageGroup( - maxTsNum == 0 ? tsCnt / sgNum + 1 : Math.min(tsCnt, maxTsNum)); + if(maxTsNum == 0){ + maxTsNumValid = tsCnt / sgNum + 1; + } else { + maxTsNumValid = Math.min(tsCnt, maxTsNum); + maxTsNumValid = Math.max(maxTsNumValid, tsCnt / sgNum + 1); + } + MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(maxTsNumValid); } } catch (ConfigAdjusterException e) { @@ -85,7 +91,8 @@ public class MemEstToolCmd implements Runnable { } break; } - System.out.println(String.format("SG: %d, TS: %d, MTS: %d, Memory for writing: %dGB", sgNum, - tsNum, maxTsNum == 0 ? tsNum / sgNum + 1 : maxTsNum, memory / IoTDBConstant.GB)); + System.out.println(String + .format("Memory for writing: %dGB, SG: %d, TS: %d, MTS: %d", memory / IoTDBConstant.GB, + sgNum, tsNum, maxTsNumValid)); } } -- GitLab