diff --git a/README.md b/README.md
index 11074c8b118a792fd449044afc0a172eab59eadf..f62258238f212800dc791604efd6db324413f970 100644
--- a/README.md
+++ b/README.md
@@ -96,8 +96,8 @@ then by default `IOTDB_HOME` is the direct parent directory of `sbin/start-serve
(or that of `sbin\start-server.bat` on Windows).
* if `IOTDB_CLI_HOME` is not explicitly assigned,
-then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-client.sh` on
-Unix/OS X (or that of `sbin\start-client.bat` on Windows).
+then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-cli.sh` on
+Unix/OS X (or that of `sbin\start-cli.bat` on Windows).
If you are not the first time that building IoTDB, remember deleting the following files:
@@ -199,10 +199,10 @@ After build, the IoTDB client will be at the folder "client/target/iotdb-client-
```
# Unix/OS X
-> $IOTDB_CLI_HOME/sbin/start-client.sh -h -p -u
+> $IOTDB_CLI_HOME/sbin/start-cli.sh -h -p -u
# Windows
-> $IOTDB_CLI_HOME\sbin\start-client.bat -h -p -u
+> $IOTDB_CLI_HOME\sbin\start-cli.bat -h -p -u
```
> NOTE: In the system, we set a default user in IoTDB named 'root'. The default password for 'root' is 'root'. You can use this default user if you are making the first try or you didn't create users by yourself.
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 16330850588aa287f28c96cf5aa89009a687604f..10745b267eea395ad50a0500855ba8da8f8c9f1d 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -130,7 +130,7 @@ If you use the previous unofficial version 0.7.0. It is incompatible with 0.8.0.
* IOTDB-44 Error message in server log when select timeseries
* IOTDB-49 Authorizer module outputs too many debug log info
* IOTDB-50 DataSetWithoutTimeGenerator's initHeap behaves wrongly
-* IOTDB-52 Client doesn't support aggregate
+* IOTDB-52 Cli doesn't support aggregate
* IOTDB-54 Predicates doesn't take effect
* IOTDB-67 ValueDecoder reading new page bug
* IOTDB-70 Disconnect from server when logging in fails
@@ -147,7 +147,7 @@ If you use the previous unofficial version 0.7.0. It is incompatible with 0.8.0.
* IOTDB-103 Does not give a hint when encountering unsupported data types
* IOTDB-104 MManager is incorrectly recovered when system reboots
* IOTDB-108 Mistakes in documents
-* IOTDB-110 Clients inserts data normally even if there is no space left on the disk
+* IOTDB-110 Clis inserts data normally even if there is no space left on the disk
* IOTDB-118 When the disk space is full, the storage group is created successfully
* IOTDB-121 A bug of query on value columns
* IOTDB-128 Probably a bug in iotdb official website
diff --git a/client/pom.xml b/client/pom.xml
index 07da5c62323642b2d49b2b68d7dbe9b989da6fb8..ef1ea30ae9a655c71d9fb87015b746665d77fbca 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -29,7 +29,6 @@
iotdb-client
IoTDB Client
- A Client tool.
false
${cli.test.skip}
diff --git a/client/src/assembly/resources/sbin/start-client.bat b/client/src/assembly/resources/sbin/start-client.bat
index b7f2dbb1691500ca280e9016665ef75fb112ca76..2642dfd9860e13478468b72b713c8f99bb32bc15 100755
--- a/client/src/assembly/resources/sbin/start-client.bat
+++ b/client/src/assembly/resources/sbin/start-client.bat
@@ -28,7 +28,7 @@ pushd %~dp0..
if NOT DEFINED IOTDB_CLI_HOME set IOTDB_CLI_HOME=%CD%
popd
-if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cli.client.Client
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.client.Client
if NOT DEFINED JAVA_HOME goto :err
@REM -----------------------------------------------------------------------------
diff --git a/client/src/assembly/resources/sbin/start-client.sh b/client/src/assembly/resources/sbin/start-client.sh
index 70e85037479569de7214ca26793ee367b07b0c34..4591159ddf783170aaae7711f6c4e86127904506 100755
--- a/client/src/assembly/resources/sbin/start-client.sh
+++ b/client/src/assembly/resources/sbin/start-client.sh
@@ -24,7 +24,7 @@ if [ -z "${IOTDB_CLI_HOME}" ]; then
fi
-MAIN_CLASS=org.apache.iotdb.cli.client.Client
+MAIN_CLASS=org.apache.iotdb.client.Client
CLASSPATH=""
diff --git a/client/src/assembly/resources/tools/export-csv.bat b/client/src/assembly/resources/tools/export-csv.bat
index a5c1ce26e9384aa2a0a40980d3319b51c113543d..f2432b8b6ce4d6c781be5bd2e2e0630528535145 100644
--- a/client/src/assembly/resources/tools/export-csv.bat
+++ b/client/src/assembly/resources/tools/export-csv.bat
@@ -28,7 +28,7 @@ pushd %~dp0..
if NOT DEFINED IOTDB_CLI_HOME set IOTDB_CLI_HOME=%CD%
popd
-if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cli.tool.ExportCsv
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ExportCsv
if NOT DEFINED JAVA_HOME goto :err
@REM -----------------------------------------------------------------------------
diff --git a/client/src/assembly/resources/tools/export-csv.sh b/client/src/assembly/resources/tools/export-csv.sh
index 940e2416e2f59406bbffb0fb2b70e5ed69e455f0..375e461361214a03b0767792c35887f817545440 100755
--- a/client/src/assembly/resources/tools/export-csv.sh
+++ b/client/src/assembly/resources/tools/export-csv.sh
@@ -47,7 +47,7 @@ for f in ${IOTDB_CLI_HOME}/lib/*.jar; do
CLASSPATH=${CLASSPATH}":"$f
done
-MAIN_CLASS=org.apache.iotdb.cli.tool.ExportCsv
+MAIN_CLASS=org.apache.iotdb.tool.ExportCsv
"$JAVA" -DIOTDB_CLI_HOME=${IOTDB_CLI_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
exit $?
\ No newline at end of file
diff --git a/client/src/assembly/resources/tools/import-csv.bat b/client/src/assembly/resources/tools/import-csv.bat
index 1d4cab7e1a82d7c8793d8215fe6fff781b7b64fe..ea514eae46acb077f70d46206c1fbd492855071a 100644
--- a/client/src/assembly/resources/tools/import-csv.bat
+++ b/client/src/assembly/resources/tools/import-csv.bat
@@ -28,7 +28,7 @@ pushd %~dp0..
if NOT DEFINED IOTDB_CLI_HOME set IOTDB_CLI_HOME=%CD%
popd
-if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cli.tool.ImportCsv
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ImportCsv
if NOT DEFINED JAVA_HOME goto :err
@REM -----------------------------------------------------------------------------
diff --git a/client/src/assembly/resources/tools/import-csv.sh b/client/src/assembly/resources/tools/import-csv.sh
index 80b700b7b97dd57413baa7c84fcf154de7587f9e..413dc4273431770023a4eaf6762681e4f189dc3f 100755
--- a/client/src/assembly/resources/tools/import-csv.sh
+++ b/client/src/assembly/resources/tools/import-csv.sh
@@ -47,7 +47,7 @@ for f in ${IOTDB_CLI_HOME}/lib/*.jar; do
CLASSPATH=${CLASSPATH}":"$f
done
-MAIN_CLASS=org.apache.iotdb.cli.tool.ImportCsv
+MAIN_CLASS=org.apache.iotdb.tool.ImportCsv
"$JAVA" -DIOTDB_CLI_HOME=${IOTDB_CLI_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
exit $?
\ No newline at end of file
diff --git a/client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java b/client/src/main/java/org/apache/iotdb/client/AbstractClient.java
similarity index 80%
rename from client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java
rename to client/src/main/java/org/apache/iotdb/client/AbstractClient.java
index 3cbf4522eec470491abf107b397388eec98d7e7e..c35cb0801608ca834a84920d5ecebf42ab248e8f 100644
--- a/client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java
+++ b/client/src/main/java/org/apache/iotdb/client/AbstractClient.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.client;
+package org.apache.iotdb.client;
import java.io.PrintStream;
import java.sql.ResultSet;
@@ -38,65 +38,64 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.iotdb.cli.exception.ArgsErrorException;
-import org.apache.iotdb.cli.tool.ImportCsv;
+import org.apache.iotdb.exception.ArgsErrorException;
+import org.apache.iotdb.tool.ImportCsv;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.iotdb.jdbc.IoTDBDatabaseMetadata;
import org.apache.iotdb.jdbc.IoTDBMetadataResultSet;
import org.apache.iotdb.jdbc.IoTDBQueryResultSet;
-import org.apache.iotdb.jdbc.IoTDBSQLException;
import org.apache.iotdb.service.rpc.thrift.ServerProperties;
import org.apache.thrift.TException;
public abstract class AbstractClient {
- protected static final String HOST_ARGS = "h";
- protected static final String HOST_NAME = "host";
-
- protected static final String HELP_ARGS = "help";
-
- protected static final String PORT_ARGS = "p";
- protected static final String PORT_NAME = "port";
-
- protected static final String PASSWORD_ARGS = "pw";
- protected static final String PASSWORD_NAME = "password";
-
- protected static final String USERNAME_ARGS = "u";
- protected static final String USERNAME_NAME = "username";
-
- protected static final String EXECUTE_ARGS = "e";
- protected static final String EXECUTE_NAME = "execute";
-
- protected static final String ISO8601_ARGS = "disableISO8601";
- protected static final List AGGREGRATE_TIME_LIST = new ArrayList<>();
- protected static final String MAX_PRINT_ROW_COUNT_ARGS = "maxPRC";
- protected static final String MAX_PRINT_ROW_COUNT_NAME = "maxPrintRowCount";
- protected static final String SET_MAX_DISPLAY_NUM = "set max_display_num";
- protected static final String SET_TIMESTAMP_DISPLAY = "set time_display_type";
- protected static final String SHOW_TIMESTAMP_DISPLAY = "show time_display_type";
- protected static final String SET_TIME_ZONE = "set time_zone";
- protected static final String SHOW_TIMEZONE = "show time_zone";
- protected static final String SET_FETCH_SIZE = "set fetch_size";
- protected static final String SHOW_FETCH_SIZE = "show fetch_size";
- protected static final String HELP = "help";
- protected static final String IOTDB_CLI_PREFIX = "IoTDB";
- protected static final String SCRIPT_HINT = "./start-client.sh(start-client.bat if Windows)";
- protected static final String QUIT_COMMAND = "quit";
- protected static final String EXIT_COMMAND = "exit";
- protected static final String SHOW_METADATA_COMMAND = "show timeseries";
- protected static final int MAX_HELP_CONSOLE_WIDTH = 88;
- protected static final String TIMESTAMP_STR = "Time";
- protected static final int ISO_DATETIME_LEN = 35;
- protected static final String IMPORT_CMD = "import";
+ static final String HOST_ARGS = "h";
+ static final String HOST_NAME = "host";
+
+ static final String HELP_ARGS = "help";
+
+ static final String PORT_ARGS = "p";
+ static final String PORT_NAME = "port";
+
+ static final String PASSWORD_ARGS = "pw";
+ private static final String PASSWORD_NAME = "password";
+
+ static final String USERNAME_ARGS = "u";
+ static final String USERNAME_NAME = "username";
+
+ private static final String EXECUTE_ARGS = "e";
+ private static final String EXECUTE_NAME = "execute";
+
+ static final String ISO8601_ARGS = "disableISO8601";
+ static final List AGGREGRATE_TIME_LIST = new ArrayList<>();
+ static final String MAX_PRINT_ROW_COUNT_ARGS = "maxPRC";
+ private static final String MAX_PRINT_ROW_COUNT_NAME = "maxPrintRowCount";
+ static final String SET_MAX_DISPLAY_NUM = "set max_display_num";
+ static final String SET_TIMESTAMP_DISPLAY = "set time_display_type";
+ static final String SHOW_TIMESTAMP_DISPLAY = "show time_display_type";
+ static final String SET_TIME_ZONE = "set time_zone";
+ static final String SHOW_TIMEZONE = "show time_zone";
+ static final String SET_FETCH_SIZE = "set fetch_size";
+ static final String SHOW_FETCH_SIZE = "show fetch_size";
+ private static final String HELP = "help";
+ static final String IOTDB_CLI_PREFIX = "IoTDB";
+ static final String SCRIPT_HINT = "./start-client.sh(start-client.bat if Windows)";
+ static final String QUIT_COMMAND = "quit";
+ static final String EXIT_COMMAND = "exit";
+ static final String SHOW_METADATA_COMMAND = "show timeseries";
+ static final int MAX_HELP_CONSOLE_WIDTH = 88;
+ static final String TIMESTAMP_STR = "Time";
+ static final int ISO_DATETIME_LEN = 35;
+ private static final String IMPORT_CMD = "import";
private static final String NEED_NOT_TO_PRINT_TIMESTAMP = "AGGREGATION";
private static final String DEFAULT_TIME_FORMAT = "default";
- protected static String timeFormat = DEFAULT_TIME_FORMAT;
- protected static int maxPrintRowCount = 1000;
- protected static int fetchSize = 10000;
- protected static int maxTimeLength = ISO_DATETIME_LEN;
- protected static int maxValueLength = 15;
- protected static boolean isQuit = false;
- protected static String TIMESTAMP_PRECISION = "ms";
+ private static String timeFormat = DEFAULT_TIME_FORMAT;
+ static int maxPrintRowCount = 1000;
+ private static int fetchSize = 10000;
+ static int maxTimeLength = ISO_DATETIME_LEN;
+ static int maxValueLength = 15;
+ private static boolean isQuit = false;
+ static String TIMESTAMP_PRECISION = "ms";
/**
* control the width of columns for 'show timeseries path' and 'show storage group'.
@@ -138,22 +137,22 @@ public abstract class AbstractClient {
*
*
*/
- protected static int[] maxValueLengthForShow = new int[]{75, 45, 8, 8};
- protected static String formatTime = "%" + maxTimeLength + "s|";
- protected static String formatValue = "%" + maxValueLength + "s|";
+ private static int[] maxValueLengthForShow = new int[]{75, 45, 8, 8};
+ static String formatTime = "%" + maxTimeLength + "s|";
+ private static String formatValue = "%" + maxValueLength + "s|";
private static final int DIVIDING_LINE_LENGTH = 40;
- protected static String host = "127.0.0.1";
- protected static String port = "6667";
- protected static String username;
- protected static String password;
- protected static String execute;
- protected static boolean hasExecuteSQL = false;
+ static String host = "127.0.0.1";
+ static String port = "6667";
+ static String username;
+ static String password;
+ static String execute;
+ static boolean hasExecuteSQL = false;
- protected static boolean printToConsole = true;
+ private static boolean printToConsole = true;
- protected static Set keywordSet = new HashSet<>();
+ static Set keywordSet = new HashSet<>();
- protected static ServerProperties properties = null;
+ static ServerProperties properties = null;
private static boolean printHeader = false;
private static int displayCnt = 0;
@@ -165,7 +164,7 @@ public abstract class AbstractClient {
*/
private static boolean showException = false;
- protected static void init() {
+ static void init() {
keywordSet.add("-" + HOST_ARGS);
keywordSet.add("-" + HELP_ARGS);
keywordSet.add("-" + PORT_ARGS);
@@ -177,14 +176,14 @@ public abstract class AbstractClient {
}
/**
- * CLI result output.
+ * Client result output.
*
* @param res result set
* @param printToConsole print to console
* @param zoneId time-zone ID
* @throws SQLException SQLException
*/
- public static void output(ResultSet res, boolean printToConsole, ZoneId zoneId)
+ private static void output(ResultSet res, boolean printToConsole, ZoneId zoneId)
throws SQLException {
int cnt = 0;
boolean printTimestamp = true;
@@ -225,15 +224,14 @@ public abstract class AbstractClient {
}
}
- println(StringUtils.repeat('-', DIVIDING_LINE_LENGTH));
printCount(isShow, res, cnt);
}
- protected static String getTimestampPrecision() {
+ private static String getTimestampPrecision() {
return TIMESTAMP_PRECISION;
}
- protected static void printCount(boolean isShow, ResultSet res, int cnt) throws SQLException {
+ private static void printCount(boolean isShow, ResultSet res, int cnt) throws SQLException {
if (isShow) {
int type = res.getType();
if (type == IoTDBMetadataResultSet.MetadataType.STORAGE_GROUP.ordinal()) { // storage group
@@ -247,7 +245,7 @@ public abstract class AbstractClient {
}
}
- protected static void printRow(boolean printTimestamp, int colCount,
+ private static void printRow(boolean printTimestamp, int colCount,
ResultSetMetaData resultSetMetaData, boolean isShow, ResultSet res, ZoneId zoneId)
throws SQLException {
// Output Labels
@@ -263,7 +261,7 @@ public abstract class AbstractClient {
}
}
- protected static void printHeader(boolean printTimestamp, int colCount,
+ private static void printHeader(boolean printTimestamp, int colCount,
ResultSetMetaData resultSetMetaData, boolean isShow) throws SQLException {
if (!printHeader) {
printBlockLine(printTimestamp, colCount, resultSetMetaData, isShow);
@@ -273,7 +271,7 @@ public abstract class AbstractClient {
}
}
- protected static void printShow(int colCount, ResultSet res) throws SQLException {
+ private static void printShow(int colCount, ResultSet res) throws SQLException {
print("|");
for (int i = 1; i <= colCount; i++) {
formatValue = "%" + maxValueLengthForShow[i - 1] + "s|";
@@ -282,7 +280,7 @@ public abstract class AbstractClient {
println();
}
- protected static void printRowData(boolean printTimestamp, ResultSet res, ZoneId zoneId,
+ private static void printRowData(boolean printTimestamp, ResultSet res, ZoneId zoneId,
ResultSetMetaData resultSetMetaData, int colCount)
throws SQLException {
if (displayCnt < maxPrintRowCount) { // NOTE displayCnt only works on queried data results
@@ -298,7 +296,7 @@ public abstract class AbstractClient {
}
}
- protected static void printColumnData(ResultSetMetaData resultSetMetaData, ResultSet res, int i,
+ private static void printColumnData(ResultSetMetaData resultSetMetaData, ResultSet res, int i,
ZoneId zoneId) throws SQLException {
boolean flag = false;
for (String timeStr : AGGREGRATE_TIME_LIST) {
@@ -319,7 +317,7 @@ public abstract class AbstractClient {
}
}
- protected static Options createOptions() {
+ static Options createOptions() {
Options options = new Options();
Option help = new Option(HELP_ARGS, false, "Display help information(optional)");
help.setRequired(false);
@@ -420,7 +418,7 @@ public abstract class AbstractClient {
}
}
- protected static String checkRequiredArg(String arg, String name, CommandLine commandLine,
+ static String checkRequiredArg(String arg, String name, CommandLine commandLine,
boolean isRequired,
String defaultValue) throws ArgsErrorException {
String str = commandLine.getOptionValue(arg);
@@ -442,7 +440,7 @@ public abstract class AbstractClient {
return str;
}
- protected static void setTimeFormat(String newTimeFormat) {
+ static void setTimeFormat(String newTimeFormat) {
switch (newTimeFormat.trim().toLowerCase()) {
case "long":
case "number":
@@ -475,7 +473,7 @@ public abstract class AbstractClient {
}
}
- protected static void setMaxDisplayNumber(String maxDisplayNum) {
+ static void setMaxDisplayNumber(String maxDisplayNum) {
long tmp = Long.parseLong(maxDisplayNum.trim());
if (tmp > Integer.MAX_VALUE || tmp < 0) {
maxPrintRowCount = Integer.MAX_VALUE;
@@ -484,7 +482,7 @@ public abstract class AbstractClient {
}
}
- protected static void printBlockLine(boolean printTimestamp, int colCount,
+ private static void printBlockLine(boolean printTimestamp, int colCount,
ResultSetMetaData resultSetMetaData,
boolean isShowTs) throws SQLException {
StringBuilder blockLine = new StringBuilder();
@@ -512,7 +510,7 @@ public abstract class AbstractClient {
println(blockLine);
}
- protected static void printName(boolean printTimestamp, int colCount,
+ private static void printName(boolean printTimestamp, int colCount,
ResultSetMetaData resultSetMetaData,
boolean isShowTs) throws SQLException {
print("|");
@@ -533,7 +531,7 @@ public abstract class AbstractClient {
println();
}
- protected static String[] removePasswordArgs(String[] args) {
+ static String[] removePasswordArgs(String[] args) {
int index = -1;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-" + PASSWORD_ARGS)) {
@@ -548,7 +546,7 @@ public abstract class AbstractClient {
return args;
}
- protected static String[] processExecuteArgs(String[] args) {
+ static String[] processExecuteArgs(String[] args) {
int index = -1;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-" + EXECUTE_ARGS)) {
@@ -574,7 +572,7 @@ public abstract class AbstractClient {
}
}
- protected static void displayLogo(String version) {
+ static void displayLogo(String version) {
println(" _____ _________ ______ ______ \n"
+ "|_ _| | _ _ ||_ _ `.|_ _ \\ \n"
+ " | | .--.|_/ | | \\_| | | `. \\ | |_) | \n"
@@ -584,13 +582,13 @@ public abstract class AbstractClient {
+ " \n");
}
- protected static void echoStarting(){
+ static void echoStarting(){
println("---------------------");
println("Starting IoTDB Client");
println("---------------------");
}
- protected static OperationResult handleInputCmd(String cmd, IoTDBConnection connection) {
+ static OperationResult handleInputCmd(String cmd, IoTDBConnection connection) {
String specialCmd = cmd.toLowerCase().trim();
if (QUIT_COMMAND.equals(specialCmd) || EXIT_COMMAND.equals(specialCmd)) {
@@ -647,7 +645,7 @@ public abstract class AbstractClient {
return OperationResult.NO_OPER;
}
- protected static void showHelp() {
+ private static void showHelp() {
println(" \t\t\t execute your sql statment");
println(String.format(" %s\t\t show how many timeseries are in iotdb",
SHOW_METADATA_COMMAND));
@@ -665,7 +663,7 @@ public abstract class AbstractClient {
SET_MAX_DISPLAY_NUM));
}
- protected static void showMetaData(IoTDBConnection connection) {
+ private static void showMetaData(IoTDBConnection connection) {
try {
println(((IoTDBDatabaseMetadata) connection.getMetaData()).getMetadataInJson());
} catch (SQLException e) {
@@ -674,7 +672,7 @@ public abstract class AbstractClient {
}
}
- protected static void setTimestampDisplay(String specialCmd, String cmd) {
+ private static void setTimestampDisplay(String specialCmd, String cmd) {
String[] values = specialCmd.split("=");
if (values.length != 2) {
println(String.format("Time display format error, please input like %s=ISO8601",
@@ -691,7 +689,7 @@ public abstract class AbstractClient {
println("Time display type has set to " + cmd.split("=")[1].trim());
}
- protected static void setTimeZone(String specialCmd, String cmd, IoTDBConnection connection) {
+ private static void setTimeZone(String specialCmd, String cmd, IoTDBConnection connection) {
String[] values = specialCmd.split("=");
if (values.length != 2) {
println(
@@ -708,7 +706,7 @@ public abstract class AbstractClient {
println("Time zone has set to " + values[1].trim());
}
- protected static void setFetchSize(String specialCmd, String cmd) {
+ private static void setFetchSize(String specialCmd, String cmd) {
String[] values = specialCmd.split("=");
if (values.length != 2) {
println(String
@@ -725,7 +723,7 @@ public abstract class AbstractClient {
println("Fetch size has set to " + values[1].trim());
}
- protected static void setMaxDisplaNum(String specialCmd, String cmd) {
+ private static void setMaxDisplaNum(String specialCmd, String cmd) {
String[] values = specialCmd.split("=");
if (values.length != 2) {
println(String.format("Max display number format error, please input like %s = 10000",
@@ -742,7 +740,7 @@ public abstract class AbstractClient {
println("Max display number has set to " + values[1].trim());
}
- protected static void showTimeZone(IoTDBConnection connection) {
+ private static void showTimeZone(IoTDBConnection connection) {
try {
println("Current time zone: " + connection.getTimeZone());
} catch (Exception e) {
@@ -751,7 +749,7 @@ public abstract class AbstractClient {
}
}
- protected static void importCmd(String specialCmd, String cmd, IoTDBConnection connection) {
+ private static void importCmd(String specialCmd, String cmd, IoTDBConnection connection) {
String[] values = specialCmd.split(" ");
if (values.length != 2) {
println("Please input like: import /User/myfile. "
@@ -772,12 +770,10 @@ public abstract class AbstractClient {
}
}
- protected static void executeQuery(IoTDBConnection connection, String cmd) {
- Statement statement = null;
+ private static void executeQuery(IoTDBConnection connection, String cmd) {
long startTime = System.currentTimeMillis();
- try {
+ try (Statement statement = connection.createStatement();) {
ZoneId zoneId = ZoneId.of(connection.getTimeZone());
- statement = connection.createStatement();
statement.setFetchSize(fetchSize);
boolean hasResultSet = statement.execute(cmd.trim());
if (hasResultSet) {
@@ -790,15 +786,6 @@ public abstract class AbstractClient {
} catch (Exception e) {
println("Msg: " + e.getMessage());
handleException(e);
- } finally {
- if (statement != null) {
- try {
- statement.close();
- } catch (SQLException e) {
- println("Cannot close statement because: " + e.getMessage());
- handleException(e);
- }
- }
}
long costTime = System.currentTimeMillis() - startTime;
println(String.format("It costs %.3fs", costTime / 1000.0));
@@ -808,27 +795,27 @@ public abstract class AbstractClient {
STOP_OPER, CONTINUE_OPER, NO_OPER
}
- protected static void printf(String format, Object... args) {
+ private static void printf(String format, Object... args) {
SCREEN_PRINTER.printf(format, args);
}
- protected static void print(String msg) {
+ static void print(String msg) {
SCREEN_PRINTER.print(msg);
}
- protected static void println() {
+ private static void println() {
SCREEN_PRINTER.println();
}
- protected static void println(String msg) {
+ static void println(String msg) {
SCREEN_PRINTER.println(msg);
}
- protected static void println(Object obj) {
+ private static void println(Object obj) {
SCREEN_PRINTER.println(obj);
}
- protected static void handleException(Exception e) {
+ static void handleException(Exception e) {
if (showException) {
e.printStackTrace(SCREEN_PRINTER);
}
diff --git a/client/src/main/java/org/apache/iotdb/cli/client/Client.java b/client/src/main/java/org/apache/iotdb/client/Client.java
similarity index 97%
rename from client/src/main/java/org/apache/iotdb/cli/client/Client.java
rename to client/src/main/java/org/apache/iotdb/client/Client.java
index 8e865032e223ff3e9695b50560712072540c9c61..9678abcc3b2a33d5017517d6ed9e38f04359852a 100644
--- a/client/src/main/java/org/apache/iotdb/cli/client/Client.java
+++ b/client/src/main/java/org/apache/iotdb/client/Client.java
@@ -16,12 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.client;
+package org.apache.iotdb.client;
import java.io.IOException;
import java.sql.DriverManager;
import java.sql.SQLException;
-import java.util.Arrays;
import jline.console.ConsoleReader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -29,17 +28,20 @@ import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.iotdb.cli.exception.ArgsErrorException;
+import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
+/**
+ * usage: -h 127.0.0.1 -p 6667 -u root -pw root
+ */
public class Client extends AbstractClient {
private static CommandLine commandLine;
/**
- * IoTDB CLI main function.
+ * IoTDB Client main function.
*
* @param args launch arguments
* @throws ClassNotFoundException ClassNotFoundException
diff --git a/client/src/main/java/org/apache/iotdb/cli/client/WinClient.java b/client/src/main/java/org/apache/iotdb/client/WinClient.java
similarity index 97%
rename from client/src/main/java/org/apache/iotdb/cli/client/WinClient.java
rename to client/src/main/java/org/apache/iotdb/client/WinClient.java
index b66f6586187c16d8888e3768755faa07d223a666..a174e10379eb7805d00cfa2ce3d0679cdf359b18 100644
--- a/client/src/main/java/org/apache/iotdb/cli/client/WinClient.java
+++ b/client/src/main/java/org/apache/iotdb/client/WinClient.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.client;
+package org.apache.iotdb.client;
import java.io.Console;
import java.sql.DriverManager;
@@ -28,7 +28,7 @@ import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.iotdb.cli.exception.ArgsErrorException;
+import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
@@ -71,7 +71,7 @@ public class WinClient extends AbstractClient {
private static String readPassword() {
Console c = System.console();
- if (c == null) { // IN ECLIPSE IDE
+ if (c == null) { // IN ECLIENTPSE IDE
print(IOTDB_CLI_PREFIX + "> please input password: ");
Scanner scanner = new Scanner(System.in);
return scanner.nextLine();
diff --git a/client/src/main/java/org/apache/iotdb/cli/exception/ArgsErrorException.java b/client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java
similarity index 96%
rename from client/src/main/java/org/apache/iotdb/cli/exception/ArgsErrorException.java
rename to client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java
index 66f35f1e2b91fd7972e6c3acdf4e14e56fda922b..f3c7c2b383d7da6b322eb8fe59c8ce2359045681 100644
--- a/client/src/main/java/org/apache/iotdb/cli/exception/ArgsErrorException.java
+++ b/client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.exception;
+package org.apache.iotdb.exception;
public class ArgsErrorException extends Exception {
diff --git a/client/src/main/java/org/apache/iotdb/cli/tool/AbstractCsvTool.java b/client/src/main/java/org/apache/iotdb/tool/AbstractCsvTool.java
similarity index 98%
rename from client/src/main/java/org/apache/iotdb/cli/tool/AbstractCsvTool.java
rename to client/src/main/java/org/apache/iotdb/tool/AbstractCsvTool.java
index 5045f7ad212fd471db54419bde04c2eebead3468..bd625b866a53f77f2c37dd337e6843631622a0fa 100644
--- a/client/src/main/java/org/apache/iotdb/cli/tool/AbstractCsvTool.java
+++ b/client/src/main/java/org/apache/iotdb/tool/AbstractCsvTool.java
@@ -16,13 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.tool;
+package org.apache.iotdb.tool;
import java.io.IOException;
import java.time.ZoneId;
import jline.console.ConsoleReader;
import org.apache.commons.cli.CommandLine;
-import org.apache.iotdb.cli.exception.ArgsErrorException;
+import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.iotdb.jdbc.IoTDBSQLException;
import org.apache.thrift.TException;
diff --git a/client/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java b/client/src/main/java/org/apache/iotdb/tool/ExportCsv.java
similarity index 98%
rename from client/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
rename to client/src/main/java/org/apache/iotdb/tool/ExportCsv.java
index fd358d92c26b1f7795ae9bb8f019113511bb3f27..702d4b210f36dcd0595ee3b00891d7d69165c2be 100644
--- a/client/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
+++ b/client/src/main/java/org/apache/iotdb/tool/ExportCsv.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.cli.tool;
+package org.apache.iotdb.tool;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -41,8 +41,8 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.iotdb.cli.client.AbstractClient;
-import org.apache.iotdb.cli.exception.ArgsErrorException;
+import org.apache.iotdb.client.AbstractClient;
+import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
diff --git a/client/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java b/client/src/main/java/org/apache/iotdb/tool/ImportCsv.java
similarity index 99%
rename from client/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
rename to client/src/main/java/org/apache/iotdb/tool/ImportCsv.java
index 498af0c2893c17082d54d162985b35e0a2854a1c..81669505243556d8915f57b3a96d3b1b17f12850 100644
--- a/client/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
+++ b/client/src/main/java/org/apache/iotdb/tool/ImportCsv.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.tool;
+package org.apache.iotdb.tool;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -46,13 +46,11 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
-import org.apache.iotdb.cli.exception.ArgsErrorException;
+import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.Constant;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* read a CSV formatted data File and insert all the data into IoTDB.
diff --git a/client/src/test/java/org/apache/iotdb/cli/client/AbstractClientIT.java b/client/src/test/java/org/apache/iotdb/client/AbstractClientIT.java
similarity index 74%
rename from client/src/test/java/org/apache/iotdb/cli/client/AbstractClientIT.java
rename to client/src/test/java/org/apache/iotdb/client/AbstractClientIT.java
index 5f302b2c3c216d7a3e3ecdc98573c2fbba53c914..e9c6f15f9f29fac1d80480e8341b0d86a9630861 100644
--- a/client/src/test/java/org/apache/iotdb/cli/client/AbstractClientIT.java
+++ b/client/src/test/java/org/apache/iotdb/client/AbstractClientIT.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.client;
+package org.apache.iotdb.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -27,8 +27,8 @@ import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.iotdb.cli.client.AbstractClient.OperationResult;
-import org.apache.iotdb.cli.exception.ArgsErrorException;
+import org.apache.iotdb.client.AbstractClient.OperationResult;
+import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.iotdb.jdbc.IoTDBDatabaseMetadata;
import org.junit.After;
@@ -143,30 +143,46 @@ public class AbstractClientIT {
@Test
public void testHandleInputInputCmd() {
- assertEquals(OperationResult.STOP_OPER, AbstractClient.handleInputCmd(AbstractClient.EXIT_COMMAND, connection));
- assertEquals(OperationResult.STOP_OPER, AbstractClient.handleInputCmd(AbstractClient.QUIT_COMMAND, connection));
-
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(AbstractClient.SHOW_METADATA_COMMAND, connection));
-
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=", AbstractClient.SET_TIMESTAMP_DISPLAY), connection));
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=xxx", AbstractClient.SET_TIMESTAMP_DISPLAY), connection));
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=default", AbstractClient.SET_TIMESTAMP_DISPLAY), connection));
+ assertEquals(OperationResult.STOP_OPER, AbstractClient
+ .handleInputCmd(AbstractClient.EXIT_COMMAND, connection));
+ assertEquals(OperationResult.STOP_OPER, AbstractClient
+ .handleInputCmd(AbstractClient.QUIT_COMMAND, connection));
+
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(AbstractClient.SHOW_METADATA_COMMAND, connection));
+
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=", AbstractClient.SET_TIMESTAMP_DISPLAY), connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=xxx", AbstractClient.SET_TIMESTAMP_DISPLAY), connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=default", AbstractClient.SET_TIMESTAMP_DISPLAY), connection));
testSetTimeFormat();
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=", AbstractClient.SET_MAX_DISPLAY_NUM), connection));
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=xxx", AbstractClient.SET_MAX_DISPLAY_NUM),connection));
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=1", AbstractClient.SET_MAX_DISPLAY_NUM), connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=", AbstractClient.SET_MAX_DISPLAY_NUM), connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=xxx", AbstractClient.SET_MAX_DISPLAY_NUM),connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=1", AbstractClient.SET_MAX_DISPLAY_NUM), connection));
testSetMaxDisplayNumber();
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(AbstractClient.SHOW_TIMEZONE, connection));
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(AbstractClient.SHOW_TIMESTAMP_DISPLAY, connection));
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(AbstractClient.SHOW_FETCH_SIZE, connection));
-
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=", AbstractClient.SET_TIME_ZONE), connection));
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=+08:00", AbstractClient.SET_TIME_ZONE), connection));
-
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=", AbstractClient.SET_FETCH_SIZE), connection));
- assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=111", AbstractClient.SET_FETCH_SIZE), connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(AbstractClient.SHOW_TIMEZONE, connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(AbstractClient.SHOW_TIMESTAMP_DISPLAY, connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(AbstractClient.SHOW_FETCH_SIZE, connection));
+
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=", AbstractClient.SET_TIME_ZONE), connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=+08:00", AbstractClient.SET_TIME_ZONE), connection));
+
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=", AbstractClient.SET_FETCH_SIZE), connection));
+ assertEquals(OperationResult.CONTINUE_OPER, AbstractClient
+ .handleInputCmd(String.format("%s=111", AbstractClient.SET_FETCH_SIZE), connection));
}
private void testSetTimeFormat() {
diff --git a/client/src/test/java/org/apache/iotdb/cli/client/AbstractScript.java b/client/src/test/java/org/apache/iotdb/client/AbstractScript.java
similarity index 98%
rename from client/src/test/java/org/apache/iotdb/cli/client/AbstractScript.java
rename to client/src/test/java/org/apache/iotdb/client/AbstractScript.java
index 4cd1a403b5336f54d66289c802d22b244f900b90..79c5daf19e819c80a6a95d134a807ede86d25817 100644
--- a/client/src/test/java/org/apache/iotdb/cli/client/AbstractScript.java
+++ b/client/src/test/java/org/apache/iotdb/client/AbstractScript.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.client;
+package org.apache.iotdb.client;
import static org.junit.Assert.assertEquals;
diff --git a/client/src/test/java/org/apache/iotdb/cli/client/StartClientScriptIT.java b/client/src/test/java/org/apache/iotdb/client/StartClientScriptIT.java
similarity index 98%
rename from client/src/test/java/org/apache/iotdb/cli/client/StartClientScriptIT.java
rename to client/src/test/java/org/apache/iotdb/client/StartClientScriptIT.java
index 85fdcd827e2e65bb4e26b48a719bb93f4b9ffd07..28a291701ebf66a8a2f3f0a3b23f8bc6defea4e0 100644
--- a/client/src/test/java/org/apache/iotdb/cli/client/StartClientScriptIT.java
+++ b/client/src/test/java/org/apache/iotdb/client/StartClientScriptIT.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.client;
+package org.apache.iotdb.client;
import java.io.File;
import java.io.IOException;
diff --git a/client/src/test/java/org/apache/iotdb/cli/tool/ExportCsvTestIT.java b/client/src/test/java/org/apache/iotdb/tool/ExportCsvTestIT.java
similarity index 96%
rename from client/src/test/java/org/apache/iotdb/cli/tool/ExportCsvTestIT.java
rename to client/src/test/java/org/apache/iotdb/tool/ExportCsvTestIT.java
index de30750a22152e480a337de3618e09d7cd73a4fa..7d0fd202f759795ae2715525e9b84cc698992203 100644
--- a/client/src/test/java/org/apache/iotdb/cli/tool/ExportCsvTestIT.java
+++ b/client/src/test/java/org/apache/iotdb/tool/ExportCsvTestIT.java
@@ -16,11 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.tool;
+package org.apache.iotdb.tool;
import java.io.File;
import java.io.IOException;
-import org.apache.iotdb.cli.client.AbstractScript;
+import org.apache.iotdb.client.AbstractScript;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
diff --git a/client/src/test/java/org/apache/iotdb/cli/tool/ImportCsvTestIT.java b/client/src/test/java/org/apache/iotdb/tool/ImportCsvTestIT.java
similarity index 96%
rename from client/src/test/java/org/apache/iotdb/cli/tool/ImportCsvTestIT.java
rename to client/src/test/java/org/apache/iotdb/tool/ImportCsvTestIT.java
index 8bbe91bf56ad265177568ef7ae140dc7a23d24d2..95863b492710c3e3b110eb72687a43811f731a0b 100644
--- a/client/src/test/java/org/apache/iotdb/cli/tool/ImportCsvTestIT.java
+++ b/client/src/test/java/org/apache/iotdb/tool/ImportCsvTestIT.java
@@ -16,11 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.cli.tool;
+package org.apache.iotdb.tool;
import java.io.File;
import java.io.IOException;
-import org.apache.iotdb.cli.client.AbstractScript;
+import org.apache.iotdb.client.AbstractScript;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
diff --git a/docs/Development.md b/docs/Development.md
index ee00bef054a4c1e3b7c132a002ff0a30e2b18708..d3d026d51e3c69124534fc923ccd867159b141f0 100644
--- a/docs/Development.md
+++ b/docs/Development.md
@@ -74,8 +74,8 @@ you can just choose menu "import" -> "Maven" -> "Existing Maven Projects".
## Debugging IoTDB
The main class of IoTDB server is `org.apache.iotdb.db.service.IoTDB`.
-The main class of IoTDB client is `org.apache.iotdb.cli.client.Client`
-(or `org.apache.iotdb.cli.client.WinClient` on Win OS).
+The main class of IoTDB cli is `org.apache.iotdb.client.Client`
+(or `org.apache.iotdb.client.WinClient` on Win OS).
You can run/debug IoTDB by using the two classes as the entrance.
diff --git a/docs/Documentation-CHN/QuickStart.md b/docs/Documentation-CHN/QuickStart.md
index 9fa9a16462a7582b7e060a4a6568923676456bc2..4d189d03d100f7583015e963d17e647eb3a6fe40 100755
--- a/docs/Documentation-CHN/QuickStart.md
+++ b/docs/Documentation-CHN/QuickStart.md
@@ -88,12 +88,12 @@ IoTDB为用户提供多种与服务器交互的方式,您可以选择使用Cli
Linux系统与MacOS系统启动命令如下:
```
-> $IOTDB_HOME/sbin/start-client.sh -h 127.0.0.1 -p 6667 -u root
+> $IOTDB_HOME/sbin/start-cli.sh -h 127.0.0.1 -p 6667 -u root
```
Windows系统启动命令如下:
```
-> $IOTDB_HOME\sbin\start-client.bat -h 127.0.0.1 -p 6667 -u root
+> $IOTDB_HOME\sbin\start-cli.bat -h 127.0.0.1 -p 6667 -u root
```
回车后输入root用户的密码,即可成功启动客户端。启动后出现如图提示即为启动成功。
@@ -286,7 +286,7 @@ IoTDB> SELECT * FROM root.ln.wf01.wt01
+-----------------------+--------------------------+-----------------------------+
```
-输入quit或exit可退出Client结束本次会话,Client输出quit normally表示退出成功,操作语句与返回结果如下:
+输入quit或exit可退出Cli结束本次会话,Cli输出quit normally表示退出成功,操作语句与返回结果如下:
```
IoTDB> quit
quit normally
diff --git a/docs/Documentation-CHN/UserGuide/1-Overview/3-Scenario.md b/docs/Documentation-CHN/UserGuide/1-Overview/3-Scenario.md
index 95176b33a840045c1f02d2088842497e0038bce8..79fe86e82dbaab7b690662a85a4148aa5457d113 100644
--- a/docs/Documentation-CHN/UserGuide/1-Overview/3-Scenario.md
+++ b/docs/Documentation-CHN/UserGuide/1-Overview/3-Scenario.md
@@ -57,7 +57,7 @@
为了保证机械手的监控数据能够及时监控和分析,公司需要收集这些机械手传感器信息,将其发送至可以连接外部网络的服务器上,而后将原始数据信息上传到数据中心进行复杂的计算和分析。
-此时,可以采用IoTDB套件中的IoTDB、IoTDB-CLI工具、TsFileSync工具和Hadoop/Spark集成组件等。将IoTDB服务器安装在工厂连接外网的服务器上,用户接收机械手传输的数据并将数据上传到数据中心。将IoTDB-CLI工具安装在每一个连接工厂内网的机械手上,用于将传感器产生的实时数据上传到工厂内部服务器。再使用TsFileSync工具将原始数据上传到数据中心。此外还需要部署Hadoop/Spark集群用于数据中心端的数据存储和分析。如图1.6中间场景所示。
+此时,可以采用IoTDB套件中的IoTDB、IoTDB-Client工具、TsFileSync工具和Hadoop/Spark集成组件等。将IoTDB服务器安装在工厂连接外网的服务器上,用户接收机械手传输的数据并将数据上传到数据中心。将IoTDB-Client工具安装在每一个连接工厂内网的机械手上,用于将传感器产生的实时数据上传到工厂内部服务器。再使用TsFileSync工具将原始数据上传到数据中心。此外还需要部署Hadoop/Spark集群用于数据中心端的数据存储和分析。如图1.6中间场景所示。
@@ -71,7 +71,7 @@
为了能够实时接收汽车传感器所采集的物联网数据,公司需要在车辆行驶的过程中将传感器数据通过窄带物联网实时发送至数据中心,而后在数据中心的服务器上进行复杂的计算和分析。
-此时,可以采用IoTDB套件中的IoTDB、IoTDB-CLI和Hadoop/Spark集成组件等。将IoTDB-CLI工具安装在每一辆车联网内的车辆上,使用IoTDB-JDBC工具将数据直接传回数据中心的服务器。
+此时,可以采用IoTDB套件中的IoTDB、IoTDB-Client和Hadoop/Spark集成组件等。将IoTDB-Client工具安装在每一辆车联网内的车辆上,使用IoTDB-JDBC工具将数据直接传回数据中心的服务器。
此外还需要部署Hadoop/Spark集群用于数据中心端的数据存储和分析。如图1.8所示。
diff --git a/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md b/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md
new file mode 100644
index 0000000000000000000000000000000000000000..15f77d90e1a69c6875a2da796db013be8402e69f
--- /dev/null
+++ b/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md
@@ -0,0 +1,64 @@
+
+
+# Chapter6: JDBC API
+
+## Status Code
+
+在最新版本中引入了**状态码**这一概念。例如,因为IoTDB需要在写入数据之前首先注册时间序列,一种可能的解决方案是:
+
+```
+try {
+ writeData();
+} catch (SQLException e) {
+ // the most case is that the time series does not exist
+ if (e.getMessage().contains("exist")) {
+ //However, using the content of the error message is not so efficient
+ registerTimeSeries();
+ //write data once again
+ writeData();
+ }
+}
+
+```
+
+利用状态码,我们就可以不必写诸如`if (e.getErrorMessage().contains("exist"))`的代码,只需要使用`e.getStatusType().getCode() == TSStatusType.TIME_SERIES_NOT_EXIST_ERROR.getStatusCode()`。
+
+这里是状态码和相对应信息的列表:
+
+|状态码|状态类型|状态信息|
+|:---|:---|:---|
+|200|SUCCESS_STATUS||
+|201|STILL_EXECUTING_STATUS||
+|202|INVALID_HANDLE_STATUS||
+|301|TIMESERIES_NOT_EXIST_ERROR|时间序列不存在|
+|302|UNSUPPORTED_FETCH_METADATA_OPERATION_ERROR|不支持的获取元数据操作|
+|303|FETCH_METADATA_ERROR|获取元数据失败|
+|304|CHECK_FILE_LEVEL_ERROR|检查文件层级错误|
+|400|EXECUTE_STATEMENT_ERROR|执行语句错误|
+|401|SQL_PARSE_ERROR|SQL语句分析错误|
+|402|GENERATE_TIME_ZONE_ERROR|生成时区错误|
+|403|SET_TIME_ZONE_ERROR|设置时区错误|
+|500|INTERNAL_SERVER_ERROR|服务器内部错误|
+|600|WRONG_LOGIN_PASSWORD_ERROR|用户名或密码错误|
+|601|NOT_LOGIN_ERROR|没有登录|
+|602|NO_PERMISSION_ERROR|没有操作权限|
+|603|UNINITIALIZED_AUTH_ERROR|授权人未初始化|
diff --git a/docs/Documentation-CHN/UserGuide/7-Session API/1-Session API.md b/docs/Documentation-CHN/UserGuide/7-Session API/1-Session API.md
new file mode 100644
index 0000000000000000000000000000000000000000..9b21474072f7ca30c7a77511a8de3b52ad44a305
--- /dev/null
+++ b/docs/Documentation-CHN/UserGuide/7-Session API/1-Session API.md
@@ -0,0 +1,105 @@
+
+
+# 第7章: Session API
+
+# 使用方式
+
+## 依赖
+
+* JDK >= 1.8
+* Maven >= 3.0
+
+## 安装到本地 maven 库
+
+In root directory:
+> mvn clean install -pl session -am -Dmaven.test.skip=true
+
+## 在 maven 中使用 session 接口
+
+```
+
+
+ org.apache.iotdb
+ iotdb-session
+ 0.9.0-SNAPSHOT
+
+
+```
+
+## Session 接口使用示例
+
+
+```Java
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.write.record.RowBatch;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
+
+public class SessionExample {
+
+ public static void main(String[] args) throws ClassNotFoundException, IoTDBSessionException {
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open();
+
+ session.setStorageGroup("root.sg1");
+ session.createTimeseriesResp("root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE);
+ session.createTimeseriesResp("root.sg1.d1.s2", TSDataType.INT64, TSEncoding.RLE);
+ session.createTimeseriesResp("root.sg1.d1.s3", TSDataType.INT64, TSEncoding.RLE);
+
+ Schema schema = new Schema();
+ schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
+ schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
+ schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
+
+ RowBatch rowBatch = schema.createRowBatch("root.sg1.d1", 100);
+
+ long[] timestamps = rowBatch.timestamps;
+ Object[] values = rowBatch.values;
+
+ for (long time = 0; time < 30000; time++) {
+ int row = rowBatch.batchSize++;
+ timestamps[row] = time;
+ for (int i = 0; i < 3; i++) {
+ long[] sensor = (long[]) values[i];
+ sensor[row] = time;
+ }
+ if (rowBatch.batchSize == rowBatch.getMaxBatchSize()) {
+ session.insertBatch(rowBatch);
+ rowBatch.reset();
+ }
+ }
+
+ if (rowBatch.batchSize != 0) {
+ session.insertBatch(rowBatch);
+ rowBatch.reset();
+ }
+ session.close();
+ }
+}
+```
+
+> The code is in example/session/src/main/java/org/apache/iotdb/session/SessionExample.java
\ No newline at end of file
diff --git a/docs/Documentation-CHN/UserGuide/7-TsFile/1-Installation.md b/docs/Documentation-CHN/UserGuide/8-TsFile/1-Installation.md
similarity index 96%
rename from docs/Documentation-CHN/UserGuide/7-TsFile/1-Installation.md
rename to docs/Documentation-CHN/UserGuide/8-TsFile/1-Installation.md
index 1813bb63df807f0582d127bcc619ac5128092cdb..77409d34ddf6562e9ebc55299f4542645608a649 100644
--- a/docs/Documentation-CHN/UserGuide/7-TsFile/1-Installation.md
+++ b/docs/Documentation-CHN/UserGuide/8-TsFile/1-Installation.md
@@ -19,6 +19,6 @@
-->
-# Chaper 7: TsFile
+# 第8章: TsFile
Coming Soon.
\ No newline at end of file
diff --git a/docs/Documentation-CHN/UserGuide/7-TsFile/2-Usage.md b/docs/Documentation-CHN/UserGuide/8-TsFile/2-Usage.md
similarity index 96%
rename from docs/Documentation-CHN/UserGuide/7-TsFile/2-Usage.md
rename to docs/Documentation-CHN/UserGuide/8-TsFile/2-Usage.md
index 1813bb63df807f0582d127bcc619ac5128092cdb..77409d34ddf6562e9ebc55299f4542645608a649 100644
--- a/docs/Documentation-CHN/UserGuide/7-TsFile/2-Usage.md
+++ b/docs/Documentation-CHN/UserGuide/8-TsFile/2-Usage.md
@@ -19,6 +19,6 @@
-->
-# Chaper 7: TsFile
+# 第8章: TsFile
Coming Soon.
\ No newline at end of file
diff --git a/docs/Documentation-CHN/UserGuide/7-TsFile/3-Hierarchy.md b/docs/Documentation-CHN/UserGuide/8-TsFile/3-Hierarchy.md
similarity index 96%
rename from docs/Documentation-CHN/UserGuide/7-TsFile/3-Hierarchy.md
rename to docs/Documentation-CHN/UserGuide/8-TsFile/3-Hierarchy.md
index 1813bb63df807f0582d127bcc619ac5128092cdb..77409d34ddf6562e9ebc55299f4542645608a649 100644
--- a/docs/Documentation-CHN/UserGuide/7-TsFile/3-Hierarchy.md
+++ b/docs/Documentation-CHN/UserGuide/8-TsFile/3-Hierarchy.md
@@ -19,6 +19,6 @@
-->
-# Chaper 7: TsFile
+# 第8章: TsFile
Coming Soon.
\ No newline at end of file
diff --git a/docs/Documentation-CHN/UserGuide/8-System Tools/1-Sync.md b/docs/Documentation-CHN/UserGuide/9-System Tools/1-Sync.md
similarity index 99%
rename from docs/Documentation-CHN/UserGuide/8-System Tools/1-Sync.md
rename to docs/Documentation-CHN/UserGuide/9-System Tools/1-Sync.md
index dbe5eb9a0800f3f26bc8bf62cf854a700870cf2a..cddf0c173911d91aca39e9126384c2872ee97915 100644
--- a/docs/Documentation-CHN/UserGuide/8-System Tools/1-Sync.md
+++ b/docs/Documentation-CHN/UserGuide/9-System Tools/1-Sync.md
@@ -19,7 +19,7 @@
-->
-# 同步工具
+# 第9章: 同步工具
- [同步工具](#同步工具)
diff --git a/docs/Documentation-CHN/UserGuide/8-System Tools/2-Memory Estimation Tool.md b/docs/Documentation-CHN/UserGuide/9-System Tools/2-Memory Estimation Tool.md
similarity index 99%
rename from docs/Documentation-CHN/UserGuide/8-System Tools/2-Memory Estimation Tool.md
rename to docs/Documentation-CHN/UserGuide/9-System Tools/2-Memory Estimation Tool.md
index 60526702424cd44bba68f74ad52fc3258523628f..692d30bbdf411d581c2044914ad1ac9399d7bc5c 100644
--- a/docs/Documentation-CHN/UserGuide/8-System Tools/2-Memory Estimation Tool.md
+++ b/docs/Documentation-CHN/UserGuide/9-System Tools/2-Memory Estimation Tool.md
@@ -19,7 +19,7 @@
-->
-# 内存预估工具
+# 第9章: 内存预估工具
diff --git a/docs/Documentation-CHN/UserGuide/8-Tools-Cli.md b/docs/Documentation-CHN/UserGuide/9-Tools-Cli.md
similarity index 76%
rename from docs/Documentation-CHN/UserGuide/8-Tools-Cli.md
rename to docs/Documentation-CHN/UserGuide/9-Tools-Cli.md
index 0b00442a3a5029674eb9a0fe40ad61a53b03bb48..f4e0c1e309b18eb4abb82517059688bf24e4f0f5 100644
--- a/docs/Documentation-CHN/UserGuide/8-Tools-Cli.md
+++ b/docs/Documentation-CHN/UserGuide/9-Tools-Cli.md
@@ -29,7 +29,7 @@
# Cli / Shell工具
-IOTDB为用户提供CLI/Shell工具用于启动客户端和服务端程序。下面介绍每个CLI/Shell工具的运行方式和相关参数。
+IOTDB为用户提供Client/Shell工具用于启动客户端和服务端程序。下面介绍每个Client/Shell工具的运行方式和相关参数。
> \$IOTDB\_HOME表示IoTDB的安装目录所在路径。
## Cli / Shell运行方式
@@ -38,12 +38,12 @@ IOTDB为用户提供CLI/Shell工具用于启动客户端和服务端程序。下
Linux系统与MacOS系统启动命令如下:
```
- Shell > ./sbin/start-client.sh -h 127.0.0.1 -p 6667 -u root -pw root
+ Shell > ./sbin/start-cli.sh -h 127.0.0.1 -p 6667 -u root -pw root
```
Windows系统启动命令如下:
```
- Shell > \sbin\start-client.bat -h 127.0.0.1 -p 6667 -u root -pw root
+ Shell > \sbin\start-cli.bat -h 127.0.0.1 -p 6667 -u root -pw root
```
回车后即可成功启动客户端。启动后出现如图提示即为启动成功。
```
@@ -79,22 +79,22 @@ IoTDB>
Linux系统与MacOS系统启动命令如下:
```
- Shell >./sbin/start-client.sh -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10
+ Shell >./sbin/start-cli.sh -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10
```
Windows系统启动命令如下:
```
- Shell > \sbin\start-client.bat -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10
+ Shell > \sbin\start-cli.bat -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10
```
## Cli / Shell的-e参数
当您想要通过脚本的方式通过Cli / Shell对IoTDB进行批量操作时,可以使用-e参数。通过使用该参数,您可以在不进入客户端输入模式的情况下操作IoTDB。
为了避免SQL语句和其他参数混淆,现在只支持-e参数作为最后的参数使用。
-针对CLI/Shell工具的-e参数用法如下:
+针对Client/Shell工具的-e参数用法如下:
```
- Shell > ./sbin/start-client.sh -h {host} -p {port} -u {user} -pw {password} -e {sql for iotdb}
+ Shell > ./sbin/start-cli.sh -h {host} -p {port} -u {user} -pw {password} -e {sql for iotdb}
```
为了更好的解释-e参数的使用,可以参考下面的例子。
@@ -109,7 +109,7 @@ Windows系统启动命令如下:
4.查询验证数据是否插入成功
-那么通过使用CLI/Shell工具的-e参数,可以采用如下的脚本:
+那么通过使用Client/Shell工具的-e参数,可以采用如下的脚本:
```
# !/bin/bash
@@ -119,12 +119,12 @@ port=6667
user=root
pass=root
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "set storage group to root.demo"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "create timeseries root.demo.s1 WITH DATATYPE=INT32, ENCODING=RLE"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(1,10)"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(2,11)"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(3,12)"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "select s1 from root.demo"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "set storage group to root.demo"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "create timeseries root.demo.s1 WITH DATATYPE=INT32, ENCODING=RLE"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(1,10)"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(2,11)"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(3,12)"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "select s1 from root.demo"
```
打印出来的结果显示在下图,通过这种方式进行的操作与客户端的输入模式以及通过JDBC进行操作结果是一致的。
diff --git a/docs/Documentation-CHN/UserGuide/8-Tools-Grafana.md b/docs/Documentation-CHN/UserGuide/9-Tools-Grafana.md
similarity index 100%
rename from docs/Documentation-CHN/UserGuide/8-Tools-Grafana.md
rename to docs/Documentation-CHN/UserGuide/9-Tools-Grafana.md
diff --git a/docs/Documentation-CHN/UserGuide/8-Tools-Hadoop.md b/docs/Documentation-CHN/UserGuide/9-Tools-Hadoop.md
similarity index 100%
rename from docs/Documentation-CHN/UserGuide/8-Tools-Hadoop.md
rename to docs/Documentation-CHN/UserGuide/9-Tools-Hadoop.md
diff --git a/docs/Documentation-CHN/UserGuide/8-Tools-spark.md b/docs/Documentation-CHN/UserGuide/9-Tools-spark.md
similarity index 100%
rename from docs/Documentation-CHN/UserGuide/8-Tools-spark.md
rename to docs/Documentation-CHN/UserGuide/9-Tools-spark.md
diff --git a/docs/Documentation/Frequently asked questions.md b/docs/Documentation/Frequently asked questions.md
index b359c67ee4a7ba20c3306c220ca3ec3bdb2b9e29..096e4decfa1d86f3920926ad7eeb00b53acb4700 100644
--- a/docs/Documentation/Frequently asked questions.md
+++ b/docs/Documentation/Frequently asked questions.md
@@ -31,7 +31,7 @@
- Can I use Hadoop and Spark to read TsFile in IoTDB?
- How does IoTDB handle duplicate points?
- How can I tell what type of the specific timeseries?
- - How can I change IoTDB's CLI time display format?
+ - How can I change IoTDB's Client time display format?
# Frequently Asked Questions
@@ -43,19 +43,19 @@ There are several ways to identify the version of IoTDB that you are using:
* Launch IoTDB's Command Line Interface:
```
-> ./start-client.sh -p 6667 -pw root -u root -h localhost
+> ./start-cli.sh -p 6667 -pw root -u root -h localhost
_____ _________ ______ ______
|_ _| | _ _ ||_ _ `.|_ _ \
| | .--.|_/ | | \_| | | `. \ | |_) |
| | / .'`\ \ | | | | | | | __'.
_| |_| \__. | _| |_ _| |_.' /_| |__) |
-|_____|'.__.' |_____| |______.'|_______/ version 0.7.0
+|_____|'.__.' |_____| |______.'|_______/ version x.x.x
```
* Check pom.xml file:
```
-0.7.0
+0.9.0-SNAPSHOT
```
* Use JDBC API:
@@ -137,11 +137,11 @@ Otherwise, you can also use wildcard in timeseries path:
IoTDB> show timeseries root.fit.d1.*
```
-## How can I change IoTDB's CLI time display format?
+## How can I change IoTDB's Client time display format?
-The default IoTDB's CLI time display format is human readable (e.g. ```1970-01-01T08:00:00.001```), if you want to display time in timestamp type or other readable format, add parameter ```-disableIS08601``` in start command:
+The default IoTDB's Client time display format is human readable (e.g. ```1970-01-01T08:00:00.001```), if you want to display time in timestamp type or other readable format, add parameter ```-disableIS08601``` in start command:
```
-> $IOTDB_CLI_HOME/sbin/start-client.sh -h 127.0.0.1 -p 6667 -u root -pw root -disableIS08601
+> $IOTDB_CLI_HOME/sbin/start-cli.sh -h 127.0.0.1 -p 6667 -u root -pw root -disableIS08601
```
diff --git a/docs/Documentation/QuickStart.md b/docs/Documentation/QuickStart.md
index 3d5e579901a9b8fdf59d834246021a6a89a3242e..d36489a44fcb28559952638ddca8becac1145b54 100755
--- a/docs/Documentation/QuickStart.md
+++ b/docs/Documentation/QuickStart.md
@@ -30,7 +30,7 @@
- Configure
- Start
- Start Server
- - Start Client
+ - Start Cli
- Have a try
- Stop Server
@@ -89,8 +89,8 @@ then by default `IOTDB_HOME` is the direct parent directory of `sbin/start-serve
(or that of `sbin\start-server.bat` on Windows).
* if `IOTDB_CLI_HOME` is not explicitly assigned,
-then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-client.sh` on
-Unix/OS X (or that of `sbin\start-client.bat` on Windows).
+then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-cli.sh` on
+Unix/OS X (or that of `sbin\start-cli.bat` on Windows).
If you are not the first time that building IoTDB, remember deleting the following files:
@@ -170,9 +170,9 @@ After that we start the server. Running the startup script:
> $IOTDB_HOME\sbin\start-server.bat
```
-### Start Client
+### Start Cli
-Now let's trying to read and write some data from IoTDB using our Client. To start the client, you need to explicit the server's IP and PORT as well as the USER_NAME and PASSWORD.
+Now let's trying to read and write some data from IoTDB using our Cli. To start the client, you need to explicit the server's IP and PORT as well as the USER_NAME and PASSWORD.
```
# You can first build cli project
@@ -182,10 +182,10 @@ Now let's trying to read and write some data from IoTDB using our Client. To sta
> mvn clean package -pl client -am -Dmaven.test.skip=true
# Unix/OS X
-> $IOTDB_CLI_HOME/sbin/start-client.sh -h -p -u
+> $IOTDB_CLI_HOME/sbin/start-cli.sh -h -p -u
# Windows
-> $IOTDB_CLI_HOME\sbin\start-client.bat -h -p -u
+> $IOTDB_CLI_HOME\sbin\start-cli.bat -h -p -u
```
> NOTE: In the system, we set a default user in IoTDB named 'root'. The default password for 'root' is 'root'. You can use this default user if you are making the first try or you didn't create users by yourself.
diff --git a/docs/Documentation/UserGuide/0-Content.md b/docs/Documentation/UserGuide/0-Content.md
index 1b245d2135fb2a5c3c24b614304511e0a3f825c2..07aa1cf618e8196529a1a164410687fd411b6992 100644
--- a/docs/Documentation/UserGuide/0-Content.md
+++ b/docs/Documentation/UserGuide/0-Content.md
@@ -49,10 +49,13 @@
* 2-Reference
# Chapter 6: JDBC API
* 1-JDBC API
-# Chapter 7: TsFile
+* 2-Status Code
+# Chapter 7: Session API
+* 1-Session API
+# Chapter 8: TsFile
* 1-Installation
* 2-Usage
* 3-Hierarchy
-# Chapter 8: System Tools
+# Chapter 9: System Tools
* 1-Sync
* 2-Memory Estimation Tool
diff --git a/docs/Documentation/UserGuide/1-Overview/3-Scenario.md b/docs/Documentation/UserGuide/1-Overview/3-Scenario.md
index 879c5cbbceb9d2e952276c20c0fd2abbae464305..bac5ec91b8007eb47f9c0e9d2229c28fb75e12a6 100644
--- a/docs/Documentation/UserGuide/1-Overview/3-Scenario.md
+++ b/docs/Documentation/UserGuide/1-Overview/3-Scenario.md
@@ -61,7 +61,7 @@ In order to ensure that the data of the robot can be monitored and analyzed in t
-At this point, IoTDB, IoTDB-CLI tools, TsFileSync tools, and Hadoop/Spark integration components in the IoTDB suite can be used. IoTDB-CLI tool is installed on the robot and each of them is connected to the LAN of the factory. When sensors generate real-time data, the data will be uploaded to the server in the factory. The IoTDB server and TsFileSync is installed on the server connected to the external network. Once triggered, the data on the server will be upload to the data center. In addition, Hadoop/Spark clusters need to be deployed for data storage and analysis on the data center side. As shown in Figure 1.6. Figure 1.7 shows the architecture at this time.
+At this point, IoTDB, IoTDB-Client tools, TsFileSync tools, and Hadoop/Spark integration components in the IoTDB suite can be used. IoTDB-Client tool is installed on the robot and each of them is connected to the LAN of the factory. When sensors generate real-time data, the data will be uploaded to the server in the factory. The IoTDB server and TsFileSync is installed on the server connected to the external network. Once triggered, the data on the server will be upload to the data center. In addition, Hadoop/Spark clusters need to be deployed for data storage and analysis on the data center side. As shown in Figure 1.6. Figure 1.7 shows the architecture at this time.
@@ -71,7 +71,7 @@ A car company installed sensors on its cars to collect monitoring information su
In order to receive the IoT data collected by the car sensor in real time, the company needs to send the sensor data to the data center in real time through the narrowband IoT while the vehicle is running. Thus, they can perform complex calculations and analysis on the server in the data center.
-At this point, IoTDB, IoTDB-CLI, and Hadoop/Spark integration components in the IoTDB suite can be used. IoTDB-CLI tool is installed on each car and use IoTDB-JDBC tool to send data directly back to the server in the data center.
+At this point, IoTDB, IoTDB-Client, and Hadoop/Spark integration components in the IoTDB suite can be used. IoTDB-Client tool is installed on each car and use IoTDB-JDBC tool to send data directly back to the server in the data center.
In addition, Hadoop/Spark clusters need to be deployed for data storage and analysis on the data center side. As shown in Figure 1.8.
diff --git a/docs/Documentation/UserGuide/3-Operation Manual/3-Data Import.md b/docs/Documentation/UserGuide/3-Operation Manual/3-Data Import.md
index 78740162affcb4d8fc034a2d90473c89073518d0..542e8ed4b08170e8595cb8097c1622387107e092 100644
--- a/docs/Documentation/UserGuide/3-Operation Manual/3-Data Import.md
+++ b/docs/Documentation/UserGuide/3-Operation Manual/3-Data Import.md
@@ -28,7 +28,7 @@ This feature is not supported in version 0.7.0.
### Import Real-time Data
-IoTDB provides users with a variety of ways to insert real-time data, such as directly inputting [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1) in [Cli/Shell tools](/#/Tools/Cli), or using [Java JDBC](/#/Documents/0.8.0/chap6/sec1) to perform single or batch execution of [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1).
+IoTDB provides users with a variety of ways to insert real-time data, such as directly inputting [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1) in [Client/Shell tools](/#/Tools/Client), or using [Java JDBC](/#/Documents/0.8.0/chap6/sec1) to perform single or batch execution of [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1).
This section mainly introduces the use of [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1) for real-time data import in the scenario. See Section 5.1 for a detailed syntax of [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1).
diff --git a/docs/Documentation/UserGuide/4-Deployment and Management/1-Deployment.md b/docs/Documentation/UserGuide/4-Deployment and Management/1-Deployment.md
index 13960415f8388577fd23f594839394cc2bdc3ef8..5d6ebf7f8e64d290f1e6b2e9f801ccaa45cb9a09 100644
--- a/docs/Documentation/UserGuide/4-Deployment and Management/1-Deployment.md
+++ b/docs/Documentation/UserGuide/4-Deployment and Management/1-Deployment.md
@@ -104,8 +104,8 @@ then by default `IOTDB_HOME` is the direct parent directory of `sbin/start-serve
(or that of `sbin\start-server.bat` on Windows).
* if `IOTDB_CLI_HOME` is not explicitly assigned,
-then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-client.sh` on
-Unix/OS X (or that of `sbin\start-client.bat` on Windows).
+then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-cli.sh` on
+Unix/OS X (or that of `sbin\start-cli.bat` on Windows).
If you are not the first time that building IoTDB, remember deleting the following files:
diff --git a/docs/Documentation/UserGuide/4-Deployment and Management/7-Build and use IoTDB by Dockerfile.md b/docs/Documentation/UserGuide/4-Deployment and Management/7-Build and use IoTDB by Dockerfile.md
index 8b1f2a0f3a6fadc008437f11733b9fb8086db126..1924d1145104fe7a1cdf1ae2c9c78e4f4e63c893 100644
--- a/docs/Documentation/UserGuide/4-Deployment and Management/7-Build and use IoTDB by Dockerfile.md
+++ b/docs/Documentation/UserGuide/4-Deployment and Management/7-Build and use IoTDB by Dockerfile.md
@@ -66,16 +66,16 @@ suppose the IP is .
4. If you just want to have a try by using iotdb-cli, you can:
```
$ docker exec -it /bin/bash
-$ (now you have enter the container): /cli/sbin/start-client.sh -h localhost -p 6667 -u root -pw root
+$ (now you have enter the container): /cli/sbin/start-cli.sh -h localhost -p 6667 -u root -pw root
```
Or, run a new docker container as the client:
```
-$ docker run -it iotdb:base /cli/sbin/start-client.sh -h -p 6667 -u root -pw root
+$ docker run -it iotdb:base /cli/sbin/start-cli.sh -h -p 6667 -u root -pw root
```
Or, if you have a iotdb-cli locally (e.g., you have compiled the source code by `mvn package`), and suppose your work_dir is cli/bin, then you can just run:
```
-$ start-client.sh -h localhost -p 6667 -u root -pw root
+$ start-cli.sh -h localhost -p 6667 -u root -pw root
```
5. If you want to write codes to insert data and query data, please add the following dependence:
```xml
diff --git a/docs/Documentation/UserGuide/6-JDBC API/1-JDBC API.md b/docs/Documentation/UserGuide/6-JDBC API/1-JDBC API.md
index e194a35bffac22c0391f3f781c6690ce7ca8e27d..d1c865f7f7e107f492db147bc4d4ee0bca3192f6 100644
--- a/docs/Documentation/UserGuide/6-JDBC API/1-JDBC API.md
+++ b/docs/Documentation/UserGuide/6-JDBC API/1-JDBC API.md
@@ -19,7 +19,7 @@
-->
-# Chaper6: JDBC API
+# Chapter6: JDBC API
# Usage
@@ -57,6 +57,8 @@ This chapter provides an example of how to open a database connection, execute a
Requires that you include the packages containing the JDBC classes needed for database programming.
+**NOTE: For faster insertion, the insertBatch() in Session is recommended.**
+
```Java
import java.sql.*;
import org.apache.iotdb.jdbc.IoTDBSQLException;
diff --git a/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md b/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md
new file mode 100644
index 0000000000000000000000000000000000000000..ea62c35776b7a5b93691a52d9fc42e52ccd4a26c
--- /dev/null
+++ b/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md
@@ -0,0 +1,64 @@
+
+
+# Chapter6: JDBC API
+
+## Status Code
+
+**Status Code** is introduced in the latest version. For example, as IoTDB requires registering the time series first before writing data, a kind of solution is:
+
+```
+try {
+ writeData();
+} catch (SQLException e) {
+ // the most case is that the time series does not exist
+ if (e.getMessage().contains("exist")) {
+ //However, using the content of the error message is not so efficient
+ registerTimeSeries();
+ //write data once again
+ writeData();
+ }
+}
+
+```
+
+With Status Code, instead of writing codes like `if (e.getErrorMessage().contains("exist"))`, we can simply use `e.getStatusType().getCode() == TSStatusType.TIME_SERIES_NOT_EXIST_ERROR.getStatusCode()`.
+
+Here is a list of Status Code and related message:
+
+|Status Code|Status Type|Status Message|
+|:---|:---|:---|
+|200|SUCCESS_STATUS||
+|201|STILL_EXECUTING_STATUS||
+|202|INVALID_HANDLE_STATUS||
+|301|TIMESERIES_NOT_EXIST_ERROR|Timeseries does not exist|
+|302|UNSUPPORTED_FETCH_METADATA_OPERATION_ERROR|Unsupported fetch metadata operation|
+|303|FETCH_METADATA_ERROR|Failed to fetch metadata|
+|304|CHECK_FILE_LEVEL_ERROR|Meet error while checking file level|
+|400|EXECUTE_STATEMENT_ERROR|Execute statement error|
+|401|SQL_PARSE_ERROR|Meet error while parsing SQL|
+|402|GENERATE_TIME_ZONE_ERROR|Meet error while generating time zone|
+|403|SET_TIME_ZONE_ERROR|Meet error while setting time zone|
+|500|INTERNAL_SERVER_ERROR|Internal server error|
+|600|WRONG_LOGIN_PASSWORD_ERROR|Username or password is wrong|
+|601|NOT_LOGIN_ERROR|Has not logged in|
+|602|NO_PERMISSION_ERROR|No permissions for this operation|
+|603|UNINITIALIZED_AUTH_ERROR|Uninitialized authorizer|
diff --git a/docs/Documentation/UserGuide/7-Session API/1-Session API.md b/docs/Documentation/UserGuide/7-Session API/1-Session API.md
new file mode 100644
index 0000000000000000000000000000000000000000..b2cd10cf21b05ed1cc36a4809f86490f6377e1e5
--- /dev/null
+++ b/docs/Documentation/UserGuide/7-Session API/1-Session API.md
@@ -0,0 +1,115 @@
+
+
+# Chapter7: Session API
+
+# Usage
+
+## Dependencies
+
+* JDK >= 1.8
+* Maven >= 3.0
+
+## How to package only client module
+
+In root directory:
+> mvn clean package -pl client -am -Dmaven.test.skip=true
+
+## How to install in local maven repository
+
+In root directory:
+> mvn clean install -pl client -am -Dmaven.test.skip=true
+
+## Using IoTDB Session with Maven
+
+```
+
+
+ org.apache.iotdb
+ iotdb-client
+ 0.9.0-SNAPSHOT
+
+
+```
+
+
+## Examples with Session
+
+This chapter provides an example of how to open an IoTDB session, execute a batch insertion.
+
+Requires that you include the packages containing the Client classes needed for database programming.
+
+```Java
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.write.record.RowBatch;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
+
+public class SessionExample {
+
+ public static void main(String[] args) throws ClassNotFoundException, IoTDBSessionException {
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open();
+
+ session.setStorageGroup("root.sg1");
+ session.createTimeseriesResp("root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE);
+ session.createTimeseriesResp("root.sg1.d1.s2", TSDataType.INT64, TSEncoding.RLE);
+ session.createTimeseriesResp("root.sg1.d1.s3", TSDataType.INT64, TSEncoding.RLE);
+
+ Schema schema = new Schema();
+ schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
+ schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
+ schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
+
+ RowBatch rowBatch = schema.createRowBatch("root.sg1.d1", 100);
+
+ long[] timestamps = rowBatch.timestamps;
+ Object[] values = rowBatch.values;
+
+ for (long time = 0; time < 30000; time++) {
+ int row = rowBatch.batchSize++;
+ timestamps[row] = time;
+ for (int i = 0; i < 3; i++) {
+ long[] sensor = (long[]) values[i];
+ sensor[row] = time;
+ }
+ if (rowBatch.batchSize == rowBatch.getMaxBatchSize()) {
+ session.insertBatch(rowBatch);
+ rowBatch.reset();
+ }
+ }
+
+ if (rowBatch.batchSize != 0) {
+ session.insertBatch(rowBatch);
+ rowBatch.reset();
+ }
+
+ session.close();
+ }
+}
+```
+
+> The code is in example/session/src/main/java/org/apache/iotdb/session/SessionExample.java
\ No newline at end of file
diff --git a/docs/Documentation/UserGuide/7-TsFile/1-Installation.md b/docs/Documentation/UserGuide/8-TsFile/1-Installation.md
similarity index 99%
rename from docs/Documentation/UserGuide/7-TsFile/1-Installation.md
rename to docs/Documentation/UserGuide/8-TsFile/1-Installation.md
index 9c3c9ae950b3daf4d55d91cca1dab26366e6b99d..269d524c8c97f4931fab19361c573eae54e5e057 100644
--- a/docs/Documentation/UserGuide/7-TsFile/1-Installation.md
+++ b/docs/Documentation/UserGuide/8-TsFile/1-Installation.md
@@ -19,7 +19,7 @@
-->
-# Chapter 7: TsFile
+# Chapter 8: TsFile
## Installation
diff --git a/docs/Documentation/UserGuide/7-TsFile/2-Usage.md b/docs/Documentation/UserGuide/8-TsFile/2-Usage.md
similarity index 96%
rename from docs/Documentation/UserGuide/7-TsFile/2-Usage.md
rename to docs/Documentation/UserGuide/8-TsFile/2-Usage.md
index eed28a5b954af6e4528be121f81c6ec0dea60c2f..ae8bb14f8f3dd1858791571bd7fdb19b384f13da 100644
--- a/docs/Documentation/UserGuide/7-TsFile/2-Usage.md
+++ b/docs/Documentation/UserGuide/8-TsFile/2-Usage.md
@@ -19,7 +19,7 @@
-->
-# Chapter 7: TsFile
+# Chapter 8: TsFile
## Usage
@@ -85,12 +85,12 @@ A TsFile can be generated by following three steps and the complete code will be
```
* With pre-defined schema
```
- public TsFileWriter(File file, FileSchema schema) throws IOException
+ public TsFileWriter(File file, Schema schema) throws IOException
```
This one is for using the HDFS file system. `TsFileOutput` can be an instance of class `HDFSOutput`.
```
- public TsFileWriter(TsFileOutput output, FileSchema schema) throws IOException
+ public TsFileWriter(TsFileOutput output, Schema schema) throws IOException
```
**Parameters:**
@@ -100,15 +100,15 @@ A TsFile can be generated by following three steps and the complete code will be
* Second, add measurements
- Or you can make an instance of class `FileSchema` first and pass this to the constructor of class `TsFileWriter`
+ Or you can make an instance of class `Schema` first and pass this to the constructor of class `TsFileWriter`
- The class `FileSchema` contains a map whose key is the name of one measurement schema, and the value is the schema itself.
+ The class `Schema` contains a map whose key is the name of one measurement schema, and the value is the schema itself.
Here are the interfaces:
```
- // Create an empty FileSchema or from an existing map
- public FileSchema()
- public FileSchema(Map measurements)
+ // Create an empty Schema or from an existing map
+ public Schema()
+ public Schema(Map measurements)
// Use this two interfaces to add measurements
public void registerMeasurement(MeasurementSchema descriptor)
@@ -259,7 +259,7 @@ import java.io.File;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.write.TsFileWriter;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.apache.iotdb.tsfile.write.record.RowBatch;
/**
@@ -275,7 +275,7 @@ public class TsFileWriteWithRowBatch {
f.delete();
}
- FileSchema fileSchema = new FileSchema();
+ Schema schema = new Schema();
// the number of rows to include in the row batch
int rowNum = 1000000;
@@ -284,15 +284,15 @@ public class TsFileWriteWithRowBatch {
// add measurements into file schema (all with INT64 data type)
for (int i = 0; i < sensorNum; i++) {
- fileSchema.registerMeasurement(
+ schema.registerMeasurement(
new MeasurementSchema("sensor_" + (i + 1), TSDataType.INT64, TSEncoding.TS_2DIFF));
}
// add measurements into TSFileWriter
- TsFileWriter tsFileWriter = new TsFileWriter(f, fileSchema);
+ TsFileWriter tsFileWriter = new TsFileWriter(f, schema);
// construct the row batch
- RowBatch rowBatch = fileSchema.createRowBatch("device_1");
+ RowBatch rowBatch = schema.createRowBatch("device_1");
long[] timestamps = rowBatch.timestamps;
Object[] values = rowBatch.values;
diff --git a/docs/Documentation/UserGuide/7-TsFile/3-Hierarchy.md b/docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md
similarity index 68%
rename from docs/Documentation/UserGuide/7-TsFile/3-Hierarchy.md
rename to docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md
index 643417f54ec4810d0ebe27849479c7b8c99ece09..732c5e688e78da2b1acaefc5ecde125ea65f3c6d 100644
--- a/docs/Documentation/UserGuide/7-TsFile/3-Hierarchy.md
+++ b/docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md
@@ -7,9 +7,9 @@
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,53 +18,63 @@
under the License.
-->
-# Chapter 7: TsFile
+
+# Chapter 8: TsFile
+
## TsFile Hierarchy
+
Here is a brief introduction of the structure of a TsFile file.
-
+
## Variable Storage
- * **Big Endian**
-
- * For Example, the `int` `0x8` will be stored as `00 00 00 08`, not `08 00 00 00`
-
- * **String with Variable Length**
-
- * The format is `int size` plus `String literal`. Size can be zero.
-
- * Size equals the number of bytes this string will take, and it may not equal to the length of the string.
-
- * For example "sensor_1" will be stored as `00 00 00 08` plus the encoding(ASCII) of "sensor_1".
-
- * Note that for the "Magic String"(file signature) "TsFilev0.8.0", the size(12) and encoding(ASCII)
+
+- **Big Endian**
+
+ - For Example, the `int` `0x8` will be stored as `00 00 00 08`, not `08 00 00 00`
+- **String with Variable Length**
+ - The format is `int size` plus `String literal`. Size can be zero.
+ - Size equals the number of bytes this string will take, and it may not equal to the length of the string.
+ - For example "sensor_1" will be stored as `00 00 00 08` plus the encoding(ASCII) of "sensor_1".
+ - Note that for the "Magic String"(file signature) "TsFilev0.8.0", the size(12) and encoding(ASCII)
is fixed so there is no need to put the size before this string literal.
-
- * **Data Type Hardcode**
- * 0: BOOLEAN
- * 1: INT32 (`int`)
- * 2: INT64 (`long`)
- * 3: FLOAT
- * 4: DOUBLE
- * 5: TEXT (`String`)
- * **Encoding Type Hardcode**
- * 0: PLAIN
- * 1: PLAIN_DICTIONARY
- * 2: RLE
- * 3: DIFF
- * 4: TS_2DIFF
- * 5: BITMAP
- * 6: GORILLA
- * 7: REGULAR
- * **Compressing Type Hardcode**
- * 0: UNCOMPRESSED
- * 1: SNAPPY
-
-
+- **Data Type Hardcode**
+ - 0: BOOLEAN
+ - 1: INT32 (`int`)
+ - 2: INT64 (`long`)
+ - 3: FLOAT
+ - 4: DOUBLE
+ - 5: TEXT (`String`)
+- **Encoding Type Hardcode**
+ - 0: PLAIN
+ - 1: PLAIN_DICTIONARY
+ - 2: RLE
+ - 3: DIFF
+ - 4: TS_2DIFF
+ - 5: BITMAP
+ - 6: GORILLA
+ - 7: REGULAR
+- **Compressing Type Hardcode**
+ - 0: UNCOMPRESSED
+ - 1: SNAPPY
+ - 2: GZIP
+ - 3: LZO
+ - 4: SDT
+ - 5: PAA
+ - 6: PLA
+- **TsDigest Statistics Type Hardcode**
+ - 0: min_value
+ - 1: max_value
+ - 2: first_value
+ - 3: last_value
+ - 4: sum_value
+
## TsFile Overview
+
Here is a graph about the TsFile structure.
![TsFile Breakdown](https://user-images.githubusercontent.com/40447846/61616997-6fad1300-ac9c-11e9-9c17-46785ebfbc88.png)
## Magic String
+
There is a 12 bytes magic string:
`TsFilev0.8.0`
@@ -87,6 +97,7 @@ The `ChunkGroup` has an array of `Chunk`, a following byte `0x00` as the marker,
A `Chunk` represents a *sensor*. There is a byte `0x01` as the marker, following a `ChunkHeader` and an array of `Page`.
###### ChunkHeader
+
Member Description | Member Type |
@@ -136,6 +147,7 @@ PageHeader Structure
## Metadata
### TsDeviceMetaData
+
The first part of metadata is `TsDeviceMetaData`
@@ -148,6 +160,7 @@ The first part of metadata is `TsDeviceMetaData`
Then there is an array of `ChunkGroupMetaData` after `TsDeviceMetaData`
+
### ChunkGroupMetaData
@@ -179,14 +192,57 @@ Then there is an array of `ChunkMetadata` for each `ChunkGroupMetadata`
-###### TsDigest
+###### TsDigest (updated on 2019/8/27)
+
+Right now there are five statistics: `min_value, max_value, first_value, last_value, sum_value`.
+
+In v0.8.0, the storage format of statistics is a name-value pair. That is, `Map statistics`. The name is a string (remember the length is before the literal). But for the value, there is also an integer byteLength acting as the self description length of the following value because the value may be of various type. For example, if the `min_value` is an integer 0, then it will be stored as [9 "min_value" 4 0] in the TsFile.
+
+The figure below shows an example of `TsDigest.deserializeFrom(buffer)`. In v0.8.0, we will get
-There are five statistics: `min, last, sum, first, max`
+```
+Map statistics = {
+ "min_value" -> ByteBuffer of int value 0,
+ "last" -> ByteBuffer of int value 19,
+ "sum" -> ByteBuffer of double value 1093347116,
+ "first" -> ByteBuffer of int value 0,
+ "max_value" -> ByteBuffer of int value 99
+}
+```
-The storage format is a name-value pair. The name is a string (remember the length is before the literal).
+![TsDigest ByteBuffer Breakdown comparison](https://user-images.githubusercontent.com/33376433/63765352-664a4280-c8fb-11e9-869e-859edf6d00bb.png)
-But for the value, there is also a size integer before the data even if it is not string. For example, if the `min` is 3, then it will be
-stored as 3 "min" 4 3 in the TsFile.
+In v0.9.0, the storage format is changed to an array for space and time efficiency. That is, `ByteBuffer[] statistics`. Each position of the array has a fixed association with a specific type of statistic, following the order defined in StatisticType:
+
+```
+enum StatisticType {
+ min_value, max_value, first_value, last_value, sum_value
+}
+```
+
+Therefore, in the above example, we will get
+
+```
+ByteBuffer[] statistics = [
+ ByteBuffer of int value 0, // associated with "min_value"
+ ByteBuffer of int value 99, // associated with "max_value"
+ ByteBuffer of int value 0, // associated with "first_value"
+ ByteBuffer of int value 19, // associated with "last_value"
+ ByteBuffer of double value 1093347116 // associated with "sum_value"
+]
+```
+
+As another example in v0.9.0, when deserializing a TsDigest from buffer [3, 0,4,0, 1,4,99, 3,4,19], we get
+
+```
+ByteBuffer[] statistics = [
+ ByteBuffer of int value 0, // associated with "min_value"
+ ByteBuffer of int value 99, // associated with "max_value"
+ null, // associated with "first_value"
+ ByteBuffer of int value 19, // associated with "last_value"
+ null // associated with "sum_value"
+]
+```
#### File Metadata
@@ -207,6 +263,7 @@ After the array of `ChunkGroupMetadata`, here is the last part of the metadata.
##### DeviceIndexMetadata
+
Member Description | Member Type |
@@ -219,6 +276,7 @@ After the array of `ChunkGroupMetadata`, here is the last part of the metadata.
##### MeasurementSchema
+
Member Description | Member Type |
diff --git a/docs/Documentation/UserGuide/8-System Tools/1-Sync.md b/docs/Documentation/UserGuide/9-System Tools/1-Sync.md
similarity index 99%
rename from docs/Documentation/UserGuide/8-System Tools/1-Sync.md
rename to docs/Documentation/UserGuide/9-System Tools/1-Sync.md
index 741b36528edee82064802f3900a3a400cacd55b3..c41c34d44516c0ae5689ade0c365e629bf316c3a 100644
--- a/docs/Documentation/UserGuide/8-System Tools/1-Sync.md
+++ b/docs/Documentation/UserGuide/9-System Tools/1-Sync.md
@@ -19,7 +19,7 @@
-->
-# Chapter 8: System Tools
+# Chapter 9: System Tools
## Data Import
diff --git a/docs/Documentation/UserGuide/8-System Tools/2-Memory Estimation Tool.md b/docs/Documentation/UserGuide/9-System Tools/2-Memory Estimation Tool.md
similarity index 99%
rename from docs/Documentation/UserGuide/8-System Tools/2-Memory Estimation Tool.md
rename to docs/Documentation/UserGuide/9-System Tools/2-Memory Estimation Tool.md
index e11aedbf05d30c522682e3785e1b8adb1467f7f0..a38793f8f2b2a94a617c60746e9c789fc4131ea7 100644
--- a/docs/Documentation/UserGuide/8-System Tools/2-Memory Estimation Tool.md
+++ b/docs/Documentation/UserGuide/9-System Tools/2-Memory Estimation Tool.md
@@ -19,7 +19,7 @@
-->
-# Chapter 8: System Tools
+# Chapter 9: System Tools
# Memory Estimation Tool
diff --git a/docs/Documentation/UserGuide/8-Tools-Cli.md b/docs/Documentation/UserGuide/9-Tools-Cli.md
similarity index 80%
rename from docs/Documentation/UserGuide/8-Tools-Cli.md
rename to docs/Documentation/UserGuide/9-Tools-Cli.md
index ff5df5419dceb21a07672b77d2fcfba0b653fd4a..adc42f6f9f12a4c1ff82e3f56d6f0400107423fc 100644
--- a/docs/Documentation/UserGuide/8-Tools-Cli.md
+++ b/docs/Documentation/UserGuide/9-Tools-Cli.md
@@ -44,12 +44,12 @@ the port number of the server running, set the specific IP and PORT at -h and -p
The Linux and MacOS system startup commands are as follows:
```
- Shell > ./sbin/start-client.sh -h 127.0.0.1 -p 6667 -u root -pw root
+ Shell > ./sbin/start-cli.sh -h 127.0.0.1 -p 6667 -u root -pw root
```
The Windows system startup commands are as follows:
```
- Shell > \sbin\start-client.bat -h 127.0.0.1 -p 6667 -u root -pw root
+ Shell > \sbin\start-cli.bat -h 127.0.0.1 -p 6667 -u root -pw root
```
After using these commands, the client can be started successfully. The successful status will be as follows:
@@ -86,12 +86,12 @@ Following is a client command which connects the host with IP
The Linux and MacOS system startup commands are as follows:
```
- Shell > ./sbin/start-client.sh -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10
+ Shell > ./sbin/start-cli.sh -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10
```
The Windows system startup commands are as follows:
```
- Shell > \sbin\start-client.bat -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10
+ Shell > \sbin\start-cli.bat -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10
```
## Cli/shell tool with -e parameter
@@ -102,7 +102,7 @@ In order to avoid confusion between statements and other parameters, the current
The usage of -e parameter for Cli/shell is as follows:
```
- Shell > ./sbin/start-client.sh -h {host} -p {port} -u {user} -pw {password} -e {sql for iotdb}
+ Shell > ./sbin/start-cli.sh -h {host} -p {port} -u {user} -pw {password} -e {sql for iotdb}
```
In order to better explain the use of -e parameter, take following as an example.
@@ -117,12 +117,12 @@ port=6667
user=root
pass=root
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "set storage group to root.demo"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "create timeseries root.demo.s1 WITH DATATYPE=INT32, ENCODING=RLE"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(1,10)"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(2,11)"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(3,12)"
-./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "select s1 from root.demo"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "set storage group to root.demo"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "create timeseries root.demo.s1 WITH DATATYPE=INT32, ENCODING=RLE"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(1,10)"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(2,11)"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(3,12)"
+./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "select s1 from root.demo"
```
The print results are shown in the figure, which are consistent with the client and jdbc operations.
diff --git a/docs/Documentation/UserGuide/8-Tools-Grafana.md b/docs/Documentation/UserGuide/9-Tools-Grafana.md
similarity index 100%
rename from docs/Documentation/UserGuide/8-Tools-Grafana.md
rename to docs/Documentation/UserGuide/9-Tools-Grafana.md
diff --git a/docs/Documentation/UserGuide/8-Tools-Hadoop.md b/docs/Documentation/UserGuide/9-Tools-Hadoop.md
similarity index 100%
rename from docs/Documentation/UserGuide/8-Tools-Hadoop.md
rename to docs/Documentation/UserGuide/9-Tools-Hadoop.md
diff --git a/docs/Documentation/UserGuide/8-Tools-spark.md b/docs/Documentation/UserGuide/9-Tools-spark.md
similarity index 100%
rename from docs/Documentation/UserGuide/8-Tools-spark.md
rename to docs/Documentation/UserGuide/9-Tools-spark.md
diff --git a/example/jdbc/pom.xml b/example/jdbc/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..065731f471b988f4e3b717b36dc346536d9f2bc1
--- /dev/null
+++ b/example/jdbc/pom.xml
@@ -0,0 +1,43 @@
+
+
+
+
+ iotdb-examples
+ org.apache.iotdb
+ 0.9.0-SNAPSHOT
+
+ 4.0.0
+ jdbc-example
+ jdbc-example
+
+ UTF-8
+ 1.7
+ 1.7
+
+
+
+ org.apache.iotdb
+ iotdb-jdbc
+ 0.9.0-SNAPSHOT
+
+
+
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/StatementDemo.java b/example/jdbc/src/main/java/org/apache/iotdb/JDBCExample.java
similarity index 52%
rename from jdbc/src/test/java/org/apache/iotdb/jdbc/demo/StatementDemo.java
rename to example/jdbc/src/main/java/org/apache/iotdb/JDBCExample.java
index a122e8a00fd8238e4d0b363884c0e099f1ea9048..f67caf03f500bb296e81447942288f763be13dd1 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/StatementDemo.java
+++ b/example/jdbc/src/main/java/org/apache/iotdb/JDBCExample.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.jdbc.demo;
+package org.apache.iotdb;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -25,31 +25,24 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
-public class StatementDemo {
+public class JDBCExample {
public static void main(String[] args) throws ClassNotFoundException, SQLException {
Class.forName("org.apache.iotdb.jdbc.IoTDBDriver");
- Connection connection = null;
- try {
- connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
- Statement statement = connection.createStatement();
- statement.execute("SET STORAGE GROUP TO root.ln.wf01.wt01");
- statement.execute(
- "CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN, ENCODING=PLAIN");
- statement.execute(
- "CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE");
- statement
- .execute("insert into root.ln.wf01.wt01(timestamp,status) values(1509465600000,true)");
- statement
- .execute("insert into root.ln.wf01.wt01(timestamp,status) values(1509465660000,true)");
- statement
- .execute("insert into root.ln.wf01.wt01(timestamp,status) values(1509465720000,false)");
- statement.execute(
- "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465600000,25.957603)");
- statement.execute(
- "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465660000,24.359503)");
- statement.execute(
- "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465720000,20.092794)");
+ try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("SET STORAGE GROUP TO root.sg1");
+ statement.execute("CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT64, ENCODING=RLE");
+ statement.execute("CREATE TIMESERIES root.sg1.d1.s2 WITH DATATYPE=INT64, ENCODING=RLE");
+ statement.execute("CREATE TIMESERIES root.sg1.d1.s3 WITH DATATYPE=INT64, ENCODING=RLE");
+
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0 ; j < 10; j++) {
+ statement.addBatch("insert into root.sg1.d1(timestamp, s1, s2, s3) values("+ (i * 10 + j) + "," + 1 + "," + 1 + "," + 1 + ")");
+ }
+ statement.executeBatch();
+ statement.clearBatch();
+ }
ResultSet resultSet = statement.executeQuery("select * from root");
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
while (resultSet.next()) {
@@ -57,12 +50,7 @@ public class StatementDemo {
for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
builder.append(resultSet.getString(i)).append(",");
}
- System.out.println(builder);
}
- statement.close();
-
- } finally {
- connection.close();
}
}
}
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/PrepareStatementDemo.java b/example/jdbc/src/main/java/org/apache/iotdb/PrepareStatementDemo.java
similarity index 69%
rename from jdbc/src/test/java/org/apache/iotdb/jdbc/demo/PrepareStatementDemo.java
rename to example/jdbc/src/main/java/org/apache/iotdb/PrepareStatementDemo.java
index 3f0e3a0d5137efaf71c8c5c9784db23e0481acf3..5ab8d24c2e7e316ddac1136ae5bab146008eb83c 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/PrepareStatementDemo.java
+++ b/example/jdbc/src/main/java/org/apache/iotdb/PrepareStatementDemo.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.jdbc.demo;
+package org.apache.iotdb;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -30,12 +30,12 @@ public class PrepareStatementDemo {
public static void main(String[] args) throws ClassNotFoundException, SQLException {
Class.forName("org.apache.iotdb.jdbc.IoTDBDriver");
- Connection connection = null;
- try {
- connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
- PreparedStatement preparedStatement = connection
- .prepareStatement(
- "insert into root.ln.wf01.wt01(timestamp,status,temperature) values(?,?,?)");
+ try (Connection connection = DriverManager
+ .getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ PreparedStatement preparedStatement = connection
+ .prepareStatement(
+ "insert into root.ln.wf01.wt01(timestamp,status,temperature) values(?,?,?)")) {
+
preparedStatement.setLong(1, 1509465600000L);
preparedStatement.setBoolean(2, true);
preparedStatement.setFloat(3, 25.957603f);
@@ -60,24 +60,22 @@ public class PrepareStatementDemo {
preparedStatement.execute();
preparedStatement.clearParameters();
- preparedStatement.close();
- ResultSet resultSet = preparedStatement.executeQuery("select * from root");
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- while (resultSet.next()) {
- StringBuilder builder = new StringBuilder();
+
+ try(ResultSet resultSet = preparedStatement.executeQuery("select * from root")) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ while (resultSet.next()) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ builder.append(resultSet.getString(i)).append(",");
+ }
+ System.out.println(builder);
+ }
for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- builder.append(resultSet.getString(i)).append(",");
+ System.out
+ .println(resultSetMetaData.getColumnType(i) + "-" + resultSetMetaData.getColumnName(i));
}
- System.out.println(builder);
- }
- preparedStatement.close();
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- System.out
- .println(resultSetMetaData.getColumnType(i) + "-" + resultSetMetaData.getColumnName(i));
}
- } finally {
- connection.close();
}
}
diff --git a/example/pom.xml b/example/pom.xml
index 31e2d3f9bcba3d156f600be5aa26fa41ec10d104..4750be31d473f65e86db72dd6f166853553b2244 100644
--- a/example/pom.xml
+++ b/example/pom.xml
@@ -36,6 +36,7 @@
kafka
rocketmq
+ session
tsfile
diff --git a/example/rocketmq/src/main/java/org/apache/iotdb/example/Constant.java b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Constant.java
similarity index 99%
rename from example/rocketmq/src/main/java/org/apache/iotdb/example/Constant.java
rename to example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Constant.java
index 1a290554a08b3ab4ac42c241568a9009183d4162..72b08e446f0f4bce539e5f833dbd817998d306c0 100644
--- a/example/rocketmq/src/main/java/org/apache/iotdb/example/Constant.java
+++ b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Constant.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.example;
+package org.apache.iotdb.rocketmq;
import org.apache.iotdb.jdbc.Config;
diff --git a/example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQConsumer.java b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQConsumer.java
similarity index 99%
rename from example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQConsumer.java
rename to example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQConsumer.java
index 2fbda076873f8ca399246639a5b8efa9bfda2d2c..c9bcf91ad258afa9ed5025f013b4444a8d40f38d 100644
--- a/example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQConsumer.java
+++ b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQConsumer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.example;
+package org.apache.iotdb.rocketmq;
import java.sql.Connection;
import java.sql.DriverManager;
diff --git a/example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQProducer.java b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQProducer.java
similarity index 98%
rename from example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQProducer.java
rename to example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQProducer.java
index 2e231d95b28d6dfe09c745d6db39dc22172d68bd..fc44e9511c144d22c188c429ae3cda40c964553c 100644
--- a/example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQProducer.java
+++ b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQProducer.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.example;
+package org.apache.iotdb.rocketmq;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
diff --git a/example/rocketmq/src/main/java/org/apache/iotdb/example/Utils.java b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Utils.java
similarity index 97%
rename from example/rocketmq/src/main/java/org/apache/iotdb/example/Utils.java
rename to example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Utils.java
index c0a9eb1c24e3a79a2a966ced3386ee1802760fdc..e6b0ad1af0e8710f2ff2eb943dc19372fa3e059b 100644
--- a/example/rocketmq/src/main/java/org/apache/iotdb/example/Utils.java
+++ b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Utils.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.example;
+package org.apache.iotdb.rocketmq;
/**
* @author lta
diff --git a/example/session/pom.xml b/example/session/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..8b1b2152f9b05393ca0440e4235fb782fc1c581c
--- /dev/null
+++ b/example/session/pom.xml
@@ -0,0 +1,43 @@
+
+
+
+
+ iotdb-examples
+ org.apache.iotdb
+ 0.9.0-SNAPSHOT
+
+ 4.0.0
+ client-example
+ client-example
+
+ UTF-8
+ 1.7
+ 1.7
+
+
+
+ org.apache.iotdb
+ iotdb-session
+ 0.9.0-SNAPSHOT
+
+
+
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
new file mode 100644
index 0000000000000000000000000000000000000000..0becd798271edc40af8c47ccfffa6969d390ed1e
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb;
+
+import org.apache.iotdb.session.IoTDBSessionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.write.record.RowBatch;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
+
+public class SessionExample {
+
+ public static void main(String[] args) throws IoTDBSessionException {
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open();
+
+ session.setStorageGroup("root.sg1");
+ session.createTimeseries("root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE);
+ session.createTimeseries("root.sg1.d1.s2", TSDataType.INT64, TSEncoding.RLE);
+ session.createTimeseries("root.sg1.d1.s3", TSDataType.INT64, TSEncoding.RLE);
+
+ Schema schema = new Schema();
+ schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
+ schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
+ schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
+
+ RowBatch rowBatch = schema.createRowBatch("root.sg1.d1", 100);
+
+ long[] timestamps = rowBatch.timestamps;
+ Object[] values = rowBatch.values;
+
+ for (long time = 0; time < 30000; time++) {
+ int row = rowBatch.batchSize++;
+ timestamps[row] = time;
+ for (int i = 0; i < 3; i++) {
+ long[] sensor = (long[]) values[i];
+ sensor[row] = time;
+ }
+ if (rowBatch.batchSize == rowBatch.getMaxBatchSize()) {
+ session.insertBatch(rowBatch);
+ rowBatch.reset();
+ }
+ }
+
+ if (rowBatch.batchSize != 0) {
+ session.insertBatch(rowBatch);
+ rowBatch.reset();
+ }
+ session.close();
+ }
+}
\ No newline at end of file
diff --git a/example/tsfile/pom.xml b/example/tsfile/pom.xml
index 1dab0e0400f9e8b10a0977829368173a6945841c..7a99f0134a5b2646024b6ac9f921c2b3043bbc15 100644
--- a/example/tsfile/pom.xml
+++ b/example/tsfile/pom.xml
@@ -19,32 +19,22 @@
under the License.
-->
-
- 4.0.0
-
- org.apache.iotdb
- iotdb-examples
+
+ 4.0.0
+
+ org.apache.iotdb
+ iotdb-examples
+ 0.9.0-SNAPSHOT
+ ../pom.xml
+
+ tsfile-example
0.9.0-SNAPSHOT
- ../pom.xml
-
- org.apache.iotdb
- tsfile-example
- 0.9.0-SNAPSHOT
- tsfile-example
-
- http://www.example.com
-
-
- junit
- junit
- 4.11
- test
-
-
- org.apache.iotdb
- tsfile
- 0.9.0-SNAPSHOT
-
-
+ tsfile-example
+
+
+ org.apache.iotdb
+ tsfile
+ 0.9.0-SNAPSHOT
+
+
diff --git a/example/tsfile/readme.md b/example/tsfile/readme.md
index 9615cdd05aa8de58362d175960b8e35fba66fbad..000cb7c2d21a75d9c12109fe522d6ec6218f8dae 100644
--- a/example/tsfile/readme.md
+++ b/example/tsfile/readme.md
@@ -81,4 +81,4 @@ The example is to show how to write and read a TsFile File.
This class is to show the structure of a TsFile.
### Notice
- For detail, please refer to https://github.com/thulab/tsfile/wiki/Get-Started.
+ For detail, please refer to https://github.com/apache/incubator-iotdb/blob/master/tsfile/README.md.
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
index 416ef742dc9f819ff3725549e677779d98c0e646..4bac3731c78cb9f728f5c38c74c148f1bbc20734 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java
@@ -23,9 +23,10 @@ import java.io.File;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.write.TsFileWriter;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.apache.iotdb.tsfile.write.record.RowBatch;
+
/**
* An example of writing data with RowBatch to TsFile
*/
@@ -39,7 +40,7 @@ public class TsFileWriteWithRowBatch {
f.delete();
}
- FileSchema fileSchema = new FileSchema();
+ Schema schema = new Schema();
// the number of rows to include in the row batch
int rowNum = 1000000;
@@ -48,15 +49,15 @@ public class TsFileWriteWithRowBatch {
// add measurements into file schema (all with INT64 data type)
for (int i = 0; i < sensorNum; i++) {
- fileSchema.registerMeasurement(
+ schema.registerMeasurement(
new MeasurementSchema("sensor_" + (i + 1), TSDataType.INT64, TSEncoding.TS_2DIFF));
}
// add measurements into TSFileWriter
- TsFileWriter tsFileWriter = new TsFileWriter(f, fileSchema);
+ TsFileWriter tsFileWriter = new TsFileWriter(f, schema);
// construct the row batch
- RowBatch rowBatch = fileSchema.createRowBatch("device_1");
+ RowBatch rowBatch = schema.createRowBatch("device_1");
long[] timestamps = rowBatch.timestamps;
Object[] values = rowBatch.values;
diff --git a/hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java b/hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java
index 3f23cf6c1ea77abeeeea8e8ce33709336147b65b..bfc2f8912fb7182cd78c4e935f7675f381ca69a9 100644
--- a/hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java
@@ -30,7 +30,7 @@ import org.apache.iotdb.tsfile.hadoop.io.HDFSOutputStream;
import org.apache.iotdb.tsfile.timeseries.basis.TsFile;
import org.apache.iotdb.tsfile.write.exception.InvalidJsonSchemaException;
import org.apache.iotdb.tsfile.write.exception.WriteProcessException;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,9 +42,9 @@ public class TSFRecordWriter extends RecordWriter {
public TSFRecordWriter(Path path, JSONObject schema) throws InterruptedException, IOException {
// construct the internalrecordwriter
- FileSchema fileSchema = null;
+ Schema fileSchema = null;
try {
- fileSchema = new FileSchema(schema);
+ fileSchema = new Schema(schema);
} catch (InvalidJsonSchemaException e) {
throw new InterruptedException(String.format("Construct the tsfile schema failed"), e);
}
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java
index c366c0c984b32d33cafff6c1c59f60cdb9ade013..85270e1a267d29020e129e316b85847d323ea3af 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java
@@ -18,11 +18,6 @@
*/
package org.apache.iotdb.jdbc;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.net.SocketException;
import java.sql.Array;
import java.sql.Blob;
import java.sql.CallableStatement;
@@ -40,21 +35,12 @@ import java.sql.Savepoint;
import java.sql.Statement;
import java.sql.Struct;
import java.time.ZoneId;
-import java.util.LinkedList;
-import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Executor;
-import org.apache.iotdb.service.rpc.thrift.ServerProperties;
-import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq;
-import org.apache.iotdb.service.rpc.thrift.TSGetTimeZoneResp;
-import org.apache.iotdb.service.rpc.thrift.TSIService;
-import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq;
-import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp;
-import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion;
-import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq;
-import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneResp;
-import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle;
+import org.apache.iotdb.rpc.IoTDBRPCException;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TCompactProtocol;
@@ -65,8 +51,8 @@ import org.slf4j.LoggerFactory;
public class IoTDBConnection implements Connection {
- Logger logger = LoggerFactory.getLogger(IoTDBConnection.class);
- private final List supportedProtocols = new LinkedList<>();
+ private static final Logger logger = LoggerFactory.getLogger(IoTDBConnection.class);
+ private final TSProtocolVersion protocolVersion = TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1;
public TSIService.Iface client = null;
public TS_SessionHandle sessionHandle = null;
private IoTDBConnectionParams params;
@@ -87,8 +73,6 @@ public class IoTDBConnection implements Connection {
}
params = Utils.parseUrl(url, info);
- supportedProtocols.add(TSProtocolVersion.TSFILE_SERVICE_PROTOCOL_V1);
-
openTransport();
if(Config.rpcThriftCompressionEnable) {
client = new TSIService.Client(new TCompactProtocol(transport));
@@ -99,15 +83,10 @@ public class IoTDBConnection implements Connection {
// open client session
openSession();
// Wrap the client with a thread-safe proxy to serialize the RPC calls
- client = newSynchronizedClient(client);
+ client = RpcUtils.newSynchronizedClient(client);
autoCommit = false;
}
- public static TSIService.Iface newSynchronizedClient(TSIService.Iface client) {
- return (TSIService.Iface) Proxy.newProxyInstance(IoTDBConnection.class.getClassLoader(),
- new Class[]{TSIService.Iface.class}, new SynchronizedHandler(client));
- }
-
@Override
public boolean isWrapperFor(Class> arg0) throws SQLException {
throw new SQLException("Method not supported");
@@ -416,18 +395,13 @@ public class IoTDBConnection implements Connection {
private void openTransport() throws TTransportException {
transport = new TSocket(params.getHost(), params.getPort(), Config.connectionTimeoutInMs);
- try {
- transport.getSocket().setKeepAlive(true);
- } catch (SocketException e) {
- logger.error("Cannot set socket keep alive because: ", e);
- }
if (!transport.isOpen()) {
transport.open();
}
}
private void openSession() throws SQLException {
- TSOpenSessionReq openReq = new TSOpenSessionReq(TSProtocolVersion.TSFILE_SERVICE_PROTOCOL_V1);
+ TSOpenSessionReq openReq = new TSOpenSessionReq(TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1);
openReq.setUsername(params.getUsername());
openReq.setPassword(params.getPassword());
@@ -437,14 +411,16 @@ public class IoTDBConnection implements Connection {
// validate connection
try {
- Utils.verifySuccess(openResp.getStatus());
- } catch (IoTDBSQLException e) {
+ RpcUtils.verifySuccess(openResp.getStatus());
+ } catch (IoTDBRPCException e) {
// failed to connect, disconnect from the server
transport.close();
- throw e;
+ throw new IoTDBSQLException(e.getMessage());
}
- if (!supportedProtocols.contains(openResp.getServerProtocolVersion())) {
- throw new TException("Unsupported TsFile protocol");
+ if (protocolVersion.getValue() != openResp.getServerProtocolVersion().getValue()) {
+ throw new TException(String
+ .format("Protocol not supported, Client version is {}, but Server version is {}",
+ protocolVersion.getValue(), openResp.getServerProtocolVersion().getValue()));
}
setProtocol(openResp.getServerProtocolVersion());
sessionHandle = openResp.getSessionHandle();
@@ -476,7 +452,7 @@ public class IoTDBConnection implements Connection {
client = new TSIService.Client(new TBinaryProtocol(transport));
}
openSession();
- client = newSynchronizedClient(client);
+ client = RpcUtils.newSynchronizedClient(client);
flag = true;
break;
}
@@ -497,14 +473,22 @@ public class IoTDBConnection implements Connection {
}
TSGetTimeZoneResp resp = client.getTimeZone();
- Utils.verifySuccess(resp.getStatus());
+ try {
+ RpcUtils.verifySuccess(resp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
return resp.getTimeZone();
}
public void setTimeZone(String zoneId) throws TException, IoTDBSQLException {
TSSetTimeZoneReq req = new TSSetTimeZoneReq(zoneId);
- TSSetTimeZoneResp resp = client.setTimeZone(req);
- Utils.verifySuccess(resp.getStatus());
+ TSRPCResp resp = client.setTimeZone(req);
+ try {
+ RpcUtils.verifySuccess(resp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
this.zoneId = ZoneId.of(zoneId);
}
@@ -520,32 +504,4 @@ public class IoTDBConnection implements Connection {
this.protocol = protocol;
}
- private static class SynchronizedHandler implements InvocationHandler {
-
- private final TSIService.Iface client;
-
- SynchronizedHandler(TSIService.Iface client) {
- this.client = client;
- }
-
- @Override
- public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
- try {
- synchronized (client) {
- return method.invoke(client, args);
- }
- } catch (InvocationTargetException e) {
- // all IFace APIs throw TException
- if (e.getTargetException() instanceof TException) {
- throw e.getTargetException();
- } else {
- // should not happen
- throw new TException("Error in calling method " + method.getName(),
- e.getTargetException());
- }
- } catch (Exception e) {
- throw new TException("Error in calling method " + method.getName(), e);
- }
- }
- }
}
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java
index 21e1405aa8ed21157e3917d48da87c0da2ab9116..36d01c2c830a2c28afe8ec7e7c4d67a490561819 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java
@@ -25,6 +25,8 @@ import java.sql.RowIdLifetime;
import java.sql.SQLException;
import java.util.List;
import java.util.Set;
+import org.apache.iotdb.rpc.IoTDBRPCException;
+import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq;
import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp;
import org.apache.iotdb.service.rpc.thrift.TSIService;
@@ -82,7 +84,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData {
req.setColumnPath(schemaPattern);
try {
TSFetchMetadataResp resp = client.fetchMetadata(req);
- Utils.verifySuccess(resp.getStatus());
+ try {
+ RpcUtils.verifySuccess(resp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
return new IoTDBMetadataResultSet(resp.getColumnsList(), null, null);
} catch (TException e) {
throw new TException("Conncetion error when fetching column metadata", e);
@@ -92,7 +98,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData {
req.setColumnPath(schemaPattern);
try {
TSFetchMetadataResp resp = client.fetchMetadata(req);
- Utils.verifySuccess(resp.getStatus());
+ try {
+ RpcUtils.verifySuccess(resp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
return new IoTDBMetadataResultSet(resp.getColumnsList(), null, null);
} catch (TException e) {
throw new TException("Conncetion error when fetching delta object metadata", e);
@@ -101,7 +111,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData {
req = new TSFetchMetadataReq(Constant.GLOBAL_SHOW_STORAGE_GROUP_REQ);
try {
TSFetchMetadataResp resp = client.fetchMetadata(req);
- Utils.verifySuccess(resp.getStatus());
+ try {
+ RpcUtils.verifySuccess(resp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
Set showStorageGroup = resp.getShowStorageGroups();
return new IoTDBMetadataResultSet(null, showStorageGroup, null);
} catch (TException e) {
@@ -112,7 +126,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData {
req.setColumnPath(schemaPattern);
try {
TSFetchMetadataResp resp = client.fetchMetadata(req);
- Utils.verifySuccess(resp.getStatus());
+ try {
+ RpcUtils.verifySuccess(resp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
List> showTimeseriesList = resp.getShowTimeseriesList();
return new IoTDBMetadataResultSet(null, null, showTimeseriesList);
} catch (TException e) {
@@ -1247,7 +1265,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData {
TSFetchMetadataReq req = new TSFetchMetadataReq("METADATA_IN_JSON");
TSFetchMetadataResp resp;
resp = client.fetchMetadata(req);
- Utils.verifySuccess(resp.getStatus());
+ try {
+ RpcUtils.verifySuccess(resp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
return resp.getMetadataInJson();
}
}
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java
index fbdb3ab5ad25a32179598b9bdf2bbbd55d52dc5e..2fe037073d9bcbac404ec8d01a5d95e460c3f90e 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java
@@ -22,11 +22,12 @@ package org.apache.iotdb.jdbc;
import java.sql.SQLException;
import java.time.ZoneId;
import java.util.List;
+
+import org.apache.iotdb.rpc.TSStatusType;
import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp;
import org.apache.iotdb.service.rpc.thrift.TSIService.Iface;
import org.apache.iotdb.service.rpc.thrift.TSInsertionReq;
import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
import org.apache.thrift.TException;
public class IoTDBPreparedInsertionStatement extends IoTDBPreparedStatement {
@@ -44,12 +45,12 @@ public class IoTDBPreparedInsertionStatement extends IoTDBPreparedStatement {
public boolean execute() throws SQLException {
try {
- TSExecuteStatementResp resp = client.executeInsertion(req);
+ TSExecuteStatementResp resp = client.insert(req);
req.unsetDeviceId();
req.unsetMeasurements();
req.unsetTimestamp();
req.unsetValues();
- return resp.getStatus().getStatusCode() == TS_StatusCode.SUCCESS_STATUS;
+ return resp.getStatus().getStatusType().getCode() == TSStatusType.SUCCESS_STATUS.getStatusCode();
} catch (TException e) {
throw new SQLException(e);
}
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java
index 3052dfc444703ea36638ce022e9b213a7603f2f5..52e05817c187580a1ed371e82f8735ac9cab68df 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java
@@ -48,6 +48,7 @@ import java.util.List;
import java.util.Map;
import org.apache.iotdb.service.rpc.thrift.TSIService.Iface;
import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle;
+import org.apache.thrift.TException;
public class IoTDBPreparedStatement extends IoTDBStatement implements PreparedStatement {
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java
index 632e1b11071ae34d8ba3275925e371676f85c2ef..012c7666f7f3e994732b5f1765cea7dccfa7b4bc 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java
@@ -46,13 +46,9 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq;
-import org.apache.iotdb.service.rpc.thrift.TSCloseOperationResp;
-import org.apache.iotdb.service.rpc.thrift.TSFetchResultsReq;
-import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp;
-import org.apache.iotdb.service.rpc.thrift.TSIService;
-import org.apache.iotdb.service.rpc.thrift.TSOperationHandle;
-import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet;
+import org.apache.iotdb.rpc.IoTDBRPCException;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.apache.iotdb.tsfile.read.common.Field;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.thrift.TException;
@@ -213,10 +209,10 @@ public class IoTDBQueryResultSet implements ResultSet {
try {
if (operationHandle != null) {
TSCloseOperationReq closeReq = new TSCloseOperationReq(operationHandle, queryId);
- TSCloseOperationResp closeResp = client.closeOperation(closeReq);
- Utils.verifySuccess(closeResp.getStatus());
+ TSRPCResp closeResp = client.closeOperation(closeReq);
+ RpcUtils.verifySuccess(closeResp.getStatus());
}
- } catch (SQLException e) {
+ } catch (IoTDBRPCException e) {
throw new SQLException("Error occurs for close opeation in server side becasuse " + e);
} catch (TException e) {
throw new SQLException(
@@ -708,7 +704,11 @@ public class IoTDBQueryResultSet implements ResultSet {
try {
TSFetchResultsResp resp = client.fetchResults(req);
- Utils.verifySuccess(resp.getStatus());
+ try {
+ RpcUtils.verifySuccess(resp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
if (!resp.hasResultSet) {
emptyResultSet = true;
} else {
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBSQLException.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBSQLException.java
index c265fe0d39510367e5985cdfbc727f3f2f89d963..8a1a916d2b4db3073881c4fef3da0a8df30fc357 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBSQLException.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBSQLException.java
@@ -29,4 +29,8 @@ public class IoTDBSQLException extends SQLException {
super(reason);
}
+ public IoTDBSQLException(Throwable cause) {
+ super(cause);
+ }
+
}
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java
index 704bc5ff9f9fc6d329a72f9c2335fde232f66806..b728831f454eee65ab6ccd463691f4b6c345a82d 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java
@@ -30,18 +30,11 @@ import java.time.ZoneId;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.iotdb.service.rpc.thrift.TSCancelOperationReq;
-import org.apache.iotdb.service.rpc.thrift.TSCancelOperationResp;
-import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq;
-import org.apache.iotdb.service.rpc.thrift.TSCloseOperationResp;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementResp;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp;
-import org.apache.iotdb.service.rpc.thrift.TSIService;
-import org.apache.iotdb.service.rpc.thrift.TSOperationHandle;
-import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
+
+import org.apache.iotdb.rpc.IoTDBRPCException;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusType;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.apache.thrift.TException;
public class IoTDBStatement implements Statement {
@@ -49,6 +42,7 @@ public class IoTDBStatement implements Statement {
private static final String SHOW_TIMESERIES_COMMAND_LOWERCASE = "show timeseries";
private static final String SHOW_STORAGE_GROUP_COMMAND_LOWERCASE = "show storage group";
private static final String METHOD_NOT_SUPPORTED_STRING = "Method not supported";
+
ZoneId zoneId;
private ResultSet resultSet = null;
private IoTDBConnection connection;
@@ -130,8 +124,8 @@ public class IoTDBStatement implements Statement {
try {
if (operationHandle != null) {
TSCancelOperationReq closeReq = new TSCancelOperationReq(operationHandle);
- TSCancelOperationResp closeResp = client.cancelOperation(closeReq);
- Utils.verifySuccess(closeResp.getStatus());
+ TSRPCResp closeResp = client.cancelOperation(closeReq);
+ RpcUtils.verifySuccess(closeResp.getStatus());
}
} catch (Exception e) {
throw new SQLException("Error occurs when canceling statement.", e);
@@ -157,8 +151,8 @@ public class IoTDBStatement implements Statement {
if (operationHandle != null) {
TSCloseOperationReq closeReq = new TSCloseOperationReq(operationHandle, -1);
closeReq.setStmtId(stmtId);
- TSCloseOperationResp closeResp = client.closeOperation(closeReq);
- Utils.verifySuccess(closeResp.getStatus());
+ TSRPCResp closeResp = client.closeOperation(closeReq);
+ RpcUtils.verifySuccess(closeResp.getStatus());
}
} catch (Exception e) {
throw new SQLException("Error occurs when closing statement.", e);
@@ -246,7 +240,11 @@ public class IoTDBStatement implements Statement {
TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionHandle, sql);
TSExecuteStatementResp execResp = client.executeStatement(execReq);
operationHandle = execResp.getOperationHandle();
- Utils.verifySuccess(execResp.getStatus());
+ try {
+ RpcUtils.verifySuccess(execResp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
if (execResp.getOperationHandle().hasResultSet) {
IoTDBQueryResultSet resSet = new IoTDBQueryResultSet(this,
execResp.getColumns(), client,
@@ -288,7 +286,7 @@ public class IoTDBStatement implements Statement {
TSExecuteBatchStatementReq execReq = new TSExecuteBatchStatementReq(sessionHandle,
batchSQLList);
TSExecuteBatchStatementResp execResp = client.executeBatchStatement(execReq);
- if (execResp.getStatus().statusCode == TS_StatusCode.SUCCESS_STATUS) {
+ if (execResp.getStatus().getStatusType().getCode() == TSStatusType.SUCCESS_STATUS.getStatusCode()) {
if (execResp.getResult() == null) {
return new int[0];
} else {
@@ -303,7 +301,7 @@ public class IoTDBStatement implements Statement {
} else {
BatchUpdateException exception;
if (execResp.getResult() == null) {
- exception = new BatchUpdateException(execResp.getStatus().errorMessage, new int[0]);
+ exception = new BatchUpdateException(execResp.getStatus().getStatusType().getMessage(), new int[0]);
} else {
List result = execResp.getResult();
int len = result.size();
@@ -311,7 +309,7 @@ public class IoTDBStatement implements Statement {
for (int i = 0; i < len; i++) {
updateArray[i] = result.get(i);
}
- exception = new BatchUpdateException(execResp.getStatus().errorMessage, updateArray);
+ exception = new BatchUpdateException(execResp.getStatus().getStatusType().getMessage(), updateArray);
}
throw exception;
}
@@ -346,7 +344,11 @@ public class IoTDBStatement implements Statement {
TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionHandle, sql);
TSExecuteStatementResp execResp = client.executeQueryStatement(execReq);
operationHandle = execResp.getOperationHandle();
- Utils.verifySuccess(execResp.getStatus());
+ try {
+ RpcUtils.verifySuccess(execResp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
IoTDBQueryResultSet resSet = new IoTDBQueryResultSet(this, execResp.getColumns(), client,
operationHandle, sql, execResp.getOperationType(), execResp.getDataTypeList(),
queryId.getAndIncrement());
@@ -399,7 +401,11 @@ public class IoTDBStatement implements Statement {
TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionHandle, sql);
TSExecuteStatementResp execResp = client.executeUpdateStatement(execReq);
operationHandle = execResp.getOperationHandle();
- Utils.verifySuccess(execResp.getStatus());
+ try {
+ RpcUtils.verifySuccess(execResp.getStatus());
+ } catch (IoTDBRPCException e) {
+ throw new IoTDBSQLException(e.getMessage());
+ }
return 0;
}
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java
index f00402c71ce6496ba14edb2ad1262bba03d8b927..9ca73b268c05ea1050c64d46869b2f9a4f31ed94 100644
--- a/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java
+++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java
@@ -26,8 +26,6 @@ import java.util.regex.Pattern;
import org.apache.iotdb.service.rpc.thrift.TSDataValue;
import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet;
import org.apache.iotdb.service.rpc.thrift.TSRowRecord;
-import org.apache.iotdb.service.rpc.thrift.TS_Status;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Field;
@@ -39,18 +37,11 @@ import org.apache.iotdb.tsfile.utils.Binary;
*/
public class Utils {
- /**
- * Private constructor of Utils Class.
- */
- private Utils(){
- throw new IllegalAccessError("Utility class");
- }
-
/**
* Parse JDBC connection URL The only supported format of the URL is:
* jdbc:iotdb://localhost:6667/.
*/
- public static IoTDBConnectionParams parseUrl(String url, Properties info)
+ static IoTDBConnectionParams parseUrl(String url, Properties info)
throws IoTDBURLException {
IoTDBConnectionParams params = new IoTDBConnectionParams(url);
if (url.trim().equalsIgnoreCase(Config.IOTDB_URL_PREFIX)) {
@@ -79,24 +70,13 @@ public class Utils {
return params;
}
- /**
- * verify success.
- *
- * @param status -status
- */
- public static void verifySuccess(TS_Status status) throws IoTDBSQLException {
- if (status.getStatusCode() != TS_StatusCode.SUCCESS_STATUS) {
- throw new IoTDBSQLException(status.errorMessage);
- }
- }
-
/**
* convert row records.
*
* @param tsQueryDataSet -query data set
* @return -list of row record
*/
- public static List convertRowRecords(TSQueryDataSet tsQueryDataSet) {
+ static List convertRowRecords(TSQueryDataSet tsQueryDataSet) {
List records = new ArrayList<>();
for (TSRowRecord ts : tsQueryDataSet.getRecords()) {
RowRecord r = new RowRecord(ts.getTimestamp());
@@ -151,4 +131,5 @@ public class Utils {
dataType));
}
}
+
}
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java
index 912019d3c07d601c579457a990b54f2df52c8d39..09505f44d1348ddbe3046b4fc17b3fc5de2717fb 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java
@@ -29,12 +29,9 @@ import java.sql.Statement;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementResp;
-import org.apache.iotdb.service.rpc.thrift.TSIService;
-import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle;
-import org.apache.iotdb.service.rpc.thrift.TS_Status;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
+
+import org.apache.iotdb.rpc.TSStatusType;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.apache.thrift.TException;
import org.junit.After;
import org.junit.Before;
@@ -50,8 +47,10 @@ public class BatchTest {
private TSIService.Iface client;
@Mock
private TS_SessionHandle sessHandle;
- private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS);
- private TS_Status Status_ERROR = new TS_Status(TS_StatusCode.ERROR_STATUS);
+ private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), "");
+ private TS_StatusType errorStatus = new TS_StatusType(TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(), "");
+ private TS_Status Status_SUCCESS = new TS_Status(successStatus);
+ private TS_Status Status_ERROR = new TS_Status(errorStatus);
private TSExecuteBatchStatementResp resp;
private ZoneId zoneID = ZoneId.systemDefault();
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java
index e374064ef2126c22c771bd002419ccbd83d6f0c1..5012e5ed05088777116d5cc612f4e68085d81042 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java
@@ -24,13 +24,9 @@ import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.service.rpc.thrift.ServerProperties;
-import org.apache.iotdb.service.rpc.thrift.TSGetTimeZoneResp;
-import org.apache.iotdb.service.rpc.thrift.TSIService;
-import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq;
-import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneResp;
-import org.apache.iotdb.service.rpc.thrift.TS_Status;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
+
+import org.apache.iotdb.rpc.TSStatusType;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.apache.thrift.TException;
import org.junit.After;
import org.junit.Before;
@@ -44,7 +40,8 @@ public class IoTDBConnectionTest {
private TSIService.Iface client;
private IoTDBConnection connection = new IoTDBConnection();
- private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS);
+ private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), "");
+ private TS_Status Status_SUCCESS = new TS_Status(successStatus);
@Before
public void setUp() throws Exception {
@@ -59,7 +56,7 @@ public class IoTDBConnectionTest {
public void testSetTimeZone() throws TException, IoTDBSQLException {
String timeZone = "Asia/Shanghai";
when(client.setTimeZone(any(TSSetTimeZoneReq.class)))
- .thenReturn(new TSSetTimeZoneResp(Status_SUCCESS));
+ .thenReturn(new TSRPCResp(Status_SUCCESS));
connection.client = client;
connection.setTimeZone(timeZone);
assertEquals(connection.getTimeZone(), timeZone);
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java
index 1f5ac2597bf190bfa2b15dcef189d0bbeb87d031..2fdb40667ae688d2f61ba73e418edfcd3eb3ffb7 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java
@@ -30,11 +30,9 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq;
-import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp;
-import org.apache.iotdb.service.rpc.thrift.TSIService;
-import org.apache.iotdb.service.rpc.thrift.TS_Status;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
+
+import org.apache.iotdb.rpc.TSStatusType;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -66,7 +64,8 @@ public class IoTDBDatabaseMetadataTest {
@Mock
private TSFetchMetadataResp fetchMetadataResp;
- private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS);
+ private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), "");
+ private TS_Status Status_SUCCESS = new TS_Status(successStatus);
private DatabaseMetaData databaseMetaData;
@@ -96,8 +95,7 @@ public class IoTDBDatabaseMetadataTest {
String standard =
"Column,\n" + "root.vehicle.d0.s0,\n" + "root.vehicle.d0.s1,\n" + "root.vehicle.d0.s2,\n";
- try {
- ResultSet resultSet = databaseMetaData.getColumns(Constant.CATALOG_COLUMN, "root", null, null);
+ try (ResultSet resultSet = databaseMetaData.getColumns(Constant.CATALOG_COLUMN, "root", null, null)) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
int colCount = resultSetMetaData.getColumnCount();
StringBuilder resultStr = new StringBuilder();
@@ -129,9 +127,8 @@ public class IoTDBDatabaseMetadataTest {
when(fetchMetadataResp.getColumnsList()).thenReturn(columnList);
String standard = "Column,\n" + "root.vehicle.d0,\n";
- try {
- ResultSet resultSet = databaseMetaData
- .getColumns(Constant.CATALOG_DEVICE, "vehicle", null, null);
+ try (ResultSet resultSet = databaseMetaData
+ .getColumns(Constant.CATALOG_DEVICE, "vehicle", null, null)) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
int colCount = resultSetMetaData.getColumnCount();
StringBuilder resultStr = new StringBuilder();
@@ -189,9 +186,8 @@ public class IoTDBDatabaseMetadataTest {
+ "root.vehicle.d0.s0,root.vehicle,INT32,RLE,\n"
+ "root.vehicle.d0.s1,root.vehicle,INT64,RLE,\n"
+ "root.vehicle.d0.s2,root.vehicle,FLOAT,RLE,\n";
- try {
- ResultSet resultSet = databaseMetaData
- .getColumns(Constant.CATALOG_TIMESERIES, "root", null, null);
+ try (ResultSet resultSet = databaseMetaData
+ .getColumns(Constant.CATALOG_TIMESERIES, "root", null, null);) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
int colCount = resultSetMetaData.getColumnCount();
StringBuilder resultStr = new StringBuilder();
@@ -230,10 +226,9 @@ public class IoTDBDatabaseMetadataTest {
when(fetchMetadataResp.getShowTimeseriesList()).thenReturn(tslist);
String standard = "DataType,\n" + "INT32,\n";
- try {
- ResultSet resultSet = databaseMetaData
- .getColumns(Constant.CATALOG_TIMESERIES, "root.vehicle.d0.s0", null,
- null);
+ try (ResultSet resultSet = databaseMetaData
+ .getColumns(Constant.CATALOG_TIMESERIES, "root.vehicle.d0.s0", null,
+ null)) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
StringBuilder resultStr = new StringBuilder();
resultStr.append(resultSetMetaData.getColumnName(3)).append(",\n");
@@ -259,9 +254,8 @@ public class IoTDBDatabaseMetadataTest {
when(fetchMetadataResp.getShowStorageGroups()).thenReturn(sgSet);
String standard = "Storage Group,\n" + "root.vehicle,\n";
- try {
- ResultSet resultSet = databaseMetaData
- .getColumns(Constant.CATALOG_STORAGE_GROUP, null, null, null);
+ try (ResultSet resultSet = databaseMetaData
+ .getColumns(Constant.CATALOG_STORAGE_GROUP, null, null, null)) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
int colCount = resultSetMetaData.getColumnCount();
StringBuilder resultStr = new StringBuilder();
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java
index ca9928efc738d45f2d04dd7b2c38df4ac375737e..f8c2a946ab3cf0481912dbf218c06a1632eb4297 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java
@@ -26,14 +26,10 @@ import static org.mockito.Mockito.when;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.ZoneId;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp;
-import org.apache.iotdb.service.rpc.thrift.TSGetOperationStatusResp;
+
+import org.apache.iotdb.rpc.TSStatusType;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.apache.iotdb.service.rpc.thrift.TSIService.Iface;
-import org.apache.iotdb.service.rpc.thrift.TSOperationHandle;
-import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle;
-import org.apache.iotdb.service.rpc.thrift.TS_Status;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
@@ -45,7 +41,7 @@ public class IoTDBPreparedStatementTest {
@Mock
TSExecuteStatementResp execStatementResp;
@Mock
- TSGetOperationStatusResp getOperationStatusResp;
+ TSRPCResp getOperationStatusResp;
private ZoneId zoneId = ZoneId.systemDefault();
@Mock
private IoTDBConnection connection;
@@ -53,7 +49,8 @@ public class IoTDBPreparedStatementTest {
private Iface client;
@Mock
private TS_SessionHandle sessHandle;
- private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS);
+ private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), "");
+ private TS_Status Status_SUCCESS = new TS_Status(successStatus);
@Mock
private TSOperationHandle tOperationHandle;
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java
index 353cea5bcbe7a636f2c0dd433c09ff74172352a4..f0c13f692046280e199da2cb54df9a3064ed547e 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java
@@ -26,25 +26,16 @@ import static org.mockito.Mockito.when;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.service.rpc.thrift.TSDataValue;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq;
-import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp;
-import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq;
-import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp;
-import org.apache.iotdb.service.rpc.thrift.TSFetchResultsReq;
-import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp;
-import org.apache.iotdb.service.rpc.thrift.TSIService;
-import org.apache.iotdb.service.rpc.thrift.TSOperationHandle;
-import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet;
-import org.apache.iotdb.service.rpc.thrift.TSRowRecord;
-import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle;
-import org.apache.iotdb.service.rpc.thrift.TS_Status;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
+
+import org.apache.iotdb.rpc.TSStatusType;
+import org.apache.iotdb.service.rpc.thrift.*;
+
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.junit.Assert;
import org.junit.Before;
@@ -117,7 +108,8 @@ public class IoTDBQueryResultSetTest {
@Mock
private TSFetchResultsResp fetchResultsResp;
- private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS);
+ private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), "");
+ private TS_Status Status_SUCCESS = new TS_Status(successStatus);
private ZoneId zoneID = ZoneId.systemDefault();
@Before
@@ -137,6 +129,10 @@ public class IoTDBQueryResultSetTest {
when(client.fetchResults(any(TSFetchResultsReq.class))).thenReturn(fetchResultsResp);
when(fetchResultsResp.getStatus()).thenReturn(Status_SUCCESS);
+
+ TSRPCResp closeResp = new TSRPCResp();
+ closeResp.setStatus(Status_SUCCESS);
+ when(client.closeOperation(any(TSCloseOperationReq.class))).thenReturn(closeResp);
}
@SuppressWarnings("resource")
@@ -181,50 +177,51 @@ public class IoTDBQueryResultSetTest {
when(fetchResultsResp.getQueryDataSet()).thenReturn(tsQueryDataSet);
if (hasResultSet) {
- ResultSet resultSet = statement.getResultSet();
- // check columnInfoMap
- Assert.assertEquals(resultSet.findColumn("Time"), 1);
- Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s2"), 2);
- Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s1"), 3);
- Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s0"), 4);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ // check columnInfoMap
+ Assert.assertEquals(resultSet.findColumn("Time"), 1);
+ Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s2"), 2);
+ Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s1"), 3);
+ Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s0"), 4);
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- // check columnInfoList
- Assert.assertEquals(resultSetMetaData.getColumnName(1), "Time");
- Assert.assertEquals(resultSetMetaData.getColumnName(2), "root.vehicle.d0.s2");
- Assert.assertEquals(resultSetMetaData.getColumnName(3), "root.vehicle.d0.s1");
- Assert.assertEquals(resultSetMetaData.getColumnName(4), "root.vehicle.d0.s0");
- Assert.assertEquals(resultSetMetaData.getColumnName(5), "root.vehicle.d0.s2");
- // check columnTypeList
- Assert.assertEquals(resultSetMetaData.getColumnType(1), Types.TIMESTAMP);
- Assert.assertEquals(resultSetMetaData.getColumnType(2), Types.FLOAT);
- Assert.assertEquals(resultSetMetaData.getColumnType(3), Types.BIGINT);
- Assert.assertEquals(resultSetMetaData.getColumnType(4), Types.INTEGER);
- Assert.assertEquals(resultSetMetaData.getColumnType(5), Types.FLOAT);
- // check fetched result
- int colCount = resultSetMetaData.getColumnCount();
- StringBuilder resultStr = new StringBuilder();
- for (int i = 1; i < colCount + 1; i++) { // meta title
- resultStr.append(resultSetMetaData.getColumnName(i)).append(",");
- }
- resultStr.append("\n");
- while (resultSet.next()) { // data
- for (int i = 1; i <= colCount; i++) {
- resultStr.append(resultSet.getString(i)).append(",");
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ // check columnInfoList
+ Assert.assertEquals(resultSetMetaData.getColumnName(1), "Time");
+ Assert.assertEquals(resultSetMetaData.getColumnName(2), "root.vehicle.d0.s2");
+ Assert.assertEquals(resultSetMetaData.getColumnName(3), "root.vehicle.d0.s1");
+ Assert.assertEquals(resultSetMetaData.getColumnName(4), "root.vehicle.d0.s0");
+ Assert.assertEquals(resultSetMetaData.getColumnName(5), "root.vehicle.d0.s2");
+ // check columnTypeList
+ Assert.assertEquals(resultSetMetaData.getColumnType(1), Types.TIMESTAMP);
+ Assert.assertEquals(resultSetMetaData.getColumnType(2), Types.FLOAT);
+ Assert.assertEquals(resultSetMetaData.getColumnType(3), Types.BIGINT);
+ Assert.assertEquals(resultSetMetaData.getColumnType(4), Types.INTEGER);
+ Assert.assertEquals(resultSetMetaData.getColumnType(5), Types.FLOAT);
+ // check fetched result
+ int colCount = resultSetMetaData.getColumnCount();
+ StringBuilder resultStr = new StringBuilder();
+ for (int i = 1; i < colCount + 1; i++) { // meta title
+ resultStr.append(resultSetMetaData.getColumnName(i)).append(",");
}
resultStr.append("\n");
+ while (resultSet.next()) { // data
+ for (int i = 1; i <= colCount; i++) {
+ resultStr.append(resultSet.getString(i)).append(",");
+ }
+ resultStr.append("\n");
- fetchResultsResp.hasResultSet = false; // at the second time to fetch
+ fetchResultsResp.hasResultSet = false; // at the second time to fetch
+ }
+ String standard =
+ "Time,root.vehicle.d0.s2,root.vehicle.d0.s1,root.vehicle.d0.s0,root.vehicle.d0.s2,\n"
+ + "2,2.22,40000,null,2.22,\n" + "3,3.33,null,null,3.33,\n"
+ + "4,4.44,null,null,4.44,\n"
+ + "50,null,50000,null,null,\n" + "100,null,199,null,null,\n"
+ + "101,null,199,null,null,\n"
+ + "103,null,199,null,null,\n" + "105,11.11,199,33333,11.11,\n"
+ + "1000,1000.11,55555,22222,1000.11,\n";
+ Assert.assertEquals(resultStr.toString(), standard);
}
- String standard =
- "Time,root.vehicle.d0.s2,root.vehicle.d0.s1,root.vehicle.d0.s0,root.vehicle.d0.s2,\n"
- + "2,2.22,40000,null,2.22,\n" + "3,3.33,null,null,3.33,\n"
- + "4,4.44,null,null,4.44,\n"
- + "50,null,50000,null,null,\n" + "100,null,199,null,null,\n"
- + "101,null,199,null,null,\n"
- + "103,null,199,null,null,\n" + "105,11.11,199,33333,11.11,\n"
- + "1000,1000.11,55555,22222,1000.11,\n";
- Assert.assertEquals(resultStr.toString(), standard);
}
}
@@ -276,14 +273,12 @@ public class IoTDBQueryResultSetTest {
} else {
if (i == 0) {
value.setFloat_val((float) item[3 * i + 3]);
- value.setType(item[3 * i + 2].toString());
} else if (i == 1) {
value.setLong_val((long) item[3 * i + 3]);
- value.setType(item[3 * i + 2].toString());
} else {
value.setInt_val((int) item[3 * i + 3]);
- value.setType(item[3 * i + 2].toString());
}
+ value.setType(item[3 * i + 2].toString());
}
values.add(value);
}
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java
index d8632fc92590b6100029360d45af16847151851c..44fd2f4d224680c05068fd394b9114e9f14b01e4 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.jdbc;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.when;
@@ -30,12 +31,10 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq;
-import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp;
+
+import org.apache.iotdb.rpc.TSStatusType;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.apache.iotdb.service.rpc.thrift.TSIService.Iface;
-import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle;
-import org.apache.iotdb.service.rpc.thrift.TS_Status;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
import org.apache.thrift.TException;
import org.junit.After;
import org.junit.Assert;
@@ -58,7 +57,8 @@ public class IoTDBStatementTest {
@Mock
private TSFetchMetadataResp fetchMetadataResp;
- private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS);
+ private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), "");
+ private TS_Status Status_SUCCESS = new TS_Status(successStatus);
private ZoneId zoneID = ZoneId.systemDefault();
@Before
@@ -116,9 +116,8 @@ public class IoTDBStatementTest {
+ "root.vehicle.d0.s2,root.vehicle,FLOAT,RLE,\n";
when(fetchMetadataResp.getShowTimeseriesList()).thenReturn(tslist);
boolean res = stmt.execute("show timeseries root");
- assertEquals(res, true);
- try {
- ResultSet resultSet = stmt.getResultSet();
+ assertTrue(res);
+ try (ResultSet resultSet = stmt.getResultSet()) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
int colCount = resultSetMetaData.getColumnCount();
StringBuilder resultStr = new StringBuilder();
@@ -147,9 +146,8 @@ public class IoTDBStatementTest {
when(fetchMetadataResp.getShowStorageGroups()).thenReturn(sgSet);
String standard = "Storage Group,\nroot.vehicle,\n";
boolean res = stmt.execute("show storage group");
- assertEquals(res, true);
- try {
- ResultSet resultSet = stmt.getResultSet();
+ assertTrue(res);
+ try (ResultSet resultSet = stmt.getResultSet()) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
int colCount = resultSetMetaData.getColumnCount();
StringBuilder resultStr = new StringBuilder();
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java
index ff6650dce7d328cd97376da20291c374f253f5eb..bc3a766823363e5554b4e70c4175e17ac044baaa 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java
@@ -26,11 +26,15 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusType;
import org.apache.iotdb.service.rpc.thrift.TSDataValue;
import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet;
import org.apache.iotdb.service.rpc.thrift.TSRowRecord;
import org.apache.iotdb.service.rpc.thrift.TS_Status;
-import org.apache.iotdb.service.rpc.thrift.TS_StatusCode;
+
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.service.rpc.thrift.*;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Field;
import org.apache.iotdb.tsfile.read.common.RowRecord;
@@ -69,13 +73,15 @@ public class UtilsTest {
@Test
public void testVerifySuccess() {
try {
- Utils.verifySuccess(new TS_Status(TS_StatusCode.SUCCESS_STATUS));
+ TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), "");
+ RpcUtils.verifySuccess(new TS_Status(successStatus));
} catch (Exception e) {
fail();
}
try {
- Utils.verifySuccess(new TS_Status(TS_StatusCode.ERROR_STATUS));
+ TS_StatusType errorStatus = new TS_StatusType(TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(), "");
+ RpcUtils.verifySuccess(new TS_Status(errorStatus));
} catch (Exception e) {
return;
}
@@ -114,23 +120,18 @@ public class UtilsTest {
} else {
if (i == 0) {
value.setBool_val((boolean) item[3 * i + 3]);
- value.setType(((TSDataType) item[3 * i + 2]).toString());
} else if (i == 1) {
value.setInt_val((int) item[3 * i + 3]);
- value.setType(((TSDataType) item[3 * i + 2]).toString());
} else if (i == 2) {
value.setLong_val((long) item[3 * i + 3]);
- value.setType(((TSDataType) item[3 * i + 2]).toString());
} else if (i == 3) {
value.setFloat_val((float) item[3 * i + 3]);
- value.setType(((TSDataType) item[3 * i + 2]).toString());
} else if (i == 4) {
value.setDouble_val((double) item[3 * i + 3]);
- value.setType(((TSDataType) item[3 * i + 2]).toString());
} else {
value.setBinary_val(ByteBuffer.wrap(((String) item[3 * i + 3]).getBytes()));
- value.setType(((TSDataType) item[3 * i + 2]).toString());
}
+ value.setType(item[3 * i + 2].toString());
}
values.add(value);
}
diff --git a/pom.xml b/pom.xml
index 0073930ee1e04c56d0cdcde3a5b75a836146d7a8..77a2219bdc13a27f075ab62b7d94d5a76b407790 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,6 +44,19 @@
https://github.com/apache/incubator-iotdb
rel/0.8
+
+ tsfile
+ service-rpc
+ jdbc
+ session
+ client
+ server
+ example
+ grafana
+ spark-tsfile
+
+ distribution
+
1.8
@@ -377,18 +390,6 @@
test
-
- tsfile
- service-rpc
- jdbc
- client
- server
- example
- grafana
- spark-tsfile
-
- distribution
-
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index e8705739e77d2febb5fffd90f0b8a4539582fc95..dddf1a2dc47d859991fc31d3d75a2e5b8886afc1 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -27,6 +27,8 @@ rpc_port=6667
rpc_thrift_compression_enable=false
+rpc_max_concurrent_client_num=65535
+
####################
### Dynamic Parameter Adapter Configuration
####################
@@ -157,7 +159,52 @@ concurrent_flush_thread=0
# whether take over the memory management by IoTDB rather than JVM when serializing memtable as bytes in memory
# (i.e., whether use ChunkBufferPool), value true, false
-chunk_buffer_pool_enable = false
+chunk_buffer_pool_enable=false
+
+####################
+### Merge Configurations
+####################
+
+# How many thread will be set up to perform merge main tasks, 1 by default.
+# Set to 1 when less than or equal to 0.
+merge_thread_num=1
+
+# How many thread will be set up to perform merge chunk sub-tasks, 4 by default.
+# Set to 1 when less than or equal to 0.
+merge_chunk_subthread_num=4
+
+# If one merge file selection runs for more than this time, it will be ended and its current
+# selection will be used as final selection. Unit: millis.
+# When < 0, it means time is unbounded.
+merge_fileSelection_time_budget=30000
+
+# How much memory may be used in ONE merge task (in byte), 20% of maximum JVM memory by default.
+# This is only a rough estimation, starting from a relatively small value to avoid OOM.
+# Each new merge thread may take such memory, so merge_thread_num * merge_memory_budget is the
+# total memory estimation of merge.
+# merge_memory_budget=2147483648
+
+# When set to true, if some crashed merges are detected during system rebooting, such merges will
+# be continued, otherwise, the unfinished parts of such merges will not be continued while the
+# finished parts still remains as they are.
+# If you are feeling the rebooting is too slow, set this to false, false by default
+continue_merge_after_reboot=false
+
+# A global merge will be performed each such interval, that is, each storage group will be merged
+# (if proper merge candidates can be found). Unit: second, default: 1hours.
+# When less than or equal to 0, timed merge is disabled.
+merge_interval_sec=3600
+
+# When set to true, all merges becomes full merge (the whole SeqFiles are re-written despite how
+# much they are overflowed). This may increase merge overhead depending on how much the SeqFiles
+# are overflowed.
+force_full_merge=false
+
+# During a merge, if a chunk with less number of points than this parameter, the chunk will be
+# merged with its succeeding chunks even if it is not overflowed, until the merged chunks reach
+# this threshold and the new chunk will be flushed.
+# When less than 0, this mechanism is disabled.
+chunk_merge_point_threshold=20480
####################
### Metadata Cache Configuration
diff --git a/server/src/assembly/resources/tools/print-tsfile-resource-files.bat b/server/src/assembly/resources/tools/print-tsfile-resource-files.bat
new file mode 100644
index 0000000000000000000000000000000000000000..bbc1acaf13b5c48378e4c1e99d8e02ad08e27599
--- /dev/null
+++ b/server/src/assembly/resources/tools/print-tsfile-resource-files.bat
@@ -0,0 +1,102 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+echo ````````````````````````
+echo Starting Printing the TsFileResources
+echo ````````````````````````
+
+PATH %PATH%;%JAVA_HOME%\bin\
+set "FULL_VERSION="
+set "MAJOR_VERSION="
+set "MINOR_VERSION="
+
+
+for /f tokens^=2-5^ delims^=.-_+^" %%j in ('java -fullversion 2^>^&1') do (
+ set "FULL_VERSION=%%j-%%k-%%l-%%m"
+ IF "%%j" == "1" (
+ set "MAJOR_VERSION=%%k"
+ set "MINOR_VERSION=%%l"
+ ) else (
+ set "MAJOR_VERSION=%%j"
+ set "MINOR_VERSION=%%k"
+ )
+)
+
+set JAVA_VERSION=%MAJOR_VERSION%
+
+IF NOT %JAVA_VERSION% == 8 (
+ IF NOT %JAVA_VERSION% == 11 (
+ echo IoTDB only supports jdk8 or jdk11, please check your java version.
+ goto finally
+ )
+)
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%cd%
+popd
+
+set IOTDB_CONF=%IOTDB_HOME%\conf
+set IOTDB_LOGS=%IOTDB_HOME%\logs
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.TsFileResourcePrinter
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM -----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -Dlogback.configurationFile="%IOTDB_CONF%\logback.xml"^
+ -DIOTDB_HOME="%IOTDB_HOME%"^
+ -DTSFILE_HOME="%IOTDB_HOME%"^
+ -DIOTDB_CONF="%IOTDB_CONF%"
+
+@REM ***** CLASSPATH library setting *****
+@REM Ensure that any user defined CLASSPATH variables are not used on startup
+set CLASSPATH="%IOTDB_HOME%\lib"
+
+REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
+set CLASSPATH=%CLASSPATH%
+goto okClasspath
+
+:append
+set CLASSPATH=%CLASSPATH%;%1
+goto :eof
+
+REM -----------------------------------------------------------------------------
+:okClasspath
+
+rem echo CLASSPATH: %CLASSPATH%
+set IOTDB_DATA=%IOTDB_HOME%\data
+
+goto finally
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+@REM -----------------------------------------------------------------------------
+:finally
+
+pause
+
+ENDLOCAL
diff --git a/server/src/assembly/resources/tools/print-tsfile-resource-files.sh b/server/src/assembly/resources/tools/print-tsfile-resource-files.sh
new file mode 100644
index 0000000000000000000000000000000000000000..7de56338866144af96dbec10601ada9e26c92b3d
--- /dev/null
+++ b/server/src/assembly/resources/tools/print-tsfile-resource-files.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+echo ---------------------
+echo Starting Printing the TsFileResources
+echo ---------------------
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="`dirname "$0"`/.."
+fi
+
+IOTDB_CONF=${IOTDB_HOME}/conf
+
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr
+ exit 1;
+fi
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}":"$f
+done
+classname=org.apache.iotdb.db.tools.TsFileResourcePrinter
+
+
+launch_service()
+{
+ class="$1"
+ iotdb_parms="-Dlogback.configurationFile=${IOTDB_CONF}/logback.xml"
+ iotdb_parms="$iotdb_parms -DIOTDB_HOME=${IOTDB_HOME}"
+ iotdb_parms="$iotdb_parms -DTSFILE_HOME=${IOTDB_HOME}"
+ iotdb_parms="$iotdb_parms -DIOTDB_CONF=${IOTDB_CONF}"
+# iotdb_parms="$iotdb_parms -Dname=iotdb\.IoTDB"
+ exec "$JAVA" $iotdb_parms $IOTDB_JMX_OPTS $iotdb_parms -cp "$CLASSPATH" "$class" "$WALPATH"
+ return $?
+}
+
+# Start up the service
+#launch_service "$classname"
+
+if [ ! -d ${IOTDB_HOME}/data/wal ]; then
+ echo "Can't find wal directory."
+ exit 1;
+else
+ WALPATH=${IOTDB_HOME}/data/wal
+ launch_service "$classname"
+fi
+
+exit $?
diff --git a/server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g b/server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g
index e53c4fc41abe637bdd12a5aa75a4901fd55a0862..aa5bc28d0737750fe8e546f3c05c61bbfd5018c3 100644
--- a/server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g
+++ b/server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g
@@ -119,7 +119,7 @@ ArrayList errors = new ArrayList();
private static HashMap xlateMap;
static {
- //this is used to support auto completion in CLI
+ //this is used to support auto completion in Client
xlateMap = new HashMap();
// Keywords
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 202b0c9a73e2474a48706ee231fac7a261702170..0c92dee11fb8c95590f45022c6ca67301325cb21 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -23,6 +23,7 @@ import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.iotdb.db.engine.merge.selector.MergeFileStrategy;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.service.TSServiceImpl;
import org.slf4j.Logger;
@@ -48,6 +49,11 @@ public class IoTDBConfig {
*/
private int rpcPort = 6667;
+ /**
+ * Max concurrent client number
+ */
+ private int maxConcurrentClientNum = 65535;
+
/**
* Memory allocated for the read process
*/
@@ -133,12 +139,6 @@ public class IoTDBConfig {
*/
private int maxMemtableNumber = 20;
- /**
- * The maximum concurrent thread number for merging. When the value <=0 or > CPU core number, use
- * the CPU core number.
- */
- private int mergeConcurrentThreads = Runtime.getRuntime().availableProcessors();
-
/**
* The amount of data that is read every time when IoTDB merges data.
*/
@@ -248,6 +248,57 @@ public class IoTDBConfig {
*/
private boolean chunkBufferPoolEnable = false;
+ /**
+ * How much memory (in byte) can be used by a single merge task.
+ */
+ private long mergeMemoryBudget = (long) (Runtime.getRuntime().maxMemory() * 0.2);
+
+ /**
+ * How many threads will be set up to perform main merge tasks.
+ */
+ private int mergeThreadNum = 1;
+
+ /**
+ * How many threads will be set up to perform merge chunk sub-tasks.
+ */
+ private int mergeChunkSubThreadNum = 4;
+
+ /**
+ * If one merge file selection runs for more than this time, it will be ended and its current
+ * selection will be used as final selection. Unit: millis.
+ * When < 0, it means time is unbounded.
+ */
+ private long mergeFileSelectionTimeBudget = 30 * 1000;
+
+ /**
+ * When set to true, if some crashed merges are detected during system rebooting, such merges will
+ * be continued, otherwise, the unfinished parts of such merges will not be continued while the
+ * finished parts still remain as they are.
+ */
+ private boolean continueMergeAfterReboot = true;
+
+ /**
+ * A global merge will be performed each such interval, that is, each storage group will be merged
+ * (if proper merge candidates can be found). Unit: second.
+ */
+ private long mergeIntervalSec = 2 * 3600L;
+
+ /**
+ * When set to true, all merges becomes full merge (the whole SeqFiles are re-written despite how
+ * much they are overflowed). This may increase merge overhead depending on how much the SeqFiles
+ * are overflowed.
+ */
+ private boolean forceFullMerge = false;
+
+ /**
+ * During a merge, if a chunk with less number of chunks than this parameter, the chunk will be
+ * merged with its succeeding chunks even if it is not overflowed, until the merged chunks reach
+ * this threshold and the new chunk will be flushed.
+ */
+ private int chunkMergePointThreshold = 20480;
+
+ private MergeFileStrategy mergeFileStrategy = MergeFileStrategy.MAX_SERIES_NUM;
+
public IoTDBConfig() {
// empty constructor
}
@@ -410,14 +461,6 @@ public class IoTDBConfig {
this.indexFileDir = indexFileDir;
}
- public int getMergeConcurrentThreads() {
- return mergeConcurrentThreads;
- }
-
- void setMergeConcurrentThreads(int mergeConcurrentThreads) {
- this.mergeConcurrentThreads = mergeConcurrentThreads;
- }
-
public int getFetchSize() {
return fetchSize;
}
@@ -470,6 +513,14 @@ public class IoTDBConfig {
this.enableStatMonitor = enableStatMonitor;
}
+ public int getMaxConcurrentClientNum() {
+ return maxConcurrentClientNum;
+ }
+
+ public void setMaxConcurrentClientNum(int maxConcurrentClientNum) {
+ this.maxConcurrentClientNum = maxConcurrentClientNum;
+ }
+
public int getStatMonitorDetectFreqSec() {
return statMonitorDetectFreqSec;
}
@@ -582,6 +633,38 @@ public class IoTDBConfig {
this.chunkBufferPoolEnable = chunkBufferPoolEnable;
}
+ public long getMergeMemoryBudget() {
+ return mergeMemoryBudget;
+ }
+
+ public void setMergeMemoryBudget(long mergeMemoryBudget) {
+ this.mergeMemoryBudget = mergeMemoryBudget;
+ }
+
+ public int getMergeThreadNum() {
+ return mergeThreadNum;
+ }
+
+ public void setMergeThreadNum(int mergeThreadNum) {
+ this.mergeThreadNum = mergeThreadNum;
+ }
+
+ public boolean isContinueMergeAfterReboot() {
+ return continueMergeAfterReboot;
+ }
+
+ public void setContinueMergeAfterReboot(boolean continueMergeAfterReboot) {
+ this.continueMergeAfterReboot = continueMergeAfterReboot;
+ }
+
+ public long getMergeIntervalSec() {
+ return mergeIntervalSec;
+ }
+
+ public void setMergeIntervalSec(long mergeIntervalSec) {
+ this.mergeIntervalSec = mergeIntervalSec;
+ }
+
public boolean isEnableParameterAdapter() {
return enableParameterAdapter;
}
@@ -630,6 +713,22 @@ public class IoTDBConfig {
this.performanceStatMemoryInKB = performanceStatMemoryInKB;
}
+ public boolean isForceFullMerge() {
+ return forceFullMerge;
+ }
+
+ public void setForceFullMerge(boolean forceFullMerge) {
+ this.forceFullMerge = forceFullMerge;
+ }
+
+ public int getChunkMergePointThreshold() {
+ return chunkMergePointThreshold;
+ }
+
+ public void setChunkMergePointThreshold(int chunkMergePointThreshold) {
+ this.chunkMergePointThreshold = chunkMergePointThreshold;
+ }
+
public long getMemtableSizeThreshold() {
return memtableSizeThreshold;
}
@@ -637,6 +736,31 @@ public class IoTDBConfig {
public void setMemtableSizeThreshold(long memtableSizeThreshold) {
this.memtableSizeThreshold = memtableSizeThreshold;
}
+
+ public MergeFileStrategy getMergeFileStrategy() {
+ return mergeFileStrategy;
+ }
+
+ public void setMergeFileStrategy(
+ MergeFileStrategy mergeFileStrategy) {
+ this.mergeFileStrategy = mergeFileStrategy;
+ }
+
+ public int getMergeChunkSubThreadNum() {
+ return mergeChunkSubThreadNum;
+ }
+
+ public void setMergeChunkSubThreadNum(int mergeChunkSubThreadNum) {
+ this.mergeChunkSubThreadNum = mergeChunkSubThreadNum;
+ }
+
+ public long getMergeFileSelectionTimeBudget() {
+ return mergeFileSelectionTimeBudget;
+ }
+
+ public void setMergeFileSelectionTimeBudget(long mergeFileSelectionTimeBudget) {
+ this.mergeFileSelectionTimeBudget = mergeFileSelectionTimeBudget;
+ }
public boolean isRpcThriftCompressionEnable() {
return rpcThriftCompressionEnable;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 3ea16211d94998a67a78e31771466087b8aec2e6..02fe51cd4ba7c830ffacc9843de4d51e32a4890c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -26,7 +26,7 @@ public class IoTDBConstant {
public static final String ENV_FILE_NAME = "iotdb-env";
public static final String IOTDB_CONF = "IOTDB_CONF";
public static final String GLOBAL_DB_NAME = "IoTDB";
- public static final String VERSION = "0.7.0";
+ public static final String VERSION = "0.9.0-SNAPSHOT";
public static final String REMOTE_JMX_PORT_NAME = "com.sun.management.jmxremote.port";
public static final String IOTDB_LOCAL_JMX_PORT_NAME = "iotdb.jmx.local.port";
public static final String IOTDB_REMOTE_JMX_PORT_NAME = "iotdb.jmx.remote.port";
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index c0b98498320f5febbd3505e0e76a087a8c5f8e08..9a8397f18311154269e63a53f6214e1452af8a4b 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -170,14 +170,6 @@ public class IoTDBDescriptor {
conf.setMultiDirStrategyClassName(properties.getProperty("multi_dir_strategy",
conf.getMultiDirStrategyClassName()));
- conf.setMergeConcurrentThreads(Integer
- .parseInt(properties.getProperty("merge_concurrent_threads",
- Integer.toString(conf.getMergeConcurrentThreads()))));
- if (conf.getMergeConcurrentThreads() <= 0
- || conf.getMergeConcurrentThreads() > Runtime.getRuntime().availableProcessors()) {
- conf.setMergeConcurrentThreads(Runtime.getRuntime().availableProcessors());
- }
-
conf.setFetchSize(Integer.parseInt(properties.getProperty("fetch_size",
Integer.toString(conf.getFetchSize()))));
@@ -231,6 +223,25 @@ public class IoTDBDescriptor {
conf.setZoneID(ZoneId.of(tmpTimeZone.trim()));
logger.info("Time zone has been set to {}", conf.getZoneID());
+ conf.setMergeMemoryBudget(Long.parseLong(properties.getProperty("merge_memory_budget",
+ Long.toString(conf.getMergeMemoryBudget()))));
+ conf.setMergeThreadNum(Integer.parseInt(properties.getProperty("merge_thread_num",
+ Integer.toString(conf.getMergeThreadNum()))));
+ conf.setMergeChunkSubThreadNum(Integer.parseInt(properties.getProperty
+ ("merge_chunk_subthread_num",
+ Integer.toString(conf.getMergeChunkSubThreadNum()))));
+ conf.setContinueMergeAfterReboot(Boolean.parseBoolean(properties.getProperty(
+ "continue_merge_after_reboot", Boolean.toString(conf.isContinueMergeAfterReboot()))));
+ conf.setMergeFileSelectionTimeBudget(Long.parseLong(properties.getProperty
+ ("merge_fileSelection_time_budget",
+ Long.toString(conf.getMergeFileSelectionTimeBudget()))));
+ conf.setMergeIntervalSec(Long.parseLong(properties.getProperty("merge_interval_sec",
+ Long.toString(conf.getMergeIntervalSec()))));
+ conf.setForceFullMerge(Boolean.parseBoolean(properties.getProperty("force_full_merge",
+ Boolean.toString(conf.isForceFullMerge()))));
+ conf.setChunkMergePointThreshold(Integer.parseInt(properties.getProperty(
+ "chunk_merge_point_threshold", Integer.toString(conf.getChunkMergePointThreshold()))));
+
conf.setEnablePerformanceStat(Boolean
.parseBoolean(properties.getProperty("enable_performance_stat",
Boolean.toString(conf.isEnablePerformanceStat())).trim()));
@@ -241,6 +252,15 @@ public class IoTDBDescriptor {
conf.setPerformanceStatMemoryInKB(Integer
.parseInt(properties.getProperty("performance_stat_memory_in_kb",
Integer.toString(conf.getPerformanceStatMemoryInKB())).trim()));
+
+ int maxConcurrentClientNum = Integer.parseInt(properties.
+ getProperty("max_concurrent_client_num",
+ Integer.toString(conf.getMaxConcurrentClientNum()).trim()));
+ if (maxConcurrentClientNum <= 0) {
+ maxConcurrentClientNum = 65535;
+ }
+ conf.setMaxConcurrentClientNum(maxConcurrentClientNum);
+
} catch (IOException e) {
logger.warn("Cannot load config file because, use default configuration", e);
} catch (Exception e) {
diff --git a/server/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java b/server/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
index 1c7aded57638b465e1db274088a9d5a66449cbca..377023b2bbc0caa8d149c91943f61d8f474ed5bb 100644
--- a/server/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
+++ b/server/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java
@@ -253,7 +253,7 @@ public class Measurement implements MeasurementMBean, IService {
service = IoTDBThreadPoolFactory.newScheduledThreadPool(
2, ThreadName.TIME_COST_STATSTIC.getName());
}
- //we have to check again because someone may channge the value.
+ //we have to check again because someone may change the value.
isEnableStat = IoTDBDescriptor.getInstance().getConfig().isEnablePerformanceStat();
if (isEnableStat) {
consumeFuture = service.schedule(new QueueConsumerThread(), 0, TimeUnit.MILLISECONDS);
diff --git a/server/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java b/server/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java
index 6f862ba4e441b96cf9909a327354fd86253e19ac..3ff43e31334e94059885d675b0b1c58235ec0bed 100644
--- a/server/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java
+++ b/server/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java
@@ -19,8 +19,9 @@
package org.apache.iotdb.db.cost.statistic;
public enum Operation {
- EXECUTE_BATCH("EXECUTE_BATCH"),
+ EXECUTE_JDBC_BATCH("EXECUTE_JDBC_BATCH"),
EXECUTE_ONE_SQL_IN_BATCH("EXECUTE_ONE_SQL_IN_BATCH"),
+ EXECUTE_RPC_BATCH_INSERT("EXECUTE_RPC_BATCH_INSERT"),
EXECUTE_QUERY("EXECUTE_QUERY");
public String getName() {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 8b5f811c0ed6db4a991142e183ca43251f4ee713..3e36516f301641cc4334f34e09b8ffbdbc7ba71b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -37,8 +37,10 @@ import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.StorageEngineFailureException;
import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.JobFileManager;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
import org.apache.iotdb.db.utils.FilePathUtils;
@@ -148,7 +150,7 @@ public class StorageEngine implements IService {
/**
- * execute an InsertPlan on a storage group.
+ * insert an InsertPlan to a storage group.
*
* @param insertPlan physical plan of insertion
* @return true if and only if this insertion succeeds
@@ -169,6 +171,25 @@ public class StorageEngine implements IService {
return storageGroupProcessor.insert(insertPlan);
}
+ /**
+ * insert a BatchInsertPlan to a storage group
+ * @return result of each row
+ */
+ public Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws StorageEngineException {
+ StorageGroupProcessor storageGroupProcessor;
+ try {
+ storageGroupProcessor = getProcessor(batchInsertPlan.getDeviceId());
+ } catch (Exception e) {
+ logger.warn("get StorageGroupProcessor of device {} failed, because {}",
+ batchInsertPlan.getDeviceId(),
+ e.getMessage(), e);
+ throw new StorageEngineException(e);
+ }
+
+ // TODO monitor: update statistics
+ return storageGroupProcessor.insertBatch(batchInsertPlan);
+ }
+
/**
* only for unit test
*/
@@ -209,37 +230,17 @@ public class StorageEngine implements IService {
}
}
-
- /**
- * begin a query on a given deviceId. Any TsFile contains such device should not be deleted at
- * once after merge.
- *
- * @param deviceId queried deviceId
- * @return a token for the query.
- */
- public int beginQuery(String deviceId) throws StorageEngineException {
- // TODO implement it when developing the merge function
- return -1;
- }
-
- /**
- * end query on a given deviceId. If some TsFile has been merged and this query is the last query
- * using it, the TsFile can be deleted safely.
- */
- public void endQuery(String deviceId, int token) throws StorageEngineException {
- // TODO implement it when developing the merge function
- }
-
/**
* query data.
*/
- public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
+ public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context,
+ JobFileManager filePathsManager)
throws StorageEngineException {
//TODO use context.
String deviceId = seriesExpression.getSeriesPath().getDevice();
String measurementId = seriesExpression.getSeriesPath().getMeasurement();
StorageGroupProcessor storageGroupProcessor = getProcessor(deviceId);
- return storageGroupProcessor.query(deviceId, measurementId, context);
+ return storageGroupProcessor.query(deviceId, measurementId, context, filePathsManager);
}
/**
@@ -284,8 +285,13 @@ public class StorageEngine implements IService {
*
* @throws StorageEngineException StorageEngineException
*/
- public void mergeAll() throws StorageEngineException {
- // TODO
+ public void mergeAll(boolean fullMerge) throws StorageEngineException {
+ if (IoTDBDescriptor.getInstance().getConfig().isReadOnly()) {
+ throw new StorageEngineException("Current system mode is read only, does not support merge");
+ }
+ for (StorageGroupProcessor storageGroupProcessor : processorMap.values()) {
+ storageGroupProcessor.merge(fullMerge);
+ }
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 5d2342f98df8d5f6bcb63ce6e1b4f5afbc5f6897..fd0eb14a8324e44b8933d5daf49a2dd83af97d65 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
@@ -85,16 +86,17 @@ public class DeviceMetaDataCache {
/**
* get {@link ChunkMetaData}. THREAD SAFE.
*/
- public List get(String filePath, Path seriesPath)
+ public List get(TsFileResource resource, Path seriesPath)
throws IOException {
if (!cacheEnable) {
- TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
+ TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(resource);
TsDeviceMetadata deviceMetaData = TsFileMetadataUtils
- .getTsDeviceMetaData(filePath, seriesPath, fileMetaData);
+ .getTsDeviceMetaData(resource, seriesPath, fileMetaData);
return TsFileMetadataUtils.getChunkMetaDataList(seriesPath.getMeasurement(), deviceMetaData);
}
- StringBuilder builder = new StringBuilder(filePath).append(".").append(seriesPath.getDevice());
+ StringBuilder builder = new StringBuilder(resource.getFile().getPath()).append(".").append(seriesPath
+ .getDevice());
String pathDeviceStr = builder.toString();
String key = builder.append(".").append(seriesPath.getMeasurement()).toString();
Object devicePathObject = pathDeviceStr.intern();
@@ -123,9 +125,9 @@ public class DeviceMetaDataCache {
logger.debug("Cache didn't hit: the number of requests for cache is {}",
cacheRequestNum.get());
}
- TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath);
+ TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(resource);
TsDeviceMetadata deviceMetaData = TsFileMetadataUtils
- .getTsDeviceMetaData(filePath, seriesPath, fileMetaData);
+ .getTsDeviceMetaData(resource, seriesPath, fileMetaData);
// If measurement isn't included in the tsfile, empty list is returned.
if (deviceMetaData == null) {
return new ArrayList<>();
@@ -193,4 +195,10 @@ public class DeviceMetaDataCache {
private static final DeviceMetaDataCache INSTANCE = new
DeviceMetaDataCache(MEMORY_THRESHOLD_IN_B);
}
+
+ public void remove(TsFileResource resource) {
+ synchronized (lruCache) {
+ lruCache.entrySet().removeIf(e -> e.getKey().startsWith(resource.getFile().getPath()));
+ }
+ }
}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index 6b7e448b324f475e3fef91efde6fbcbccfd3dd09..d85a18d8d705491c763ff65218192e02e74e3e9d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@ -22,7 +22,9 @@ import java.io.IOException;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +41,7 @@ public class TsFileMetaDataCache {
/**
* key: Tsfile path. value: TsFileMetaData
*/
- private LRULinkedHashMap cache;
+ private LRULinkedHashMap cache;
private AtomicLong cacheHitNum = new AtomicLong();
private AtomicLong cacheRequestNum = new AtomicLong();
@@ -60,9 +62,9 @@ public class TsFileMetaDataCache {
if (!cacheEnable) {
return;
}
- cache = new LRULinkedHashMap(MEMORY_THRESHOLD_IN_B, true) {
+ cache = new LRULinkedHashMap(MEMORY_THRESHOLD_IN_B, true) {
@Override
- protected long calEntrySize(String key, TsFileMetaData value) {
+ protected long calEntrySize(TsFileResource key, TsFileMetaData value) {
if (deviceIndexMapEntrySize == 0 && value.getDeviceMap().size() > 0) {
deviceIndexMapEntrySize = RamUsageEstimator
.sizeOf(value.getDeviceMap().entrySet().iterator().next());
@@ -74,7 +76,7 @@ public class TsFileMetaDataCache {
long valueSize = value.getDeviceMap().size() * deviceIndexMapEntrySize
+ measurementSchemaEntrySize * value.getMeasurementSchema().size()
+ versionAndCreatebySize;
- return key.length() * 2 + valueSize;
+ return key.getFile().getPath().length() * 2 + valueSize;
}
};
}
@@ -84,15 +86,16 @@ public class TsFileMetaDataCache {
}
/**
- * get the TsFileMetaData for given path.
+ * get the TsFileMetaData for given TsFile.
*
- * @param path -given path
+ * @param tsFileResource -given TsFile
*/
- public TsFileMetaData get(String path) throws IOException {
+ public TsFileMetaData get(TsFileResource tsFileResource) throws IOException {
if (!cacheEnable) {
- return TsFileMetadataUtils.getTsFileMetaData(path);
+ return TsFileMetadataUtils.getTsFileMetaData(tsFileResource);
}
+ String path = tsFileResource.getFile().getPath();
Object internPath = path.intern();
cacheRequestNum.incrementAndGet();
synchronized (cache) {
@@ -118,18 +121,18 @@ public class TsFileMetaDataCache {
logger.debug("Cache didn't hit: the number of requests for cache is {}",
cacheRequestNum.get());
}
- TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(path);
+ TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(tsFileResource);
synchronized (cache) {
- cache.put(path, fileMetaData);
+ cache.put(tsFileResource, fileMetaData);
return fileMetaData;
}
}
}
- public void remove(String path) {
+ public void remove(TsFileResource resource) {
synchronized (cache) {
if (cache != null) {
- cache.remove(path);
+ cache.remove(resource);
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
index 751939a0c5668855da96b764a1189b20b04fe66e..4bce328cf08c471fa27d2b94c8a7db4f5cfd23f3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
@@ -20,10 +20,12 @@ package org.apache.iotdb.db.engine.cache;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
@@ -46,30 +48,30 @@ public class TsFileMetadataUtils {
/**
* get tsfile meta data.
*
- * @param filePath -given path
+ * @param resource -given TsFile
* @return -meta data
*/
- public static TsFileMetaData getTsFileMetaData(String filePath) throws IOException {
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true);
+ public static TsFileMetaData getTsFileMetaData(TsFileResource resource) throws IOException {
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(resource, true);
return reader.readFileMetadata();
}
/**
* get row group block meta data.
*
- * @param filePath -file path
+ * @param resource -TsFile
* @param seriesPath -series path
* @param fileMetaData -tsfile meta data
* @return -device meta data
*/
- public static TsDeviceMetadata getTsDeviceMetaData(String filePath, Path seriesPath,
+ public static TsDeviceMetadata getTsDeviceMetaData(TsFileResource resource, Path seriesPath,
TsFileMetaData fileMetaData) throws IOException {
if (!fileMetaData.getMeasurementSchema().containsKey(seriesPath.getMeasurement())) {
return null;
} else {
// get the index information of TsDeviceMetadata
TsDeviceMetadataIndex index = fileMetaData.getDeviceMetadataIndex(seriesPath.getDevice());
- TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(resource, true);
// read TsDeviceMetadata from file
return tsFileReader.readTsDeviceMetaData(index);
}
@@ -95,6 +97,9 @@ public class TsFileMetadataUtils {
}
}
}
+ for (List chunkMetaDataList : pathToChunkMetaDataList.values()) {
+ chunkMetaDataList.sort(Comparator.comparingLong(ChunkMetaData::getStartTime));
+ }
return pathToChunkMetaDataList;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/FlushManager.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/FlushManager.java
index 2ed5305e9748a8e9955848c280c8817ebe0cbcd7..f2de30598de3e828fed63a35702ab247a49205e6 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/FlushManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/FlushManager.java
@@ -96,7 +96,7 @@ public class FlushManager implements FlushManagerMBean, IService {
}
/**
- * Add BufferWriteProcessor to asyncTryToFlush manager
+ * Add TsFileProcessor to asyncTryToFlush manager
*/
@SuppressWarnings("squid:S2445")
public void registerTsFileProcessor(TsFileProcessor tsFileProcessor) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
index b6cdb97cefdcb2962d81357a7da348a8d469225e..412f3b729f80f88abaacd2fce2cb751d9190ad5d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
@@ -30,7 +30,7 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.slf4j.Logger;
@@ -50,14 +50,14 @@ public class MemTableFlushTask {
private String storageGroup;
private IMemTable memTable;
- private FileSchema fileSchema;
+ private Schema schema;
private volatile boolean noMoreEncodingTask = false;
private volatile boolean noMoreIOTask = false;
- public MemTableFlushTask(IMemTable memTable, FileSchema fileSchema, RestorableTsFileIOWriter writer, String storageGroup) {
+ public MemTableFlushTask(IMemTable memTable, Schema schema, RestorableTsFileIOWriter writer, String storageGroup) {
this.memTable = memTable;
- this.fileSchema = fileSchema;
+ this.schema = schema;
this.writer = writer;
this.storageGroup = storageGroup;
subTaskPoolManager.submit(encodingTask);
@@ -78,7 +78,7 @@ public class MemTableFlushTask {
for (String measurementId : memTable.getMemTableMap().get(deviceId).keySet()) {
long startTime = System.currentTimeMillis();
IWritableMemChunk series = memTable.getMemTableMap().get(deviceId).get(measurementId);
- MeasurementSchema desc = fileSchema.getMeasurementSchema(measurementId);
+ MeasurementSchema desc = schema.getMeasurementSchema(measurementId);
TVList tvList = series.getSortedTVList();
sortTime += System.currentTimeMillis() - startTime;
encodingTaskQueue.add(new Pair<>(tvList, desc));
@@ -168,8 +168,7 @@ public class MemTableFlushTask {
Pair encodingMessage = (Pair) task;
ChunkBuffer chunkBuffer = ChunkBufferPool.getInstance()
.getEmptyChunkBuffer(this, encodingMessage.right);
- IChunkWriter seriesWriter = new ChunkWriterImpl(encodingMessage.right, chunkBuffer,
- PAGE_SIZE_THRESHOLD);
+ IChunkWriter seriesWriter = new ChunkWriterImpl(chunkBuffer, PAGE_SIZE_THRESHOLD);
writeOneSeries(encodingMessage.left, seriesWriter, encodingMessage.right.getType());
ioTaskQueue.add(seriesWriter);
memSerializeTime += System.currentTimeMillis() - starTime;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index a4fd2a91a62092a9c17b35d5309eb226fdae98cc..2aa882e728dcf7d9dd7c25f9135395ec31a49bab 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -26,6 +26,7 @@ import java.util.Map.Entry;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.MemUtils;
@@ -88,6 +89,14 @@ public abstract class AbstractMemTable implements IMemTable {
memSize += recordSizeInByte;
}
+ @Override
+ public void insertBatch(BatchInsertPlan batchInsertPlan, List indexes) {
+ write(batchInsertPlan, indexes);
+ long recordSizeInByte = MemUtils.getRecordSize(batchInsertPlan);
+ memSize += recordSizeInByte;
+ }
+
+
@Override
public void write(String deviceId, String measurement, TSDataType dataType, long insertTime,
String insertValue) {
@@ -96,13 +105,15 @@ public abstract class AbstractMemTable implements IMemTable {
}
@Override
- public void write(String deviceId, String measurement, TSDataType dataType, long insertTime,
- Object value) {
- IWritableMemChunk memSeries = createIfNotExistAndGet(deviceId, measurement, dataType);
- memSeries.write(insertTime, value);
- // update memory size of current memtable
+ public void write(BatchInsertPlan batchInsertPlan, List indexes) {
+ for (int i = 0; i < batchInsertPlan.getMeasurements().length; i++) {
+ IWritableMemChunk memSeries = createIfNotExistAndGet(batchInsertPlan.getDeviceId(),
+ batchInsertPlan.getMeasurements()[i], batchInsertPlan.getDataTypes()[i]);
+ memSeries.write(batchInsertPlan.getTimes(), batchInsertPlan.getColumns()[i], batchInsertPlan.getDataTypes()[i], indexes);
+ }
}
+
@Override
public long size() {
long sum = 0;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
index accbc5218563553d81ae214045e6b1fb90529621..5501e334837a3121862d59e95167de43f5656dde 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
@@ -18,9 +18,11 @@
*/
package org.apache.iotdb.db.engine.memtable;
+import java.util.List;
import java.util.Map;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -38,8 +40,7 @@ public interface IMemTable {
void write(String deviceId, String measurement, TSDataType dataType,
long insertTime, String insertValue);
- void write(String deviceId, String measurement, TSDataType dataType,
- long insertTime, Object value);
+ void write(BatchInsertPlan batchInsertPlan, List indexes);
/**
* @return the number of points
@@ -53,6 +54,8 @@ public interface IMemTable {
void insert(InsertPlan insertPlan);
+ void insertBatch(BatchInsertPlan batchInsertPlan, List indexes);
+
ReadOnlyMemChunk query(String deviceId, String measurement, TSDataType dataType,
Map props);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
index a7b4ed1211f296980c1d294b45ac68bec667e2d5..e5d683ce9e30667d9f6d62796908da4fcf35cae5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.engine.memtable;
+import java.util.List;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
@@ -36,9 +37,21 @@ public interface IWritableMemChunk extends TimeValuePairSorter {
void putBoolean(long t, boolean v);
+ void putLongs(long[] t, long[] v);
+
+ void putInts(long[] t, int[] v);
+
+ void putFloats(long[] t, float[] v);
+
+ void putDoubles(long[] t, double[] v);
+
+ void putBinaries(long[] t, Binary[] v);
+
+ void putBooleans(long[] t, boolean[] v);
+
void write(long insertTime, String insertValue);
- void write(long insertTime, Object insertValue);
+ void write(long[] times, Object valueList, TSDataType dataType, List indexes);
long count();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
index 0d34a57183c36b25a5211d0a550dcbcdbb0f096f..b37a2237fb16a74e8818ccc59434609430e66250 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
@@ -74,25 +74,67 @@ public class WritableMemChunk implements IWritableMemChunk {
}
@Override
- public void write(long insertTime, Object value) {
+ public void write(long[] times, Object valueList, TSDataType dataType, List indexes) {
switch (dataType) {
case BOOLEAN:
- putBoolean(insertTime, (Boolean)value);
+ boolean[] boolValues = (boolean[]) valueList;
+ if (times.length == indexes.size()) {
+ putBooleans(times, boolValues);
+ break;
+ }
+ for (Integer index : indexes) {
+ putBoolean(times[index], boolValues[index]);
+ }
break;
case INT32:
- putInt(insertTime, (Integer)value);
+ int[] intValues = (int[]) valueList;
+ if (times.length == indexes.size()) {
+ putInts(times, intValues);
+ break;
+ }
+ for (Integer index : indexes) {
+ putInt(times[index], intValues[index]);
+ }
break;
case INT64:
- putLong(insertTime, (Long)value);
+ long[] longValues = (long[]) valueList;
+ if (times.length == indexes.size()) {
+ putLongs(times, longValues);
+ break;
+ }
+ for (Integer index : indexes) {
+ putLong(times[index], longValues[index]);
+ }
break;
case FLOAT:
- putFloat(insertTime, (Float)value);
+ float[] floatValues = (float[]) valueList;
+ if (times.length == indexes.size()) {
+ putFloats(times, floatValues);
+ break;
+ }
+ for (Integer index : indexes) {
+ putFloat(times[index], floatValues[index]);
+ }
break;
case DOUBLE:
- putDouble(insertTime, (Double)value);
+ double[] doubleValues = (double[]) valueList;
+ if (times.length == indexes.size()) {
+ putDoubles(times, doubleValues);
+ break;
+ }
+ for (Integer index : indexes) {
+ putDouble(times[index], doubleValues[index]);
+ }
break;
case TEXT:
- putBinary(insertTime, (Binary)value);
+ Binary[] binaryValues = (Binary[]) valueList;
+ if (times.length == indexes.size()) {
+ putBinaries(times, binaryValues);
+ break;
+ }
+ for (Integer index : indexes) {
+ putBinary(times[index], binaryValues[index]);
+ }
break;
default:
throw new UnSupportedDataTypeException("Unsupported data type:" + dataType);
@@ -131,6 +173,36 @@ public class WritableMemChunk implements IWritableMemChunk {
list.putBoolean(t, v);
}
+ @Override
+ public void putLongs(long[] t, long[] v) {
+ list.putLongs(t, v);
+ }
+
+ @Override
+ public void putInts(long[] t, int[] v) {
+ list.putInts(t, v);
+ }
+
+ @Override
+ public void putFloats(long[] t, float[] v) {
+ list.putFloats(t, v);
+ }
+
+ @Override
+ public void putDoubles(long[] t, double[] v) {
+ list.putDoubles(t, v);
+ }
+
+ @Override
+ public void putBinaries(long[] t, Binary[] v) {
+ list.putBinaries(t, v);
+ }
+
+ @Override
+ public void putBooleans(long[] t, boolean[] v) {
+ list.putBooleans(t, v);
+ }
+
@Override
public synchronized TVList getSortedTVList() {
list.sort();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeContext.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeContext.java
new file mode 100644
index 0000000000000000000000000000000000000000..c6dd26c81ab8f41d01889e84c172563d15f2bd6a
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeContext.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.manage;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+/**
+ * MergeContext records the shared information between merge sub-tasks.
+ */
+public class MergeContext {
+
+ private Map mergedChunkCnt = new HashMap<>();
+ private Map unmergedChunkCnt = new HashMap<>();
+ private Map>> unmergedChunkStartTimes = new HashMap<>();
+
+ private AtomicInteger totalChunkWritten = new AtomicInteger();
+ private AtomicLong totalPointWritten = new AtomicLong();
+
+ public void clear() {
+ mergedChunkCnt.clear();
+ unmergedChunkCnt.clear();
+ unmergedChunkStartTimes.clear();
+ }
+
+ public Map getMergedChunkCnt() {
+ return mergedChunkCnt;
+ }
+
+ public void setMergedChunkCnt(
+ Map mergedChunkCnt) {
+ this.mergedChunkCnt = mergedChunkCnt;
+ }
+
+ public Map getUnmergedChunkCnt() {
+ return unmergedChunkCnt;
+ }
+
+ public void setUnmergedChunkCnt(
+ Map unmergedChunkCnt) {
+ this.unmergedChunkCnt = unmergedChunkCnt;
+ }
+
+ public Map>> getUnmergedChunkStartTimes() {
+ return unmergedChunkStartTimes;
+ }
+
+ public void setUnmergedChunkStartTimes(
+ Map>> unmergedChunkStartTimes) {
+ this.unmergedChunkStartTimes = unmergedChunkStartTimes;
+ }
+
+ public int getTotalChunkWritten() {
+ return totalChunkWritten.get();
+ }
+
+ public void incTotalChunkWritten() {
+ this.totalChunkWritten.incrementAndGet();
+ }
+
+ public void incTotalPointWritten(long increment) {
+ totalPointWritten.addAndGet(increment);
+ }
+
+ public long getTotalPointWritten() {
+ return totalPointWritten.get();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java
new file mode 100644
index 0000000000000000000000000000000000000000..c23768ad85851327b066dabd4c9041f097bf6869
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.manage;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.service.IService;
+import org.apache.iotdb.db.service.ServiceType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MergeManager provides a ThreadPool to queue and run all merge tasks to restrain the total
+ * resources occupied by merge and manages a Timer to periodically issue a global merge.
+ */
+public class MergeManager implements IService {
+
+ private static final Logger logger = LoggerFactory.getLogger(MergeManager.class);
+ private static final MergeManager INSTANCE = new MergeManager();
+
+ private AtomicInteger threadCnt = new AtomicInteger();
+ private ThreadPoolExecutor mergeTaskPool;
+ private ThreadPoolExecutor mergeChunkSubTaskPool;
+ private ScheduledExecutorService timedMergeThreadPool;
+
+ private MergeManager() {
+ }
+
+ public static MergeManager getINSTANCE() {
+ return INSTANCE;
+ }
+
+ public void submitMainTask(MergeTask mergeTask) {
+ mergeTaskPool.submit(mergeTask);
+ }
+
+ public Future submitChunkSubTask(Callable callable) {
+ return mergeChunkSubTaskPool.submit(callable);
+ }
+
+ @Override
+ public void start() {
+ if (mergeTaskPool == null) {
+ int threadNum = IoTDBDescriptor.getInstance().getConfig().getMergeThreadNum();
+ if (threadNum <= 0) {
+ threadNum = 1;
+ }
+
+ int chunkSubThreadNum = IoTDBDescriptor.getInstance().getConfig().getMergeChunkSubThreadNum();
+ if (chunkSubThreadNum <= 0) {
+ chunkSubThreadNum = 1;
+ }
+
+ mergeTaskPool =
+ (ThreadPoolExecutor) Executors.newFixedThreadPool(threadNum,
+ r -> new Thread(r, "MergeThread-" + threadCnt.getAndIncrement()));
+ mergeChunkSubTaskPool =
+ (ThreadPoolExecutor) Executors.newFixedThreadPool(threadNum * chunkSubThreadNum,
+ r -> new Thread(r, "MergeChunkSubThread-" + threadCnt.getAndIncrement()));
+ long mergeInterval = IoTDBDescriptor.getInstance().getConfig().getMergeIntervalSec();
+ if (mergeInterval > 0) {
+ timedMergeThreadPool = Executors.newSingleThreadScheduledExecutor( r -> new Thread(r,
+ "TimedMergeThread"));
+ timedMergeThreadPool.scheduleAtFixedRate(this::mergeAll, mergeInterval,
+ mergeInterval, TimeUnit.SECONDS);
+ }
+ logger.info("MergeManager started");
+ }
+ }
+
+ @Override
+ public void stop() {
+ if (mergeTaskPool != null) {
+ if (timedMergeThreadPool != null) {
+ timedMergeThreadPool.shutdownNow();
+ timedMergeThreadPool = null;
+ }
+ mergeTaskPool.shutdownNow();
+ mergeChunkSubTaskPool.shutdownNow();
+ logger.info("Waiting for task pool to shut down");
+ while (!mergeTaskPool.isTerminated() || !mergeChunkSubTaskPool.isTerminated() ) {
+ // wait
+ }
+ mergeTaskPool = null;
+ logger.info("MergeManager stopped");
+ }
+ }
+
+ @Override
+ public ServiceType getID() {
+ return ServiceType.MERGE_SERVICE;
+ }
+
+ private void mergeAll() {
+ try {
+ StorageEngine.getInstance().mergeAll(IoTDBDescriptor.getInstance().getConfig().isForceFullMerge());
+ } catch (StorageEngineException e) {
+ logger.error("Cannot perform a global merge because", e);
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
new file mode 100644
index 0000000000000000000000000000000000000000..042c499ad38a5a140ce807bdd0490d00b6f2c7d6
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
@@ -0,0 +1,262 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.manage;
+
+import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.reader.IPointReader;
+import org.apache.iotdb.db.query.reader.resourceRelated.CachedUnseqResourceMergeReader;
+import org.apache.iotdb.db.utils.MergeUtils;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
+import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
+import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MergeResource manages files and caches of readers, writers, MeasurementSchemas and
+ * modifications to avoid unnecessary object creations and file openings.
+ */
+public class MergeResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(MergeResource.class);
+
+ private List seqFiles;
+ private List unseqFiles;
+
+ private Map fileReaderCache = new HashMap<>();
+ private Map fileWriterCache = new HashMap<>();
+ private Map> modificationCache = new HashMap<>();
+ private Map measurementSchemaMap = new HashMap<>();
+ private Map chunkWriterCache = new ConcurrentHashMap<>();
+
+ private boolean cacheDeviceMeta = false;
+
+ public MergeResource(List seqFiles, List unseqFiles) {
+ this.seqFiles = seqFiles.stream().filter(TsFileResource::isClosed).collect(Collectors.toList());
+ this.unseqFiles =
+ unseqFiles.stream().filter(TsFileResource::isClosed).collect(Collectors.toList());
+ }
+
+ public void clear() throws IOException {
+ for (TsFileSequenceReader sequenceReader : fileReaderCache.values()) {
+ sequenceReader.close();
+ }
+
+ fileReaderCache.clear();
+ fileWriterCache.clear();
+ modificationCache.clear();
+ measurementSchemaMap.clear();
+ chunkWriterCache.clear();
+ }
+
+ public MeasurementSchema getSchema(String measurementId) {
+ return measurementSchemaMap.get(measurementId);
+ }
+
+ /**
+ * Construct a new or get an existing RestorableTsFileIOWriter of a merge temp file for a
+ * SeqFile. The path of the merge temp file will be the seqFile's + ".merge".
+ * @param resource
+ * @return A RestorableTsFileIOWriter of a merge temp file for a SeqFile.
+ * @throws IOException
+ */
+ public RestorableTsFileIOWriter getMergeFileWriter(TsFileResource resource) throws IOException {
+ RestorableTsFileIOWriter writer = fileWriterCache.get(resource);
+ if (writer == null) {
+ writer = new RestorableTsFileIOWriter(new File(resource.getFile().getPath() + MERGE_SUFFIX));
+ fileWriterCache.put(resource, writer);
+ }
+ return writer;
+ }
+
+ /**
+ * Query ChunkMetadata of a timeseries from the given TsFile (seq or unseq). The ChunkMetadata
+ * is not cached since it is usually huge.
+ * @param path name of the time series
+ * @param seqFile
+ * @return
+ * @throws IOException
+ */
+ public List queryChunkMetadata(Path path, TsFileResource seqFile)
+ throws IOException {
+ TsFileSequenceReader sequenceReader = getFileReader(seqFile);
+ return sequenceReader.getChunkMetadataList(path);
+ }
+
+ /**
+ * Construct the a new or get an existing TsFileSequenceReader of a TsFile.
+ * @param tsFileResource
+ * @return a TsFileSequenceReader
+ */
+ public TsFileSequenceReader getFileReader(TsFileResource tsFileResource) throws IOException {
+ TsFileSequenceReader reader = fileReaderCache.get(tsFileResource);
+ if (reader == null) {
+ reader = new TsFileSequenceReader(tsFileResource.getFile().getPath(), true, cacheDeviceMeta);
+ fileReaderCache.put(tsFileResource, reader);
+ }
+ return reader;
+ }
+
+ /**
+ * Construct UnseqResourceMergeReaders of for each timeseries over all seqFiles. The readers are
+ * not cached since the method is only called once for each timeseries.
+ * @param paths names of the timeseries
+ * @return an array of UnseqResourceMergeReaders each corresponding to a timeseries in paths
+ * @throws IOException
+ */
+ public IPointReader[] getUnseqReaders(List paths) throws IOException {
+ List[] pathChunks = MergeUtils.collectUnseqChunks(paths, unseqFiles, this);
+ IPointReader[] ret = new IPointReader[paths.size()];
+ for (int i = 0; i < paths.size(); i++) {
+ TSDataType dataType = getSchema(paths.get(i).getMeasurement()).getType();
+ ret[i] = new CachedUnseqResourceMergeReader(pathChunks[i], dataType);
+ }
+ return ret;
+ }
+
+ /**
+ * Construct the a new or get an existing ChunkWriter of a measurement. Different timeseries of
+ * the same measurement shares the same instance.
+ * @param measurementSchema
+ * @return
+ */
+ public IChunkWriter getChunkWriter(MeasurementSchema measurementSchema) {
+ return chunkWriterCache.computeIfAbsent(measurementSchema,
+ schema -> new ChunkWriterImpl(new ChunkBuffer(schema), TSFileConfig.pageCheckSizeThreshold));
+ }
+
+ /**
+ * Get the modifications of a timeseries in the ModificationFile of a TsFile. Once the
+ * modifications of the timeseries are found out, they will be removed from the list to boost
+ * the next query, so two calls of the same file and timeseries are forbidden.
+ * @param tsFileResource
+ * @param path name of the time series
+ * @return
+ */
+ public List getModifications(TsFileResource tsFileResource, Path path) {
+ // copy from TsFileResource so queries are not affected
+ List modifications = modificationCache.computeIfAbsent(tsFileResource,
+ resource -> new LinkedList<>(resource.getModFile().getModifications()));
+ List pathModifications = new ArrayList<>();
+ Iterator modificationIterator = modifications.iterator();
+ // each path is visited only once in a merge, so the modifications can be removed after visiting
+ while (modificationIterator.hasNext()) {
+ Modification modification = modificationIterator.next();
+ if (modification.getPath().equals(path)) {
+ pathModifications.add(modification);
+ modificationIterator.remove();
+ }
+ }
+ return pathModifications;
+ }
+
+ /**
+ * Remove and close the writer of the merge temp file of a SeqFile. The merge temp file is also
+ * deleted.
+ * @param tsFileResource the SeqFile
+ * @throws IOException
+ */
+ public void removeFileAndWriter(TsFileResource tsFileResource) throws IOException {
+ RestorableTsFileIOWriter newFileWriter = fileWriterCache.remove(tsFileResource);
+ if (newFileWriter != null) {
+ newFileWriter.close();
+ newFileWriter.getFile().delete();
+ }
+ }
+
+ /**
+ * Remove and close the reader of the TsFile. The TsFile is NOT deleted.
+ * @param resource the SeqFile
+ * @throws IOException
+ */
+ public void removeFileReader(TsFileResource resource) throws IOException {
+ TsFileSequenceReader sequenceReader = fileReaderCache.remove(resource);
+ if (sequenceReader != null) {
+ sequenceReader.close();
+ }
+ }
+
+ public List getSeqFiles() {
+ return seqFiles;
+ }
+
+ public void setSeqFiles(List seqFiles) {
+ this.seqFiles = seqFiles;
+ }
+
+ public List getUnseqFiles() {
+ return unseqFiles;
+ }
+
+ public void setUnseqFiles(
+ List unseqFiles) {
+ this.unseqFiles = unseqFiles;
+ }
+
+ public Map getMeasurementSchemaMap() {
+ return measurementSchemaMap;
+ }
+
+ public void removeOutdatedSeqReaders() throws IOException {
+ Iterator> entryIterator =
+ fileReaderCache.entrySet().iterator();
+ while (entryIterator.hasNext()) {
+ Entry entry = entryIterator.next();
+ TsFileResource tsFile = entry.getKey();
+ if (!seqFiles.contains(tsFile)) {
+ TsFileSequenceReader reader = entry.getValue();
+ reader.close();
+ entryIterator.remove();
+ }
+ }
+ }
+
+ public void setCacheDeviceMeta(boolean cacheDeviceMeta) {
+ this.cacheDeviceMeta = cacheDeviceMeta;
+ }
+
+ public void addMeasurements(List measurementSchemas) {
+ for (MeasurementSchema measurementSchema : measurementSchemas) {
+ measurementSchemaMap.put(measurementSchema.getMeasurementId(), measurementSchema);
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/package-info.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/package-info.java
new file mode 100644
index 0000000000000000000000000000000000000000..9ed9a76f15849700a8b1af353ddeee28c21b5744
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/package-info.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Package merge contains classes supporting merge functionality (also compaction in other
+ * systems) that compacts several data files into a new one, making it more ordered and removing
+ * duplicated data.
+ */
+package org.apache.iotdb.db.engine.merge;
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java
new file mode 100644
index 0000000000000000000000000000000000000000..b7d98fc81ec44854e0e38a8ed3be88dd7ade1254
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.recover;
+
+import static org.apache.iotdb.db.engine.merge.recover.MergeLogger.*;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.*;
+import java.util.Map.Entry;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.MetadataErrorException;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * LogAnalyzer scans the "merge.log" file and recovers information such as files of last merge,
+ * the last available positions of each file and how many timeseries and files have been merged.
+ * An example of merging 1 seqFile and 1 unseqFile containing 3 series is:
+ * seqFiles
+ * server/0seq.tsfile
+ * unseqFiles
+ * server/0unseq.tsfile
+ * merge start
+ * start root.mergeTest.device0.sensor0
+ * server/0seq.tsfile.merge 338
+ * end
+ * start root.mergeTest.device0.sensor1
+ * server/0seq.tsfile.merge 664
+ * end
+ * all ts end
+ * server/0seq.tsfile 145462
+ * end
+ * merge end
+ */
+public class LogAnalyzer {
+
+ private static final Logger logger = LoggerFactory.getLogger(LogAnalyzer.class);
+
+ private MergeResource resource;
+ private String taskName;
+ private File logFile;
+ private String storageGroupName;
+
+ private Map fileLastPositions = new HashMap<>();
+ private Map tempFileLastPositions = new HashMap<>();
+
+ private List mergedPaths = new ArrayList<>();
+ private List unmergedPaths;
+ private List unmergedFiles;
+ private String currLine;
+
+ private Status status;
+
+ public LogAnalyzer(MergeResource resource, String taskName, File logFile, String storageGroupName) {
+ this.resource = resource;
+ this.taskName = taskName;
+ this.logFile = logFile;
+ this.storageGroupName = storageGroupName;
+ }
+
+ /**
+ * Scan through the logs to find out where the last merge has stopped and store the information
+ * about the progress in the fields.
+ * @return a Status indicating the completed stage of the last merge.
+ * @throws IOException
+ */
+ public Status analyze() throws IOException, MetadataErrorException {
+ status = Status.NONE;
+ try (BufferedReader bufferedReader = new BufferedReader(new FileReader(logFile))) {
+ currLine = bufferedReader.readLine();
+ if (currLine != null) {
+ analyzeSeqFiles(bufferedReader);
+
+ analyzeUnseqFiles(bufferedReader);
+
+ List storageGroupPaths = MManager.getInstance().getPaths(storageGroupName + ".*");
+ unmergedPaths = new ArrayList<>();
+ for (String path : storageGroupPaths) {
+ unmergedPaths.add(new Path(path));
+ }
+
+ analyzeMergedSeries(bufferedReader, unmergedPaths);
+
+ analyzeMergedFiles(bufferedReader);
+ }
+ }
+ return status;
+ }
+
+ private void analyzeSeqFiles(BufferedReader bufferedReader) throws IOException {
+ if (!STR_SEQ_FILES.equals(currLine)) {
+ return;
+ }
+ long startTime = System.currentTimeMillis();
+ List mergeSeqFiles = new ArrayList<>();
+ while ((currLine = bufferedReader.readLine()) != null) {
+ if (STR_UNSEQ_FILES.equals(currLine)) {
+ break;
+ }
+ Iterator iterator = resource.getSeqFiles().iterator();
+ while (iterator.hasNext()) {
+ TsFileResource seqFile = iterator.next();
+ if (seqFile.getFile().getAbsolutePath().equals(currLine)) {
+ mergeSeqFiles.add(seqFile);
+ // remove to speed-up next iteration
+ iterator.remove();
+ break;
+ }
+ }
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} found {} seq files after {}ms", taskName, mergeSeqFiles.size(),
+ (System.currentTimeMillis() - startTime));
+ }
+ resource.setSeqFiles(mergeSeqFiles);
+ }
+
+ private void analyzeUnseqFiles(BufferedReader bufferedReader) throws IOException {
+ if (!STR_UNSEQ_FILES.equals(currLine)) {
+ return;
+ }
+ long startTime = System.currentTimeMillis();
+ List mergeUnseqFiles = new ArrayList<>();
+ while ((currLine = bufferedReader.readLine()) != null) {
+ if (currLine.equals(STR_TIMESERIES)) {
+ break;
+ }
+ Iterator iterator = resource.getUnseqFiles().iterator();
+ while (iterator.hasNext()) {
+ TsFileResource unseqFile = iterator.next();
+ if (unseqFile.getFile().getAbsolutePath().equals(currLine)) {
+ mergeUnseqFiles.add(unseqFile);
+ // remove to speed-up next iteration
+ iterator.remove();
+ break;
+ }
+ }
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} found {} unseq files after {}ms", taskName, mergeUnseqFiles.size(),
+ (System.currentTimeMillis() - startTime));
+ }
+ resource.setUnseqFiles(mergeUnseqFiles);
+ }
+
+ private void analyzeMergedSeries(BufferedReader bufferedReader, List unmergedPaths) throws IOException {
+ if (!STR_MERGE_START.equals(currLine)) {
+ return;
+ }
+
+ status = Status.MERGE_START;
+ for (TsFileResource seqFile : resource.getSeqFiles()) {
+ File mergeFile = new File(seqFile.getFile().getPath() + MergeTask.MERGE_SUFFIX);
+ fileLastPositions.put(mergeFile, 0L);
+ }
+
+ List currTSList = new ArrayList<>();
+ long startTime = System.currentTimeMillis();
+ while ((currLine = bufferedReader.readLine()) != null) {
+ if (STR_ALL_TS_END.equals(currLine)) {
+ break;
+ }
+ if (currLine.contains(STR_START)) {
+ // a TS starts to merge
+ String[] splits = currLine.split(" ");
+ for (int i = 1; i < splits.length; i ++) {
+ currTSList.add(new Path(splits[i]));
+ }
+ tempFileLastPositions.clear();
+ } else if (!currLine.contains(STR_END)) {
+ // file position
+ String[] splits = currLine.split(" ");
+ File file = new File(splits[0]);
+ Long position = Long.parseLong(splits[1]);
+ tempFileLastPositions.put(file, position);
+ } else {
+ // a TS ends merging
+ unmergedPaths.removeAll(currTSList);
+ for (Entry entry : tempFileLastPositions.entrySet()) {
+ fileLastPositions.put(entry.getKey(), entry.getValue());
+ }
+ mergedPaths.addAll(currTSList);
+ }
+ }
+ tempFileLastPositions = null;
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} found {} series have already been merged after {}ms", taskName,
+ mergedPaths.size(), (System.currentTimeMillis() - startTime));
+ }
+ }
+
+ private void analyzeMergedFiles(BufferedReader bufferedReader) throws IOException {
+ if (!STR_ALL_TS_END.equals(currLine)) {
+ return;
+ }
+
+ status = Status.ALL_TS_MERGED;
+ unmergedFiles = resource.getSeqFiles();
+
+ File currFile = null;
+ long startTime = System.currentTimeMillis();
+ int mergedCnt = 0;
+ while ((currLine = bufferedReader.readLine()) != null) {
+ if (STR_MERGE_END.equals(currLine)) {
+ status = Status.MERGE_END;
+ break;
+ }
+ if (!currLine.contains(STR_END)) {
+ String[] splits = currLine.split(" ");
+ currFile = new File(splits[0]);
+ Long lastPost = Long.parseLong(splits[1]);
+ fileLastPositions.put(currFile, lastPost);
+ } else {
+ fileLastPositions.remove(currFile);
+ String seqFilePath = currFile.getAbsolutePath().replace(MergeTask.MERGE_SUFFIX, "");
+ Iterator unmergedFileIter = unmergedFiles.iterator();
+ while (unmergedFileIter.hasNext()) {
+ TsFileResource seqFile = unmergedFileIter.next();
+ if (seqFile.getFile().getAbsolutePath().equals(seqFilePath)) {
+ mergedCnt ++;
+ unmergedFileIter.remove();
+ break;
+ }
+ }
+ }
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} found {} files have already been merged after {}ms", taskName,
+ mergedCnt, (System.currentTimeMillis() - startTime));
+ }
+ }
+
+ public List getUnmergedPaths() {
+ return unmergedPaths;
+ }
+
+ public void setUnmergedPaths(List unmergedPaths) {
+ this.unmergedPaths = unmergedPaths;
+ }
+
+ public List getUnmergedFiles() {
+ return unmergedFiles;
+ }
+
+ public void setUnmergedFiles(
+ List unmergedFiles) {
+ this.unmergedFiles = unmergedFiles;
+ }
+
+ public List getMergedPaths() {
+ return mergedPaths;
+ }
+
+ public void setMergedPaths(List mergedPaths) {
+ this.mergedPaths = mergedPaths;
+ }
+
+ public Map getFileLastPositions() {
+ return fileLastPositions;
+ }
+
+ public void setFileLastPositions(Map fileLastPositions) {
+ this.fileLastPositions = fileLastPositions;
+ }
+
+ public enum Status {
+ // almost nothing has been done
+ NONE,
+ // at least the files and timeseries to be merged are known
+ MERGE_START,
+ // all the timeseries have been merged(merged chunks are generated)
+ ALL_TS_MERGED,
+ // all the merge files are merged with the origin files and the task is almost done
+ MERGE_END
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java
new file mode 100644
index 0000000000000000000000000000000000000000..326647ab7735f2bc68af630c82c56a9c9faa0097
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.recover;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+/**
+ * MergeLogger records the progress of a merge in file "merge.log" as text lines.
+ */
+public class MergeLogger {
+
+ public static final String MERGE_LOG_NAME = "merge.log";
+
+ static final String STR_SEQ_FILES = "seqFiles";
+ static final String STR_UNSEQ_FILES = "unseqFiles";
+ static final String STR_TIMESERIES = "timeseries";
+ static final String STR_START = "start";
+ static final String STR_END = "end";
+ static final String STR_ALL_TS_END = "all ts end";
+ static final String STR_MERGE_START = "merge start";
+ static final String STR_MERGE_END = "merge end";
+
+ private BufferedWriter logStream;
+
+ public MergeLogger(String storageGroupDir) throws IOException {
+ logStream = new BufferedWriter(new FileWriter(new File(storageGroupDir, MERGE_LOG_NAME), true));
+ }
+
+ public void close() throws IOException {
+ logStream.close();
+ }
+
+ public void logTSStart(List paths) throws IOException {
+ logStream.write(STR_START);
+ for (Path path : paths) {
+ logStream.write(" " + path.getFullPath());
+ }
+ logStream.newLine();
+ logStream.flush();
+ }
+
+ public void logFilePosition(File file) throws IOException {
+ logStream.write(String.format("%s %d", file.getAbsolutePath(), file.length()));
+ logStream.newLine();
+ logStream.flush();
+ }
+
+ public void logTSEnd() throws IOException {
+ logStream.write(STR_END);
+ logStream.newLine();
+ logStream.flush();
+ }
+
+ public void logAllTsEnd() throws IOException {
+ logStream.write(STR_ALL_TS_END);
+ logStream.newLine();
+ logStream.flush();
+ }
+
+ public void logFileMergeStart(File file, long position) throws IOException {
+ logStream.write(String.format("%s %d", file.getAbsolutePath(), position));
+ logStream.newLine();
+ logStream.flush();
+ }
+
+ public void logFileMergeEnd() throws IOException {
+ logStream.write(STR_END);
+ logStream.newLine();
+ logStream.flush();
+ }
+
+ public void logMergeEnd() throws IOException {
+ logStream.write(STR_MERGE_END);
+ logStream.newLine();
+ logStream.flush();
+ }
+
+ public void logFiles(MergeResource resource) throws IOException {
+ logSeqFiles(resource.getSeqFiles());
+ logUnseqFiles(resource.getUnseqFiles());
+ }
+
+ private void logSeqFiles(List seqFiles) throws IOException {
+ logStream.write(STR_SEQ_FILES);
+ logStream.newLine();
+ for (TsFileResource tsFileResource : seqFiles) {
+ logStream.write(tsFileResource.getFile().getAbsolutePath());
+ logStream.newLine();
+ }
+ logStream.flush();
+ }
+
+ private void logUnseqFiles(List unseqFiles) throws IOException {
+ logStream.write(STR_UNSEQ_FILES);
+ logStream.newLine();
+ for (TsFileResource tsFileResource : unseqFiles) {
+ logStream.write(tsFileResource.getFile().getAbsolutePath());
+ logStream.newLine();
+ }
+ logStream.flush();
+ }
+
+ public void logMergeStart() throws IOException {
+ logStream.write(STR_MERGE_START);
+ logStream.newLine();
+ logStream.flush();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IFileQueryMemMeasurement.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IFileQueryMemMeasurement.java
new file mode 100644
index 0000000000000000000000000000000000000000..fde9be8d21b1db71bbc476d6089fb8cfb600c730
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IFileQueryMemMeasurement.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.selector;
+
+import java.io.IOException;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+
+
+@FunctionalInterface
+/**
+ * Estimate how much memory a file may occupy when being queried during merge.
+ * @param resource
+ * @return
+ * @throws IOException
+ */
+interface IFileQueryMemMeasurement {
+ long measure(TsFileResource resource) throws IOException;
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergeFileSelector.java
new file mode 100644
index 0000000000000000000000000000000000000000..a1a11cf5f5b894aaa8ea39f080effb39685eb00a
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergeFileSelector.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.selector;
+
+import java.util.List;
+import org.apache.iotdb.db.exception.MergeException;
+
+/**
+ * IMergeFileSelector selects a set of files from given seqFiles and unseqFiles which can be
+ * merged without exceeding given memory budget.
+ */
+public interface IMergeFileSelector {
+
+ List[] select() throws MergeException;
+
+ int getConcurrentMergeNum();
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergePathSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergePathSelector.java
new file mode 100644
index 0000000000000000000000000000000000000000..ed6979f186868fa8bd09ba2eadb26b82192b4045
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergePathSelector.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.selector;
+
+import java.util.Iterator;
+import java.util.List;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+/**
+ * IMergePathSelector select paths to be merged at a time if all paths of a device cannot be
+ * merged at the same time.
+ */
+public interface IMergePathSelector extends Iterator> {
+
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
new file mode 100644
index 0000000000000000000000000000000000000000..bfa9f2f0b20ce1ee4f2b930ece3d3bc8d66758ac
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.selector;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.MergeException;
+import org.apache.iotdb.db.utils.MergeUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MaxFileMergeFileSelector selects the most files from given seqFiles and unseqFiles which can be
+ * merged without exceeding given memory budget. It always assume the number of timeseries being
+ * queried at the same time is 1 to maximize the number of file merged.
+ */
+public class MaxFileMergeFileSelector implements IMergeFileSelector {
+
+ private static final Logger logger = LoggerFactory.getLogger(MaxFileMergeFileSelector.class);
+ private static final String LOG_FILE_COST = "Memory cost of file {} is {}";
+
+ MergeResource resource;
+
+ long totalCost;
+ private long memoryBudget;
+ private long maxSeqFileCost;
+
+ // the number of timeseries being queried at the same time
+ int concurrentMergeNum = 1;
+
+ /**
+ * Total metadata size of each file.
+ */
+ private Map fileMetaSizeMap = new HashMap<>();
+ /**
+ * Maximum memory cost of querying a timeseries in each file.
+ */
+ private Map maxSeriesQueryCostMap = new HashMap<>();
+
+ List selectedUnseqFiles;
+ List selectedSeqFiles;
+
+ private Collection tmpSelectedSeqFiles;
+ private long tempMaxSeqFileCost;
+
+ private boolean[] seqSelected;
+ private int seqSelectedNum;
+
+ public MaxFileMergeFileSelector(MergeResource resource, long memoryBudget) {
+ this.resource = resource;
+ this.memoryBudget = memoryBudget;
+ }
+
+ /**
+ * Select merge candidates from seqFiles and unseqFiles under the given memoryBudget.
+ * This process iteratively adds the next unseqFile from unseqFiles and its overlapping seqFiles
+ * as newly-added candidates and computes their estimated memory cost. If the current cost
+ * pluses the new cost is still under the budget, accept the unseqFile and the seqFiles as
+ * candidates, otherwise go to the next iteration.
+ * The memory cost of a file is calculated in two ways:
+ * The rough estimation: for a seqFile, the size of its metadata is used for estimation.
+ * Since in the worst case, the file only contains one timeseries and all its metadata will
+ * be loaded into memory with at most one actual data chunk (which is negligible) and writing
+ * the timeseries into a new file generate metadata of the similar size, so the size of all
+ * seqFiles' metadata (generated when writing new chunks) pluses the largest one (loaded
+ * when reading a timeseries from the seqFiles) is the total estimation of all seqFiles; for
+ * an unseqFile, since the merge reader may read all chunks of a series to perform a merge
+ * read, the whole file may be loaded into memory, so we use the file's length as the
+ * maximum estimation.
+ * The tight estimation: based on the rough estimation, we scan the file's metadata to
+ * count the number of chunks for each series, find the series which have the most
+ * chunks in the file and use its chunk proportion to refine the rough estimation.
+ * The rough estimation is performed first, if no candidates can be found using rough
+ * estimation, we run the selection again with tight estimation.
+ * @return two lists of TsFileResource, the former is selected seqFiles and the latter is
+ * selected unseqFiles or an empty array if there are no proper candidates by the budget.
+ * @throws MergeException
+ */
+ @Override
+ public List[] select() throws MergeException {
+ long startTime = System.currentTimeMillis();
+ try {
+ logger.info("Selecting merge candidates from {} seqFile, {} unseqFiles",
+ resource.getSeqFiles().size(), resource.getUnseqFiles().size());
+ select(false);
+ if (selectedUnseqFiles.isEmpty()) {
+ select(true);
+ }
+ resource.setSeqFiles(selectedSeqFiles);
+ resource.setUnseqFiles(selectedUnseqFiles);
+ resource.removeOutdatedSeqReaders();
+ if (selectedUnseqFiles.isEmpty()) {
+ logger.info("No merge candidates are found");
+ return new List[0];
+ }
+ } catch (IOException e) {
+ throw new MergeException(e);
+ }
+ if (logger.isInfoEnabled()) {
+ logger.info("Selected merge candidates, {} seqFiles, {} unseqFiles, total memory cost {}, "
+ + "time consumption {}ms",
+ selectedSeqFiles.size(), selectedUnseqFiles.size(), totalCost,
+ System.currentTimeMillis() - startTime);
+ }
+ return new List[]{selectedSeqFiles, selectedUnseqFiles};
+ }
+
+ void select(boolean useTightBound) throws IOException {
+ tmpSelectedSeqFiles = new HashSet<>();
+ seqSelected = new boolean[resource.getSeqFiles().size()];
+ seqSelectedNum = 0;
+ selectedSeqFiles = new ArrayList<>();
+ selectedUnseqFiles = new ArrayList<>();
+ maxSeqFileCost = 0;
+ tempMaxSeqFileCost = 0;
+
+ totalCost = 0;
+
+ int unseqIndex = 0;
+ long startTime = System.currentTimeMillis();
+ long timeConsumption = 0;
+ long timeLimit = IoTDBDescriptor.getInstance().getConfig().getMergeFileSelectionTimeBudget();
+ if (timeLimit < 0) {
+ timeLimit = Long.MAX_VALUE;
+ }
+ while (unseqIndex < resource.getUnseqFiles().size() && timeConsumption < timeLimit) {
+ // select next unseq files
+ TsFileResource unseqFile = resource.getUnseqFiles().get(unseqIndex);
+
+ selectOverlappedSeqFiles(unseqFile);
+
+ tempMaxSeqFileCost = maxSeqFileCost;
+ long newCost = useTightBound ? calculateTightMemoryCost(unseqFile, tmpSelectedSeqFiles,
+ startTime, timeLimit) :
+ calculateLooseMemoryCost(unseqFile, tmpSelectedSeqFiles, startTime, timeLimit);
+
+ if (totalCost + newCost < memoryBudget) {
+ selectedUnseqFiles.add(unseqFile);
+ maxSeqFileCost = tempMaxSeqFileCost;
+
+ for (Integer seqIdx : tmpSelectedSeqFiles) {
+ seqSelected[seqIdx] = true;
+ seqSelectedNum++;
+ selectedSeqFiles.add(resource.getSeqFiles().get(seqIdx));
+ }
+ totalCost += newCost;
+ logger.debug("Adding a new unseqFile {} and seqFiles {} as candidates, new cost {}, total"
+ + " cost {}",
+ unseqFile, tmpSelectedSeqFiles, newCost, totalCost);
+ }
+ tmpSelectedSeqFiles.clear();
+ unseqIndex++;
+ timeConsumption = System.currentTimeMillis() - startTime;
+ }
+ }
+
+ private void selectOverlappedSeqFiles(TsFileResource unseqFile) {
+ if (seqSelectedNum == resource.getSeqFiles().size()) {
+ return;
+ }
+ int tmpSelectedNum = 0;
+ for (Entry deviceStartTimeEntry : unseqFile.getStartTimeMap().entrySet()) {
+ String deviceId = deviceStartTimeEntry.getKey();
+ Long unseqStartTime = deviceStartTimeEntry.getValue();
+ Long unseqEndTime = unseqFile.getEndTimeMap().get(deviceId);
+
+ boolean noMoreOverlap = false;
+ for (int i = 0; i < resource.getSeqFiles().size() && !noMoreOverlap; i++) {
+ TsFileResource seqFile = resource.getSeqFiles().get(i);
+ if (seqSelected[i] || !seqFile.getEndTimeMap().containsKey(deviceId)) {
+ continue;
+ }
+ Long seqEndTime = seqFile.getEndTimeMap().get(deviceId);
+ if (unseqEndTime <= seqEndTime) {
+ // the unseqFile overlaps current seqFile
+ tmpSelectedSeqFiles.add(i);
+ tmpSelectedNum ++;
+ // the device of the unseqFile can not merge with later seqFiles
+ noMoreOverlap = true;
+ } else if (unseqStartTime <= seqEndTime) {
+ // the device of the unseqFile may merge with later seqFiles
+ // and the unseqFile overlaps current seqFile
+ tmpSelectedSeqFiles.add(i);
+ tmpSelectedNum++;
+ }
+ }
+ if (tmpSelectedNum + seqSelectedNum == resource.getSeqFiles().size()) {
+ break;
+ }
+ }
+ }
+
+ private long calculateMemoryCost(TsFileResource tmpSelectedUnseqFile,
+ Collection tmpSelectedSeqFiles, IFileQueryMemMeasurement unseqMeasurement,
+ IFileQueryMemMeasurement seqMeasurement, long startTime, long timeLimit) throws IOException {
+ long cost = 0;
+ Long fileCost = unseqMeasurement.measure(tmpSelectedUnseqFile);
+ cost += fileCost;
+
+ for (Integer seqFileIdx : tmpSelectedSeqFiles) {
+ TsFileResource seqFile = resource.getSeqFiles().get(seqFileIdx);
+ fileCost = seqMeasurement.measure(seqFile);
+ if (fileCost > tempMaxSeqFileCost) {
+ // only one file will be read at the same time, so only the largest one is recorded here
+ cost -= tempMaxSeqFileCost;
+ cost += fileCost;
+ tempMaxSeqFileCost = fileCost;
+ }
+ // but writing data into a new file may generate the same amount of metadata in memory
+ cost += calculateMetadataSize(seqFile);
+ long timeConsumption = System.currentTimeMillis() - startTime;
+ if (timeConsumption > timeLimit) {
+ return Long.MAX_VALUE;
+ }
+ }
+ return cost;
+ }
+
+ private long calculateLooseMemoryCost(TsFileResource tmpSelectedUnseqFile,
+ Collection tmpSelectedSeqFiles, long startTime, long timeLimit) throws IOException {
+ return calculateMemoryCost(tmpSelectedUnseqFile, tmpSelectedSeqFiles,
+ TsFileResource::getFileSize, this::calculateMetadataSize, startTime, timeLimit);
+ }
+
+ private long calculateTightMemoryCost(TsFileResource tmpSelectedUnseqFile,
+ Collection tmpSelectedSeqFiles, long startTime, long timeLimit) throws IOException {
+ return calculateMemoryCost(tmpSelectedUnseqFile, tmpSelectedSeqFiles,
+ this::calculateTightUnseqMemoryCost, this::calculateTightSeqMemoryCost, startTime, timeLimit);
+ }
+
+ private long calculateMetadataSize(TsFileResource seqFile) throws IOException {
+ Long cost = fileMetaSizeMap.get(seqFile);
+ if (cost == null) {
+ cost = MergeUtils.getFileMetaSize(seqFile, resource.getFileReader(seqFile));
+ fileMetaSizeMap.put(seqFile, cost);
+ logger.debug(LOG_FILE_COST, seqFile, cost);
+ }
+ return cost;
+ }
+
+ private long calculateTightFileMemoryCost(TsFileResource seqFile, IFileQueryMemMeasurement measurement)
+ throws IOException {
+ Long cost = maxSeriesQueryCostMap.get(seqFile);
+ if (cost == null) {
+ long[] chunkNums = MergeUtils.findTotalAndLargestSeriesChunkNum(seqFile, resource.getFileReader(seqFile));
+ long totalChunkNum = chunkNums[0];
+ long maxChunkNum = chunkNums[1];
+ cost = measurement.measure(seqFile) * maxChunkNum / totalChunkNum;
+ maxSeriesQueryCostMap.put(seqFile, cost);
+ logger.debug(LOG_FILE_COST, seqFile, cost);
+ }
+ return cost;
+ }
+
+ // this method traverses all ChunkMetadata to find out which series has the most chunks and uses
+ // its proportion to all series to get a maximum estimation
+ private long calculateTightSeqMemoryCost(TsFileResource seqFile) throws IOException {
+ long singleSeriesCost = calculateTightFileMemoryCost(seqFile, this::calculateMetadataSize);
+ long multiSeriesCost = concurrentMergeNum * singleSeriesCost;
+ long maxCost = calculateMetadataSize(seqFile);
+ return multiSeriesCost > maxCost ? maxCost : multiSeriesCost;
+ }
+
+ // this method traverses all ChunkMetadata to find out which series has the most chunks and uses
+ // its proportion among all series to get a maximum estimation
+ private long calculateTightUnseqMemoryCost(TsFileResource unseqFile) throws IOException {
+ long singleSeriesCost = calculateTightFileMemoryCost(unseqFile, TsFileResource::getFileSize);
+ long multiSeriesCost = concurrentMergeNum * singleSeriesCost;
+ long maxCost = unseqFile.getFileSize();
+ return multiSeriesCost > maxCost ? maxCost : multiSeriesCost;
+ }
+
+ @Override
+ public int getConcurrentMergeNum() {
+ return concurrentMergeNum;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxSeriesMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxSeriesMergeFileSelector.java
new file mode 100644
index 0000000000000000000000000000000000000000..cd0186f4a4e0e1c87e314221678b6661724e1d0b
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxSeriesMergeFileSelector.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.selector;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.MergeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MaxSeriesMergeFileSelector is an extension of IMergeFileSelector which tries to maximize the
+ * number of timeseries that can be merged at the same time.
+ */
+public class MaxSeriesMergeFileSelector extends MaxFileMergeFileSelector {
+
+ public static final int MAX_SERIES_NUM = 1024;
+ private static final Logger logger = LoggerFactory.getLogger(MaxSeriesMergeFileSelector.class);
+
+ private List lastSelectedSeqFiles = Collections.emptyList();
+ private List lastSelectedUnseqFiles = Collections.emptyList();
+ private long lastTotalMemoryCost;
+
+ public MaxSeriesMergeFileSelector(
+ MergeResource mergeResource,
+ long memoryBudget) {
+ super(mergeResource, memoryBudget);
+ }
+
+ @Override
+ public List[] select() throws MergeException {
+ long startTime = System.currentTimeMillis();
+ try {
+ logger.info("Selecting merge candidates from {} seqFile, {} unseqFiles", resource.getSeqFiles().size(),
+ resource.getUnseqFiles().size());
+
+ searchMaxSeriesNum();
+ resource.setSeqFiles(selectedSeqFiles);
+ resource.setUnseqFiles(selectedUnseqFiles);
+ resource.removeOutdatedSeqReaders();
+ if (selectedUnseqFiles.isEmpty()) {
+ logger.info("No merge candidates are found");
+ return new List[0];
+ }
+ } catch (IOException e) {
+ throw new MergeException(e);
+ }
+ if (logger.isInfoEnabled()) {
+ logger.info("Selected merge candidates, {} seqFiles, {} unseqFiles, total memory cost {}, "
+ + "concurrent merge num {}" + "time consumption {}ms",
+ selectedSeqFiles.size(), selectedUnseqFiles.size(), totalCost, concurrentMergeNum,
+ System.currentTimeMillis() - startTime);
+ }
+ return new List[]{selectedSeqFiles, selectedUnseqFiles};
+ }
+
+ private void searchMaxSeriesNum() throws IOException {
+ binSearch();
+ }
+
+ private void binSearch() throws IOException {
+ int lb = 0;
+ int ub = MAX_SERIES_NUM + 1;
+ while (true) {
+ int mid = (lb + ub) / 2;
+ if (mid == lb) {
+ break;
+ }
+ concurrentMergeNum = mid;
+ select(false);
+ if (selectedUnseqFiles.isEmpty()) {
+ select(true);
+ }
+ if (selectedUnseqFiles.isEmpty()) {
+ ub = mid;
+ } else {
+ lastSelectedSeqFiles = selectedSeqFiles;
+ lastSelectedUnseqFiles = selectedUnseqFiles;
+ lastTotalMemoryCost = totalCost;
+ lb = mid;
+ }
+ }
+ selectedUnseqFiles = lastSelectedUnseqFiles;
+ selectedSeqFiles = lastSelectedSeqFiles;
+ concurrentMergeNum = lb;
+ totalCost = lastTotalMemoryCost;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileStrategy.java
new file mode 100644
index 0000000000000000000000000000000000000000..98026fb8446be342da7a13a80c404e9afd8324e9
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileStrategy.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.selector;
+
+public enum MergeFileStrategy {
+ MAX_SERIES_NUM,
+ MAX_FILE_NUM,
+ // TODO: HOW?
+ TRADE_OFF,
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/NaivePathSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/NaivePathSelector.java
new file mode 100644
index 0000000000000000000000000000000000000000..6a0aaf7909da88997f3b1ae430049cd38657e86c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/NaivePathSelector.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.selector;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+public class NaivePathSelector implements IMergePathSelector {
+
+ private List paths;
+ private int idx;
+ private int maxSeriesNum;
+
+ public NaivePathSelector(List paths, int maxSeriesNum) {
+ this.paths = paths;
+ this.maxSeriesNum = maxSeriesNum;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return idx < paths.size();
+ }
+
+ @Override
+ public List next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ List ret = idx + maxSeriesNum <= paths.size() ? paths.subList(idx, idx + maxSeriesNum) :
+ paths.subList(idx, paths.size());
+ idx += maxSeriesNum;
+ return ret;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java
new file mode 100644
index 0000000000000000000000000000000000000000..c66edeeada96e2f6b02177979f3d40641c107114
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.task;
+
+import java.io.File;
+import java.util.List;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+
+@FunctionalInterface
+public interface MergeCallback {
+
+ /**
+ * On calling this method, the callee should:
+ * 1. replace the modification files of seqFiles with merging modifications since the old
+ * modifications have been merged into the new files.
+ * 2. remove the unseqFiles since they have been merged into new files.
+ * 3. remove the merge log file
+ * 4. exit merging status
+ * @param seqFiles
+ * @param unseqFiles
+ */
+ void call(List seqFiles, List unseqFiles, File logFile);
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..cf1f62d23ae580c1b5090046b2dc16b50519495a
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.task;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache;
+import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
+import org.apache.iotdb.db.engine.merge.manage.MergeContext;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.schema.Schema;
+import org.apache.iotdb.tsfile.write.writer.ForceAppendTsFileWriter;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MergeFileTask merges the merge temporary files with the seqFiles, either move the merged
+ * chunks in the temp files into the seqFiles or move the unmerged chunks into the merge temp
+ * files, depending on which one is the majority.
+ */
+class MergeFileTask {
+
+ private static final Logger logger = LoggerFactory.getLogger(MergeFileTask.class);
+
+ private String taskName;
+ private MergeContext context;
+ private MergeLogger mergeLogger;
+ private MergeResource resource;
+ private List unmergedFiles;
+
+ MergeFileTask(String taskName, MergeContext context, MergeLogger mergeLogger,
+ MergeResource resource, List unmergedSeqFiles) {
+ this.taskName = taskName;
+ this.context = context;
+ this.mergeLogger = mergeLogger;
+ this.resource = resource;
+ this.unmergedFiles = unmergedSeqFiles;
+ }
+
+ void mergeFiles() throws IOException {
+ // decide whether to write the unmerged chunks to the merge files or to move the merged chunks
+ // back to the origin seqFile's
+ if (logger.isInfoEnabled()) {
+ logger.info("{} starts to merge {} files", taskName, unmergedFiles.size());
+ }
+ long startTime = System.currentTimeMillis();
+ int cnt = 0;
+ for (TsFileResource seqFile : unmergedFiles) {
+ int mergedChunkNum = context.getMergedChunkCnt().getOrDefault(seqFile, 0);
+ int unmergedChunkNum = context.getUnmergedChunkCnt().getOrDefault(seqFile, 0);
+ if (mergedChunkNum >= unmergedChunkNum) {
+ // move the unmerged data to the new file
+ if (logger.isInfoEnabled()) {
+ logger.info("{} moving unmerged data of {} to the merged file, {} merged chunks, {} "
+ + "unmerged chunks", taskName, seqFile.getFile().getName(), mergedChunkNum, unmergedChunkNum);
+ }
+ moveUnmergedToNew(seqFile);
+ } else {
+ // move the merged data to the old file
+ if (logger.isInfoEnabled()) {
+ logger.info("{} moving merged data of {} to the old file {} merged chunks, {} "
+ + "unmerged chunks", taskName, seqFile.getFile().getName(), mergedChunkNum, unmergedChunkNum);
+ }
+ moveMergedToOld(seqFile);
+ }
+ cnt ++;
+ if (logger.isInfoEnabled()) {
+ logger.debug("{} has merged {}/{} files", taskName, cnt, unmergedFiles.size());
+ }
+ }
+ if (logger.isInfoEnabled()) {
+ logger.info("{} has merged all files after {}ms", taskName, System.currentTimeMillis() - startTime);
+ }
+ mergeLogger.logMergeEnd();
+ }
+
+ private void moveMergedToOld(TsFileResource seqFile) throws IOException {
+ int mergedChunkNum = context.getMergedChunkCnt().getOrDefault(seqFile, 0);
+ if (mergedChunkNum == 0) {
+ resource.removeFileAndWriter(seqFile);
+ return;
+ }
+
+ seqFile.getMergeQueryLock().writeLock().lock();
+ try {
+ TsFileMetaDataCache.getInstance().remove(seqFile);
+ DeviceMetaDataCache.getInstance().remove(seqFile);
+ resource.removeFileReader(seqFile);
+ TsFileIOWriter oldFileWriter;
+ try {
+ oldFileWriter = new ForceAppendTsFileWriter(seqFile.getFile());
+ mergeLogger.logFileMergeStart(seqFile.getFile(), ((ForceAppendTsFileWriter) oldFileWriter).getTruncatePosition());
+ logger.debug("{} moving merged chunks of {} to the old file", taskName, seqFile);
+ ((ForceAppendTsFileWriter) oldFileWriter).doTruncate();
+ } catch (TsFileNotCompleteException e) {
+ // this file may already be truncated if this merge is a system reboot merge
+ oldFileWriter = new RestorableTsFileIOWriter(seqFile.getFile());
+ }
+ // filter the chunks that have been merged
+ oldFileWriter.filterChunks(context.getUnmergedChunkStartTimes().get(seqFile));
+
+ RestorableTsFileIOWriter newFileWriter = resource.getMergeFileWriter(seqFile);
+ newFileWriter.close();
+ try (TsFileSequenceReader newFileReader =
+ new TsFileSequenceReader(newFileWriter.getFile().getPath())) {
+ List chunkGroupMetadataList = newFileWriter.getChunkGroupMetaDatas();
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} find {} merged chunk groups", taskName, chunkGroupMetadataList.size());
+ }
+ for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetadataList) {
+ writeMergedChunkGroup(chunkGroupMetaData, newFileReader, oldFileWriter);
+ }
+ }
+ oldFileWriter.endFile(new Schema(newFileWriter.getKnownSchema()));
+
+ seqFile.serialize();
+ mergeLogger.logFileMergeEnd();
+ logger.debug("{} moved merged chunks of {} to the old file", taskName, seqFile);
+
+ newFileWriter.getFile().delete();
+ } finally {
+ seqFile.getMergeQueryLock().writeLock().unlock();
+ }
+ }
+
+ private void writeMergedChunkGroup(ChunkGroupMetaData chunkGroupMetaData,
+ TsFileSequenceReader reader, TsFileIOWriter fileWriter)
+ throws IOException {
+ fileWriter.startChunkGroup(chunkGroupMetaData.getDeviceID());
+ long version = chunkGroupMetaData.getVersion();
+ for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) {
+ Chunk chunk = reader.readMemChunk(chunkMetaData);
+ fileWriter.writeChunk(chunk, chunkMetaData);
+ context.incTotalPointWritten(chunkMetaData.getNumOfPoints());
+ }
+ fileWriter.endChunkGroup(version + 1);
+ }
+
+ private void moveUnmergedToNew(TsFileResource seqFile) throws IOException {
+ Map> fileUnmergedChunkStartTimes =
+ context.getUnmergedChunkStartTimes().get(seqFile);
+ RestorableTsFileIOWriter fileWriter = resource.getMergeFileWriter(seqFile);
+
+ mergeLogger.logFileMergeStart(fileWriter.getFile(), fileWriter.getFile().length());
+ logger.debug("{} moving unmerged chunks of {} to the new file", taskName, seqFile);
+
+ int unmergedChunkNum = context.getUnmergedChunkCnt().getOrDefault(seqFile, 0);
+
+ if (unmergedChunkNum > 0) {
+ for (Entry> entry : fileUnmergedChunkStartTimes.entrySet()) {
+ Path path = entry.getKey();
+ List chunkStartTimes = entry.getValue();
+ if (chunkStartTimes.isEmpty()) {
+ continue;
+ }
+
+ List chunkMetaDataList = resource.queryChunkMetadata(path, seqFile);
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} find {} unmerged chunks", taskName, chunkMetaDataList.size());
+ }
+
+ fileWriter.startChunkGroup(path.getDevice());
+ long maxVersion = writeUnmergedChunks(chunkStartTimes, chunkMetaDataList,
+ resource.getFileReader(seqFile), fileWriter);
+ fileWriter.endChunkGroup(maxVersion + 1);
+ }
+ }
+
+ fileWriter.endFile(new Schema(fileWriter.getKnownSchema()));
+
+ seqFile.serialize();
+ mergeLogger.logFileMergeEnd();
+ logger.debug("{} moved unmerged chunks of {} to the new file", taskName, seqFile);
+
+ seqFile.getMergeQueryLock().writeLock().lock();
+ try {
+ resource.removeFileReader(seqFile);
+ TsFileMetaDataCache.getInstance().remove(seqFile);
+ DeviceMetaDataCache.getInstance().remove(seqFile);
+ seqFile.getFile().delete();
+ FileUtils.moveFile(fileWriter.getFile(), seqFile.getFile());
+ } finally {
+ seqFile.getMergeQueryLock().writeLock().unlock();
+ }
+ }
+
+ private long writeUnmergedChunks(List chunkStartTimes,
+ List chunkMetaDataList, TsFileSequenceReader reader,
+ RestorableTsFileIOWriter fileWriter) throws IOException {
+ long maxVersion = 0;
+ int chunkIdx = 0;
+ for (Long startTime : chunkStartTimes) {
+ for (; chunkIdx < chunkMetaDataList.size(); chunkIdx ++) {
+ ChunkMetaData metaData = chunkMetaDataList.get(chunkIdx);
+ if (metaData.getStartTime() == startTime) {
+ Chunk chunk = reader.readMemChunk(metaData);
+ fileWriter.writeChunk(chunk, metaData);
+ maxVersion = metaData.getVersion() > maxVersion ? metaData.getVersion() : maxVersion;
+ context.incTotalPointWritten(metaData.getNumOfPoints());
+ break;
+ }
+ }
+ }
+ return maxVersion;
+ }
+
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..ce6e602f86195a231eea57b325d68c894376c86f
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java
@@ -0,0 +1,440 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.task;
+
+import static org.apache.iotdb.db.utils.MergeUtils.writeBatchPoint;
+import static org.apache.iotdb.db.utils.MergeUtils.writeTVPair;
+import static org.apache.iotdb.db.utils.QueryUtils.modifyChunkMetaData;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.merge.manage.MergeContext;
+import org.apache.iotdb.db.engine.merge.manage.MergeManager;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
+import org.apache.iotdb.db.engine.merge.selector.IMergePathSelector;
+import org.apache.iotdb.db.engine.merge.selector.NaivePathSelector;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.reader.IPointReader;
+import org.apache.iotdb.db.utils.MergeUtils;
+import org.apache.iotdb.db.utils.MergeUtils.MetaListEntry;
+import org.apache.iotdb.db.utils.TimeValuePair;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter;
+import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class MergeMultiChunkTask {
+
+ private static final Logger logger = LoggerFactory.getLogger(MergeMultiChunkTask.class);
+ private static int minChunkPointNum = IoTDBDescriptor.getInstance().getConfig()
+ .getChunkMergePointThreshold();
+
+ private MergeLogger mergeLogger;
+ private List unmergedSeries;
+
+ private String taskName;
+ private MergeResource resource;
+ private TimeValuePair[] currTimeValuePairs;
+ private boolean fullMerge;
+
+ private MergeContext mergeContext;
+
+ private AtomicInteger mergedChunkNum = new AtomicInteger();
+ private AtomicInteger unmergedChunkNum = new AtomicInteger();
+ private int mergedSeriesCnt;
+ private double progress;
+
+ private int concurrentMergeSeriesNum;
+ private List currMergingPaths = new ArrayList<>();
+
+ MergeMultiChunkTask(MergeContext context, String taskName, MergeLogger mergeLogger,
+ MergeResource mergeResource, boolean fullMerge, List unmergedSeries,
+ int concurrentMergeSeriesNum) {
+ this.mergeContext = context;
+ this.taskName = taskName;
+ this.mergeLogger = mergeLogger;
+ this.resource = mergeResource;
+ this.fullMerge = fullMerge;
+ this.unmergedSeries = unmergedSeries;
+ this.concurrentMergeSeriesNum = concurrentMergeSeriesNum;
+ }
+
+ void mergeSeries() throws IOException {
+ if (logger.isInfoEnabled()) {
+ logger.info("{} starts to merge {} series", taskName, unmergedSeries.size());
+ }
+ long startTime = System.currentTimeMillis();
+ for (TsFileResource seqFile : resource.getSeqFiles()) {
+ mergeContext.getUnmergedChunkStartTimes().put(seqFile, new HashMap<>());
+ }
+ // merge each series and write data into each seqFile's corresponding temp merge file
+ List> devicePaths = MergeUtils.splitPathsByDevice(unmergedSeries);
+ for (List pathList : devicePaths) {
+ // TODO: use statistics of queries to better rearrange series
+ IMergePathSelector pathSelector = new NaivePathSelector(pathList, concurrentMergeSeriesNum);
+ while (pathSelector.hasNext()) {
+ currMergingPaths = pathSelector.next();
+ mergePaths();
+ mergedSeriesCnt += currMergingPaths.size();
+ logMergeProgress();
+ }
+ }
+ if (logger.isInfoEnabled()) {
+ logger.info("{} all series are merged after {}ms", taskName,
+ System.currentTimeMillis() - startTime);
+ }
+ mergeLogger.logAllTsEnd();
+ }
+
+ private void logMergeProgress() {
+ if (logger.isInfoEnabled()) {
+ double newProgress = 100 * mergedSeriesCnt / (double) (unmergedSeries.size());
+ if (newProgress - progress >= 1.0) {
+ progress = newProgress;
+ logger.info("{} has merged {}% series", taskName, progress);
+ }
+ }
+ }
+
+ private void mergePaths() throws IOException {
+ mergeLogger.logTSStart(currMergingPaths);
+ IPointReader[] unseqReaders;
+ unseqReaders = resource.getUnseqReaders(currMergingPaths);
+ currTimeValuePairs = new TimeValuePair[currMergingPaths.size()];
+ for (int i = 0; i < currMergingPaths.size(); i++) {
+ if (unseqReaders[i].hasNext()) {
+ currTimeValuePairs[i] = unseqReaders[i].current();
+ }
+ }
+
+ for (int i = 0; i < resource.getSeqFiles().size(); i++) {
+ pathsMergeOneFile(i, unseqReaders);
+ }
+ mergeLogger.logTSEnd();
+ }
+
+ private void pathsMergeOneFile(int seqFileIdx, IPointReader[] unseqReaders)
+ throws IOException {
+ TsFileResource currTsFile = resource.getSeqFiles().get(seqFileIdx);
+ String deviceId = currMergingPaths.get(0).getDevice();
+ Long currDeviceMinTime = currTsFile.getStartTimeMap().get(deviceId);
+ if (currDeviceMinTime == null) {
+ return;
+ }
+
+ for (Path path : currMergingPaths) {
+ mergeContext.getUnmergedChunkStartTimes().get(currTsFile).put(path, new ArrayList<>());
+ }
+
+ // if this TsFile receives data later than fileLimitTime, it will overlap the next TsFile,
+ // which is forbidden
+ for (TimeValuePair timeValuePair : currTimeValuePairs) {
+ if (timeValuePair != null && timeValuePair.getTimestamp() < currDeviceMinTime) {
+ currDeviceMinTime = timeValuePair.getTimestamp();
+ }
+ }
+ boolean isLastFile = seqFileIdx + 1 == resource.getSeqFiles().size();
+
+ TsFileSequenceReader fileSequenceReader = resource.getFileReader(currTsFile);
+ List[] modifications = new List[currMergingPaths.size()];
+ List[] seqChunkMeta = new List[currMergingPaths.size()];
+ for (int i = 0; i < currMergingPaths.size(); i++) {
+ modifications[i] = resource.getModifications(currTsFile, currMergingPaths.get(i));
+ seqChunkMeta[i] = resource.queryChunkMetadata(currMergingPaths.get(i), currTsFile);
+ modifyChunkMetaData(seqChunkMeta[i], modifications[i]);
+ }
+
+ List unskippedPathIndices = filterNoDataPaths(seqChunkMeta, seqFileIdx);
+ if (unskippedPathIndices.isEmpty()) {
+ return;
+ }
+
+ RestorableTsFileIOWriter mergeFileWriter = resource.getMergeFileWriter(currTsFile);
+ for (Path path : currMergingPaths) {
+ MeasurementSchema schema = resource.getSchema(path.getMeasurement());
+ mergeFileWriter.addSchema(schema);
+ }
+ // merge unseq data with seq data in this file or small chunks in this file into a larger chunk
+ mergeFileWriter.startChunkGroup(deviceId);
+ boolean dataWritten = mergeChunks(seqChunkMeta, isLastFile, fileSequenceReader, unseqReaders,
+ mergeFileWriter, currTsFile);
+ if (dataWritten) {
+ mergeFileWriter.endChunkGroup(0);
+ mergeLogger.logFilePosition(mergeFileWriter.getFile());
+ currTsFile.getStartTimeMap().put(deviceId, currDeviceMinTime);
+ }
+ }
+
+ private List filterNoDataPaths(List[] seqChunkMeta, int seqFileIdx) {
+ // if the last seqFile does not contains this series but the unseqFiles do, data of this
+ // series should also be written into a new chunk
+ List ret = new ArrayList<>();
+ for (int i = 0; i < currMergingPaths.size(); i++) {
+ if (seqChunkMeta[i].isEmpty()
+ && !(seqFileIdx + 1 == resource.getSeqFiles().size() && currTimeValuePairs[i] != null)) {
+ continue;
+ }
+ ret.add(i);
+ }
+ return ret;
+ }
+
+ private boolean mergeChunks(List[] seqChunkMeta, boolean isLastFile,
+ TsFileSequenceReader reader, IPointReader[] unseqReaders,
+ RestorableTsFileIOWriter mergeFileWriter, TsFileResource currFile)
+ throws IOException {
+ int[] ptWrittens = new int[seqChunkMeta.length];
+ int mergeChunkSubTaskNum = IoTDBDescriptor.getInstance().getConfig().getMergeChunkSubThreadNum();
+ PriorityQueue[] chunkMetaHeaps = new PriorityQueue[mergeChunkSubTaskNum];
+ for (int i = 0; i < mergeChunkSubTaskNum; i++) {
+ chunkMetaHeaps[i] = new PriorityQueue<>();
+ }
+ int idx = 0;
+ for (int i = 0; i < currMergingPaths.size(); i++) {
+ if (seqChunkMeta[i].isEmpty()) {
+ continue;
+ }
+ MetaListEntry entry = new MetaListEntry(i, seqChunkMeta[i]);
+ entry.next();
+
+ chunkMetaHeaps[idx % mergeChunkSubTaskNum].add(entry);
+ idx++;
+ ptWrittens[i] = 0;
+ }
+
+ mergedChunkNum.set(0);
+ unmergedChunkNum.set(0);
+
+ List futures = new ArrayList<>();
+ for (int i = 0; i < mergeChunkSubTaskNum; i++) {
+ int finalI = i;
+ futures.add(MergeManager.getINSTANCE().submitChunkSubTask(() -> {
+ mergeChunkHeap(chunkMetaHeaps[finalI], ptWrittens, reader, mergeFileWriter, unseqReaders, currFile,
+ isLastFile);
+ return null;
+ }));
+ }
+ for (int i = 0; i < mergeChunkSubTaskNum; i++) {
+ try {
+ futures.get(i).get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ // add merge and unmerged chunk statistic
+ mergeContext.getMergedChunkCnt().compute(currFile, (tsFileResource, anInt) -> anInt == null ?
+ mergedChunkNum.get() : anInt + mergedChunkNum.get());
+ mergeContext.getUnmergedChunkCnt().compute(currFile, (tsFileResource, anInt) -> anInt == null ?
+ unmergedChunkNum.get() : anInt + unmergedChunkNum.get());
+
+ return mergedChunkNum.get() > 0;
+ }
+
+ private void mergeChunkHeap(PriorityQueue chunkMetaHeap, int[] ptWrittens, TsFileSequenceReader reader,
+ RestorableTsFileIOWriter mergeFileWriter, IPointReader[] unseqReaders, TsFileResource currFile,
+ boolean isLastFile) throws IOException {
+ while (!chunkMetaHeap.isEmpty()) {
+ MetaListEntry metaListEntry = chunkMetaHeap.poll();
+ ChunkMetaData currMeta = metaListEntry.current();
+ int pathIdx = metaListEntry.getPathId();
+ boolean isLastChunk = !metaListEntry.hasNext();
+ Path path = currMergingPaths.get(pathIdx);
+ MeasurementSchema measurementSchema = resource.getSchema(path.getMeasurement());
+ IChunkWriter chunkWriter = resource.getChunkWriter(measurementSchema);
+
+ boolean chunkOverflowed = MergeUtils.isChunkOverflowed(currTimeValuePairs[pathIdx], currMeta);
+ boolean chunkTooSmall = MergeUtils
+ .isChunkTooSmall(ptWrittens[pathIdx], currMeta, isLastChunk, minChunkPointNum);
+
+ Chunk chunk;
+ synchronized (reader) {
+ chunk = reader.readMemChunk(currMeta);
+ }
+ ptWrittens[pathIdx] = mergeChunkV2(currMeta, chunkOverflowed, chunkTooSmall, chunk,
+ ptWrittens[pathIdx], pathIdx, mergeFileWriter, unseqReaders[pathIdx], chunkWriter,
+ currFile);
+
+ if (!isLastChunk) {
+ metaListEntry.next();
+ chunkMetaHeap.add(metaListEntry);
+ } else {
+ // this only happens when the seqFiles do not contain this series, otherwise the remaining
+ // data will be merged with the last chunk in the seqFiles
+ if (isLastFile && currTimeValuePairs[pathIdx] != null) {
+ ptWrittens[pathIdx] += writeRemainingUnseq(chunkWriter, unseqReaders[pathIdx], Long.MAX_VALUE,
+ pathIdx);
+ mergedChunkNum.incrementAndGet();
+ }
+ // the last merged chunk may still be smaller than the threshold, flush it anyway
+ if (ptWrittens[pathIdx] > 0) {
+ synchronized (mergeFileWriter) {
+ chunkWriter.writeToFileWriter(mergeFileWriter);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * merge a sequence chunk SK
+ *
+ * 1. no need to write the chunk to .merge file when:
+ * isn't full merge &
+ * there isn't unclosed chunk before &
+ * SK is big enough &
+ * SK isn't overflowed &
+ * SK isn't modified
+ *
+ *
+ * 2. write SK to .merge.file without compressing when:
+ * is full merge &
+ * there isn't unclosed chunk before &
+ * SK is big enough &
+ * SK isn't overflowed &
+ * SK isn't modified
+ *
+ * 3. other cases: need to unCompress the chunk and write
+ * 3.1 SK isn't overflowed
+ * 3.2 SK is overflowed
+ *
+ */
+ private int mergeChunkV2(ChunkMetaData currMeta, boolean chunkOverflowed,
+ boolean chunkTooSmall,Chunk chunk, int lastUnclosedChunkPoint, int pathIdx,
+ TsFileIOWriter mergeFileWriter, IPointReader unseqReader,
+ IChunkWriter chunkWriter, TsFileResource currFile) throws IOException {
+
+ int unclosedChunkPoint = lastUnclosedChunkPoint;
+ boolean chunkModified = currMeta.getDeletedAt() > Long.MIN_VALUE;
+
+ // no need to write the chunk to .merge file
+ if (!fullMerge && lastUnclosedChunkPoint == 0 && !chunkTooSmall && !chunkOverflowed && !chunkModified) {
+ unmergedChunkNum.incrementAndGet();
+ mergeContext.getUnmergedChunkStartTimes().get(currFile).get(currMergingPaths.get(pathIdx))
+ .add(currMeta.getStartTime());
+ return 0;
+ }
+
+ // write SK to .merge.file without compressing
+ if (fullMerge && lastUnclosedChunkPoint == 0 && !chunkTooSmall && !chunkOverflowed && !chunkModified) {
+ synchronized (mergeFileWriter) {
+ mergeFileWriter.writeChunk(chunk, currMeta);
+ }
+ mergeContext.incTotalPointWritten(currMeta.getNumOfPoints());
+ mergeContext.incTotalChunkWritten();
+ mergedChunkNum.incrementAndGet();
+ return 0;
+ }
+
+ // 3.1 SK isn't overflowed, just uncompress and write sequence chunk
+ if (!chunkOverflowed) {
+ unclosedChunkPoint += MergeUtils.writeChunkWithoutUnseq(chunk, chunkWriter);
+ mergedChunkNum.incrementAndGet();
+ } else {
+ // 3.2 SK is overflowed, uncompress sequence chunk and merge with unseq chunk, then write
+ unclosedChunkPoint += writeChunkWithUnseq(chunk, chunkWriter, unseqReader,
+ currMeta.getEndTime(), pathIdx);
+ mergedChunkNum.incrementAndGet();
+ }
+
+ // update points written statistics
+ mergeContext.incTotalPointWritten(unclosedChunkPoint - lastUnclosedChunkPoint);
+ if (minChunkPointNum > 0 && unclosedChunkPoint >= minChunkPointNum
+ || unclosedChunkPoint > 0 && minChunkPointNum < 0) {
+ // the new chunk's size is large enough and it should be flushed
+ synchronized (mergeFileWriter) {
+ chunkWriter.writeToFileWriter(mergeFileWriter);
+ }
+ unclosedChunkPoint = 0;
+ }
+ return unclosedChunkPoint;
+ }
+
+ private int writeRemainingUnseq(IChunkWriter chunkWriter,
+ IPointReader unseqReader, long timeLimit, int pathIdx) throws IOException {
+ int ptWritten = 0;
+ while (currTimeValuePairs[pathIdx] != null
+ && currTimeValuePairs[pathIdx].getTimestamp() < timeLimit) {
+ writeTVPair(currTimeValuePairs[pathIdx], chunkWriter);
+ ptWritten++;
+ unseqReader.next();
+ currTimeValuePairs[pathIdx] = unseqReader.hasNext() ? unseqReader.current() : null;
+ }
+ return ptWritten;
+ }
+
+ private int writeChunkWithUnseq(Chunk chunk, IChunkWriter chunkWriter, IPointReader unseqReader,
+ long chunkLimitTime, int pathIdx) throws IOException {
+ int cnt = 0;
+ ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk);
+ while (chunkReader.hasNextBatch()) {
+ BatchData batchData = chunkReader.nextBatch();
+ cnt += mergeWriteBatch(batchData, chunkWriter, unseqReader, pathIdx);
+ }
+ cnt += writeRemainingUnseq(chunkWriter, unseqReader, chunkLimitTime, pathIdx);
+ return cnt;
+ }
+
+ private int mergeWriteBatch(BatchData batchData, IChunkWriter chunkWriter,
+ IPointReader unseqReader, int pathIdx) throws IOException {
+ int cnt = 0;
+ for (int i = 0; i < batchData.length(); i++) {
+ long time = batchData.getTimeByIndex(i);
+ // merge data in batch and data in unseqReader
+
+ boolean overwriteSeqPoint = false;
+ // unseq point.time <= sequence point.time, write unseq point
+ while (currTimeValuePairs[pathIdx] != null
+ && currTimeValuePairs[pathIdx].getTimestamp() <= time) {
+ writeTVPair(currTimeValuePairs[pathIdx], chunkWriter);
+ if (currTimeValuePairs[pathIdx].getTimestamp() == time) {
+ overwriteSeqPoint = true;
+ }
+ unseqReader.next();
+ currTimeValuePairs[pathIdx] = unseqReader.hasNext() ? unseqReader.current() : null;
+ cnt++;
+ }
+ // unseq point.time > sequence point.time, write seq point
+ if (!overwriteSeqPoint) {
+ writeBatchPoint(batchData, i, chunkWriter);
+ cnt++;
+ }
+ }
+ return cnt;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..9e99747e8a64792e19007f3f78830374e05b5fc0
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.task;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.iotdb.db.engine.merge.manage.MergeContext;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.MetadataErrorException;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.utils.MergeUtils;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MergeTask merges given seqFiles and unseqFiles into new ones, which basically consists of three
+ * steps: 1. rewrite overflowed, modified or small-sized chunks into temp merge files
+ * 2. move the merged chunks in the temp files back to the seqFiles or move the unmerged
+ * chunks in the seqFiles into temp files and replace the seqFiles with the temp files.
+ * 3. remove unseqFiles
+ */
+public class MergeTask implements Callable {
+
+ public static final String MERGE_SUFFIX = ".merge";
+ private static final Logger logger = LoggerFactory.getLogger(MergeTask.class);
+
+ MergeResource resource;
+ String storageGroupSysDir;
+ String storageGroupName;
+ MergeLogger mergeLogger;
+ MergeContext mergeContext = new MergeContext();
+
+ private MergeCallback callback;
+ int concurrentMergeSeriesNum;
+ String taskName;
+ boolean fullMerge;
+
+ MergeTask(List seqFiles,
+ List unseqFiles, String storageGroupSysDir, MergeCallback callback,
+ String taskName, boolean fullMerge, String storageGroupName) {
+ this.resource = new MergeResource(seqFiles, unseqFiles);
+ this.storageGroupSysDir = storageGroupSysDir;
+ this.callback = callback;
+ this.taskName = taskName;
+ this.fullMerge = fullMerge;
+ this.concurrentMergeSeriesNum = 1;
+ this.storageGroupName = storageGroupName;
+ }
+
+ public MergeTask(MergeResource mergeResource, String storageGroupSysDir, MergeCallback callback,
+ String taskName, boolean fullMerge, int concurrentMergeSeriesNum, String storageGroupName) {
+ this.resource = mergeResource;
+ this.storageGroupSysDir = storageGroupSysDir;
+ this.callback = callback;
+ this.taskName = taskName;
+ this.fullMerge = fullMerge;
+ this.concurrentMergeSeriesNum = concurrentMergeSeriesNum;
+ this.storageGroupName = storageGroupName;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ try {
+ doMerge();
+ } catch (Exception e) {
+ logger.error("Runtime exception in merge {}", taskName, e);
+ cleanUp(false);
+ // call the callback to make sure the StorageGroup exit merging status, but passing 2
+ // empty file lists to avoid files being deleted.
+ callback.call(Collections.emptyList(), Collections.emptyList(), new File(storageGroupSysDir, MergeLogger.MERGE_LOG_NAME));
+ throw e;
+ }
+ return null;
+ }
+
+ private void doMerge() throws IOException, MetadataErrorException {
+ if (logger.isInfoEnabled()) {
+ logger.info("{} starts to merge {} seqFiles, {} unseqFiles", taskName,
+ resource.getSeqFiles().size(), resource.getUnseqFiles().size());
+ }
+ long startTime = System.currentTimeMillis();
+ long totalFileSize = MergeUtils.collectFileSizes(resource.getSeqFiles(),
+ resource.getUnseqFiles());
+ mergeLogger = new MergeLogger(storageGroupSysDir);
+
+ mergeLogger.logFiles(resource);
+
+ List measurementSchemas = MManager.getInstance()
+ .getSchemaForStorageGroup(storageGroupName);
+ resource.addMeasurements(measurementSchemas);
+
+ List storageGroupPaths = MManager.getInstance().getPaths(storageGroupName + ".*");
+ List unmergedSeries = new ArrayList<>();
+ for (String path : storageGroupPaths) {
+ unmergedSeries.add(new Path(path));
+ }
+
+ mergeLogger.logMergeStart();
+
+ MergeMultiChunkTask mergeChunkTask = new MergeMultiChunkTask(mergeContext, taskName, mergeLogger, resource,
+ fullMerge, unmergedSeries, concurrentMergeSeriesNum);
+ mergeChunkTask.mergeSeries();
+
+ MergeFileTask mergeFileTask = new MergeFileTask(taskName, mergeContext, mergeLogger, resource,
+ resource.getSeqFiles());
+ mergeFileTask.mergeFiles();
+
+ cleanUp(true);
+ if (logger.isInfoEnabled()) {
+ double elapsedTime = (double) (System.currentTimeMillis() - startTime) / 1000.0;
+ double byteRate = totalFileSize / elapsedTime / 1024 / 1024;
+ double seriesRate = unmergedSeries.size() / elapsedTime;
+ double chunkRate = mergeContext.getTotalChunkWritten() / elapsedTime;
+ double fileRate =
+ (resource.getSeqFiles().size() + resource.getUnseqFiles().size()) / elapsedTime;
+ double ptRate = mergeContext.getTotalPointWritten() / elapsedTime;
+ logger.info("{} ends after {}s, byteRate: {}MB/s, seriesRate {}/s, chunkRate: {}/s, "
+ + "fileRate: {}/s, ptRate: {}/s",
+ taskName, elapsedTime, byteRate, seriesRate, chunkRate, fileRate, ptRate);
+ }
+ }
+
+ void cleanUp(boolean executeCallback) throws IOException {
+ logger.info("{} is cleaning up", taskName);
+
+ resource.clear();
+ mergeContext.clear();
+
+ if (mergeLogger != null) {
+ mergeLogger.close();
+ }
+
+ for (TsFileResource seqFile : resource.getSeqFiles()) {
+ File mergeFile = new File(seqFile.getFile().getPath() + MERGE_SUFFIX);
+ mergeFile.delete();
+ }
+
+ File logFile = new File(storageGroupSysDir, MergeLogger.MERGE_LOG_NAME);
+ if (executeCallback) {
+ // make sure merge.log is not deleted until unseqFiles are cleared so that when system
+ // reboots, the undeleted files can be deleted again
+ callback.call(resource.getSeqFiles(), resource.getUnseqFiles(), logFile);
+ } else {
+ logFile.delete();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..a69571ad61ec0d1518f6459a592e73797d0a1707
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.task;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map.Entry;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.merge.recover.LogAnalyzer;
+import org.apache.iotdb.db.engine.merge.recover.LogAnalyzer.Status;
+import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
+import org.apache.iotdb.db.engine.merge.selector.MaxSeriesMergeFileSelector;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.MetadataErrorException;
+import org.apache.iotdb.db.utils.MergeUtils;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * RecoverMergeTask is an extension of MergeTask, which resumes the last merge progress by
+ * scanning merge.log using LogAnalyzer and continue the unfinished merge.
+ */
+public class RecoverMergeTask extends MergeTask {
+
+ private static final Logger logger = LoggerFactory.getLogger(RecoverMergeTask.class);
+
+ private LogAnalyzer analyzer;
+
+ public RecoverMergeTask(List seqFiles,
+ List unseqFiles, String storageGroupSysDir,
+ MergeCallback callback, String taskName,
+ boolean fullMerge, String storageGroupName) {
+ super(seqFiles, unseqFiles, storageGroupSysDir, callback, taskName, fullMerge, storageGroupName);
+ }
+
+ public void recoverMerge(boolean continueMerge) throws IOException, MetadataErrorException {
+ File logFile = new File(storageGroupSysDir, MergeLogger.MERGE_LOG_NAME);
+ if (!logFile.exists()) {
+ logger.info("{} no merge.log, merge recovery ends", taskName);
+ return;
+ }
+ long startTime = System.currentTimeMillis();
+
+ analyzer = new LogAnalyzer(resource, taskName, logFile, storageGroupName);
+ Status status = analyzer.analyze();
+ if (logger.isInfoEnabled()) {
+ logger.info("{} merge recovery status determined: {} after {}ms", taskName, status,
+ (System.currentTimeMillis() - startTime));
+ }
+ switch (status) {
+ case NONE:
+ logFile.delete();
+ break;
+ case MERGE_START:
+ resumeAfterFilesLogged(continueMerge);
+ break;
+ case ALL_TS_MERGED:
+ resumeAfterAllTsMerged(continueMerge);
+ break;
+ case MERGE_END:
+ cleanUp(continueMerge);
+ break;
+ default:
+ throw new UnsupportedOperationException(taskName + " found unrecognized status " + status);
+ }
+ if (logger.isInfoEnabled()) {
+ logger.info("{} merge recovery ends after {}ms", taskName,
+ (System.currentTimeMillis() - startTime));
+ }
+ }
+
+ private void resumeAfterFilesLogged(boolean continueMerge) throws IOException {
+ if (continueMerge) {
+ resumeMergeProgress();
+ calculateConcurrentSeriesNum();
+ if (concurrentMergeSeriesNum == 0) {
+ throw new IOException("Merge cannot be resumed under current memory budget, please "
+ + "increase the budget or disable continueMergeAfterReboot");
+ }
+
+ MergeMultiChunkTask mergeChunkTask = new MergeMultiChunkTask(mergeContext, taskName, mergeLogger, resource,
+ fullMerge, analyzer.getUnmergedPaths(), concurrentMergeSeriesNum);
+ analyzer.setUnmergedPaths(null);
+ mergeChunkTask.mergeSeries();
+
+ MergeFileTask mergeFileTask = new MergeFileTask(taskName, mergeContext, mergeLogger, resource,
+ resource.getSeqFiles());
+ mergeFileTask.mergeFiles();
+ }
+ cleanUp(continueMerge);
+ }
+
+ private void resumeAfterAllTsMerged(boolean continueMerge) throws IOException {
+ if (continueMerge) {
+ resumeMergeProgress();
+ MergeFileTask mergeFileTask = new MergeFileTask(taskName, mergeContext, mergeLogger, resource,
+ analyzer.getUnmergedFiles());
+ analyzer.setUnmergedFiles(null);
+ mergeFileTask.mergeFiles();
+ } else {
+ // NOTICE: although some of the seqFiles may have been truncated in last merge, we do not
+ // recover them here because later TsFile recovery will recover them
+ truncateFiles();
+ }
+ cleanUp(continueMerge);
+ }
+
+ private void resumeMergeProgress() throws IOException {
+ mergeLogger = new MergeLogger(storageGroupSysDir);
+ truncateFiles();
+ recoverChunkCounts();
+ }
+
+ private void calculateConcurrentSeriesNum() throws IOException {
+ long singleSeriesUnseqCost = 0;
+ long maxUnseqCost = 0;
+ for (TsFileResource unseqFile : resource.getUnseqFiles()) {
+ long[] chunkNums = MergeUtils.findTotalAndLargestSeriesChunkNum(unseqFile,
+ resource.getFileReader(unseqFile));
+ long totalChunkNum = chunkNums[0];
+ long maxChunkNum = chunkNums[1];
+ singleSeriesUnseqCost += unseqFile.getFileSize() * maxChunkNum / totalChunkNum;
+ maxUnseqCost += unseqFile.getFileSize();
+ }
+
+ long singleSeriesSeqReadCost = 0;
+ long maxSeqReadCost = 0;
+ long seqWriteCost = 0;
+ for (TsFileResource seqFile : resource.getSeqFiles()) {
+ long[] chunkNums = MergeUtils.findTotalAndLargestSeriesChunkNum(seqFile,
+ resource.getFileReader(seqFile));
+ long totalChunkNum = chunkNums[0];
+ long maxChunkNum = chunkNums[1];
+ long fileMetaSize = MergeUtils.getFileMetaSize(seqFile, resource.getFileReader(seqFile));
+ long newSingleSeriesSeqReadCost = fileMetaSize * maxChunkNum / totalChunkNum;
+ singleSeriesSeqReadCost = newSingleSeriesSeqReadCost > singleSeriesSeqReadCost ?
+ newSingleSeriesSeqReadCost : singleSeriesSeqReadCost;
+ maxSeqReadCost = fileMetaSize > maxSeqReadCost ? fileMetaSize : maxSeqReadCost;
+ seqWriteCost += fileMetaSize;
+ }
+
+ long memBudget = IoTDBDescriptor.getInstance().getConfig().getMergeMemoryBudget();
+ int lb = 0;
+ int ub = MaxSeriesMergeFileSelector.MAX_SERIES_NUM;
+ int mid = (lb + ub) / 2;
+ while (mid != lb) {
+ long unseqCost = singleSeriesUnseqCost * mid < maxUnseqCost ? singleSeriesUnseqCost * mid :
+ maxUnseqCost;
+ long seqReadCos = singleSeriesSeqReadCost * mid < maxSeqReadCost ?
+ singleSeriesSeqReadCost * mid : maxSeqReadCost;
+ long totalCost = unseqCost + seqReadCos + seqWriteCost;
+ if (totalCost <= memBudget) {
+ lb = mid;
+ } else {
+ ub = mid;
+ }
+ mid = (lb + ub) / 2;
+ }
+ concurrentMergeSeriesNum = lb;
+ }
+
+ // scan the metadata to compute how many chunks are merged/unmerged so at last we can decide to
+ // move the merged chunks or the unmerged chunks
+ private void recoverChunkCounts() throws IOException {
+ logger.info("{} recovering chunk counts", taskName);
+ int fileCnt = 1;
+ for (TsFileResource tsFileResource : resource.getSeqFiles()) {
+ logger.info("{} recovering {} {}/{}", taskName, tsFileResource.getFile().getName(),
+ fileCnt, resource.getSeqFiles().size());
+ RestorableTsFileIOWriter mergeFileWriter = resource.getMergeFileWriter(tsFileResource);
+ mergeFileWriter.makeMetadataVisible();
+ mergeContext.getUnmergedChunkStartTimes().put(tsFileResource, new HashMap<>());
+ List pathsToRecover = analyzer.getMergedPaths();
+ int cnt = 0;
+ double progress = 0.0;
+ for(Path path : pathsToRecover) {
+ recoverChunkCounts(path, tsFileResource, mergeFileWriter);
+ if (logger.isInfoEnabled()) {
+ cnt += 1.0;
+ double newProgress = 100.0 * cnt / pathsToRecover.size();
+ if (newProgress - progress >= 1.0) {
+ progress = newProgress;
+ logger.info("{} {}% series count of {} are recovered", taskName, progress,
+ tsFileResource.getFile().getName());
+ }
+ }
+ }
+ fileCnt++;
+ }
+ analyzer.setMergedPaths(null);
+ }
+
+ private void recoverChunkCounts(Path path, TsFileResource tsFileResource,
+ RestorableTsFileIOWriter mergeFileWriter) throws IOException {
+ mergeContext.getUnmergedChunkStartTimes().get(tsFileResource).put(path, new ArrayList<>());
+
+ List seqFileChunks = resource.queryChunkMetadata(path, tsFileResource);
+ List mergeFileChunks =
+ mergeFileWriter.getVisibleMetadataList(path.getDevice(), path.getMeasurement(), null);
+ mergeContext.getMergedChunkCnt().compute(tsFileResource, (k, v) -> v == null ?
+ mergeFileChunks.size() : v + mergeFileChunks.size());
+ int seqChunkIndex = 0;
+ int mergeChunkIndex = 0;
+ int unmergedCnt = 0;
+ while (seqChunkIndex < seqFileChunks.size() && mergeChunkIndex < mergeFileChunks.size()) {
+ ChunkMetaData seqChunk = seqFileChunks.get(seqChunkIndex);
+ ChunkMetaData mergedChunk = mergeFileChunks.get(mergeChunkIndex);
+ if (seqChunk.getStartTime() < mergedChunk.getStartTime()) {
+ // this seqChunk is unmerged
+ unmergedCnt ++;
+ seqChunkIndex ++;
+ mergeContext.getUnmergedChunkStartTimes().get(tsFileResource).get(path).add(seqChunk.getStartTime());
+ } else if (mergedChunk.getStartTime() <= seqChunk.getStartTime() &&
+ seqChunk.getStartTime() <= mergedChunk.getEndTime()) {
+ // this seqChunk is merged
+ seqChunkIndex ++;
+ } else {
+ // seqChunk.startTime > mergeChunk.endTime, find next mergedChunk that may cover the
+ // seqChunk
+ mergeChunkIndex ++;
+ }
+ }
+ int finalUnmergedCnt = unmergedCnt;
+ mergeContext.getUnmergedChunkCnt().compute(tsFileResource, (k, v) -> v == null ?
+ finalUnmergedCnt : v + finalUnmergedCnt);
+ }
+
+ private void truncateFiles() throws IOException {
+ logger.info("{} truncating {} files", taskName, analyzer.getFileLastPositions().size());
+ for (Entry entry : analyzer.getFileLastPositions().entrySet()) {
+ File file = entry.getKey();
+ Long lastPosition = entry.getValue();
+ if (file.exists() && file.length() != lastPosition) {
+ try (FileInputStream fileInputStream = new FileInputStream(file)) {
+ FileChannel channel = fileInputStream.getChannel();
+ channel.truncate(lastPosition);
+ channel.close();
+ }
+ }
+ }
+ analyzer.setFileLastPositions(null);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java b/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java
index 964d89e52b975c11cdcad036e55504c16efc551f..26438a85b30e1339e708db7333951d50402f7c15 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.engine.modification;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -117,4 +118,10 @@ public class ModificationFile {
public void setFilePath(String filePath) {
this.filePath = filePath;
}
+
+ public void remove() throws IOException {
+ close();
+ new File(filePath).delete();
+ }
+
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 6661736168d923f3b556a6779dbe8e69fce7e683..1e41c57db7816d11800f751e7cc84ba30ef2cb01 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.db.engine.storagegroup;
+import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX;
+import static org.apache.iotdb.db.engine.storagegroup.TsFileResource.TEMP_SUFFIX;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
import java.io.File;
@@ -34,25 +36,38 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
+import org.apache.iotdb.db.engine.merge.manage.MergeManager;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.merge.selector.IMergeFileSelector;
+import org.apache.iotdb.db.engine.merge.selector.MaxFileMergeFileSelector;
+import org.apache.iotdb.db.engine.merge.selector.MaxSeriesMergeFileSelector;
+import org.apache.iotdb.db.engine.merge.selector.MergeFileStrategy;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
+import org.apache.iotdb.db.engine.merge.task.RecoverMergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.exception.MergeException;
+import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.StorageGroupProcessorException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.JobFileManager;
+import org.apache.iotdb.rpc.TSStatusType;
import org.apache.iotdb.db.utils.CopyOnReadLinkedList;
import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
@@ -61,7 +76,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,10 +103,11 @@ import org.slf4j.LoggerFactory;
*/
public class StorageGroupProcessor {
+ private static final String MERGING_MODIFICAITON_FILE_NAME = "merge.mods";
private static final Logger logger = LoggerFactory.getLogger(StorageGroupProcessor.class);
/**
* a read write lock for guaranteeing concurrent safety when accessing all fields in this class
- * (i.e., fileSchema, (un)sequenceFileList, work(un)SequenceTsFileProcessor,
+ * (i.e., schema, (un)sequenceFileList, work(un)SequenceTsFileProcessor,
* closing(Un)SequenceTsFileProcessor, latestTimeForEachDevice, and
* latestFlushedTimeForEachDevice)
*/
@@ -107,7 +123,7 @@ public class StorageGroupProcessor {
/**
* the schema of time series that belong this storage group
*/
- private FileSchema fileSchema;
+ private Schema schema;
// includes sealed and unsealed sequence TsFiles
private List sequenceFileList = new ArrayList<>();
private TsFileProcessor workSequenceTsFileProcessor = null;
@@ -130,6 +146,8 @@ public class StorageGroupProcessor {
*/
private Map latestFlushedTimeForEachDevice = new HashMap<>();
private String storageGroupName;
+ private File storageGroupSysDir;
+
/**
* versionController assigns a version for each MemTable and deletion/update such that after they
* are persisted, the order of insertions, deletions and updates can be re-determined.
@@ -137,12 +155,10 @@ public class StorageGroupProcessor {
private VersionController versionController;
/**
- * mergeDeleteLock is to be used in the merge process. Concurrent deletion and merge may result in
- * losing some deletion in the merged new file, so a lock is necessary. TODO reconsidering this
- * when implementing the merge process.
+ * mergeLock is to be used in the merge process. Concurrent queries, deletions and merges may
+ * result in losing some deletion in the merged new file, so a lock is necessary.
*/
- @SuppressWarnings("unused") // to be used in merge
- private ReentrantLock mergeDeleteLock = new ReentrantLock();
+ private ReentrantReadWriteLock mergeLock = new ReentrantReadWriteLock();
/**
* This is the modification file of the result of the current merge. Because the merged file may
@@ -150,6 +166,9 @@ public class StorageGroupProcessor {
*/
private ModificationFile mergingModification;
+ private volatile boolean isMerging = false;
+ private long mergeStartTime;
+
/**
* This linked list records the access order of measurements used by query.
*/
@@ -162,15 +181,15 @@ public class StorageGroupProcessor {
this.storageGroupName = storageGroupName;
// construct the file schema
- this.fileSchema = constructFileSchema(storageGroupName);
+ this.schema = constructSchema(storageGroupName);
try {
- File storageGroupSysDir = new File(systemInfoDir, storageGroupName);
+ storageGroupSysDir = new File(systemInfoDir, storageGroupName);
if (storageGroupSysDir.mkdirs()) {
logger.info("Storage Group system Directory {} doesn't exist, create it",
storageGroupSysDir.getPath());
} else if (!storageGroupSysDir.exists()) {
- logger.error("craete Storage Group system Directory {} failed",
+ logger.error("create Storage Group system Directory {} failed",
storageGroupSysDir.getPath());
}
@@ -185,13 +204,31 @@ public class StorageGroupProcessor {
private void recover() throws ProcessorException {
logger.info("recover Storage Group {}", storageGroupName);
- // collect TsFiles from sequential data directory
- List tsFiles = getAllFiles(DirectoryManager.getInstance().getAllSequenceFileFolders());
- recoverSeqFiles(tsFiles);
-
- // collect TsFiles from unsequential data directory
- tsFiles = getAllFiles(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
- recoverUnseqFiles(tsFiles);
+ try {
+ // collect TsFiles from sequential and unsequential data directory
+ List seqTsFiles = getAllFiles(DirectoryManager.getInstance().getAllSequenceFileFolders());
+ List unseqTsFiles =
+ getAllFiles(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
+
+ recoverSeqFiles(seqTsFiles);
+ recoverUnseqFiles(unseqTsFiles);
+
+ String taskName = storageGroupName + "-" + System.currentTimeMillis();
+ File mergingMods = new File(storageGroupSysDir, MERGING_MODIFICAITON_FILE_NAME);
+ if (mergingMods.exists()) {
+ mergingModification = new ModificationFile(mergingMods.getPath());
+ }
+ RecoverMergeTask recoverMergeTask = new RecoverMergeTask(seqTsFiles, unseqTsFiles,
+ storageGroupSysDir.getPath(), this::mergeEndAction, taskName,
+ IoTDBDescriptor.getInstance().getConfig().isForceFullMerge(), storageGroupName);
+ logger.info("{} a RecoverMergeTask {} starts...", storageGroupName, taskName);
+ recoverMergeTask.recoverMerge(IoTDBDescriptor.getInstance().getConfig().isContinueMergeAfterReboot());
+ if (!IoTDBDescriptor.getInstance().getConfig().isContinueMergeAfterReboot()) {
+ mergingMods.delete();
+ }
+ } catch (IOException | MetadataErrorException e) {
+ throw new ProcessorException(e);
+ }
for (TsFileResource resource : sequenceFileList) {
latestTimeForEachDevice.putAll(resource.getEndTimeMap());
@@ -199,39 +236,63 @@ public class StorageGroupProcessor {
}
}
- private List getAllFiles(List folders) {
+ private List getAllFiles(List folders) throws IOException {
List tsFiles = new ArrayList<>();
for (String baseDir : folders) {
File fileFolder = new File(baseDir, storageGroupName);
if (!fileFolder.exists()) {
continue;
}
+ // some TsFileResource may be being persisted when the system crashed, try recovering such
+ // resources
+ continueFailedRenames(fileFolder, TEMP_SUFFIX);
+
+ // some TsFiles were going to be replaced by the merged files when the system crashed and
+ // the process was interrupted before the merged files could be named
+ continueFailedRenames(fileFolder, MERGE_SUFFIX);
+
Collections
.addAll(tsFiles, fileFolder.listFiles(file -> file.getName().endsWith(TSFILE_SUFFIX)));
}
- return tsFiles;
+ tsFiles.sort(this::compareFileName);
+ List ret = new ArrayList<>();
+ tsFiles.forEach(f -> ret.add(new TsFileResource(f)));
+ return ret;
}
- private void recoverSeqFiles(List tsFiles) throws ProcessorException {
- tsFiles.sort(this::compareFileName);
- for (File tsFile : tsFiles) {
- TsFileResource tsFileResource = new TsFileResource(tsFile);
+ private void continueFailedRenames(File fileFolder, String suffix) {
+ File[] files = fileFolder.listFiles(file -> file.getName().endsWith(suffix));
+ if (files != null) {
+ for (File tempResource : files) {
+ File originResource = new File(tempResource.getPath().replace(suffix, ""));
+ if (originResource.exists()) {
+ tempResource.delete();
+ } else {
+ tempResource.renameTo(originResource);
+ }
+ }
+ }
+ }
+
+ private void recoverSeqFiles(List tsFiles) throws ProcessorException {
+
+ for (TsFileResource tsFileResource : tsFiles) {
sequenceFileList.add(tsFileResource);
TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(storageGroupName + "-"
- , fileSchema, versionController, tsFileResource, false);
+ , schema, versionController, tsFileResource, false);
recoverPerformer.recover();
+ tsFileResource.setClosed(true);
}
}
- private void recoverUnseqFiles(List tsFiles) throws ProcessorException {
- tsFiles.sort(this::compareFileName);
- for (File tsFile : tsFiles) {
- TsFileResource tsFileResource = new TsFileResource(tsFile);
+ private void recoverUnseqFiles(List tsFiles) throws ProcessorException {
+ for (TsFileResource tsFileResource : tsFiles) {
unSequenceFileList.add(tsFileResource);
TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(storageGroupName + "-",
- fileSchema,
+ schema,
versionController, tsFileResource, true);
recoverPerformer.recover();
+ tsFileResource.setClosed(true);
}
}
@@ -247,11 +308,11 @@ public class StorageGroupProcessor {
}
}
- private FileSchema constructFileSchema(String storageGroupName) {
+ private Schema constructSchema(String storageGroupName) {
List columnSchemaList;
columnSchemaList = MManager.getInstance().getSchemaForStorageGroup(storageGroupName);
- FileSchema schema = new FileSchema();
+ Schema schema = new Schema();
for (MeasurementSchema measurementSchema : columnSchemaList) {
schema.registerMeasurement(measurementSchema);
}
@@ -260,13 +321,13 @@ public class StorageGroupProcessor {
/**
- * add a measurement into the fileSchema.
+ * add a measurement into the schema.
*/
public void addMeasurement(String measurementId, TSDataType dataType, TSEncoding encoding,
CompressionType compressor, Map props) {
writeLock();
try {
- fileSchema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding,
+ schema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding,
compressor, props));
} finally {
writeUnlock();
@@ -283,40 +344,84 @@ public class StorageGroupProcessor {
// insert to sequence or unSequence file
return insertToTsFileProcessor(insertPlan,
insertPlan.getTime() > latestFlushedTimeForEachDevice.get(insertPlan.getDeviceId()));
- } catch (IOException e) {
- logger.error("insert tsRecord to unsealed data file failed, because {}", e.getMessage(), e);
- return false;
} finally {
writeUnlock();
}
}
- private boolean insertToTsFileProcessor(InsertPlan insertPlan, boolean sequence)
- throws IOException {
- TsFileProcessor tsFileProcessor;
- boolean result;
-
+ public Integer[] insertBatch(BatchInsertPlan batchInsertPlan) {
+ writeLock();
try {
- if (sequence) {
- if (workSequenceTsFileProcessor == null) {
- // create a new TsfileProcessor
- workSequenceTsFileProcessor = createTsFileProcessor(true);
- sequenceFileList.add(workSequenceTsFileProcessor.getTsFileResource());
+ // init map
+ latestTimeForEachDevice.putIfAbsent(batchInsertPlan.getDeviceId(), Long.MIN_VALUE);
+ latestFlushedTimeForEachDevice.putIfAbsent(batchInsertPlan.getDeviceId(), Long.MIN_VALUE);
+
+ Integer[] results = new Integer[batchInsertPlan.getRowCount()];
+ List sequenceIndexes = new ArrayList<>();
+ List unsequenceIndexes = new ArrayList<>();
+
+ for (int i = 0; i < batchInsertPlan.getRowCount(); i++) {
+ results[i] = TSStatusType.SUCCESS_STATUS.getStatusCode();
+ if (batchInsertPlan.getTimes()[i] > latestFlushedTimeForEachDevice
+ .get(batchInsertPlan.getDeviceId())) {
+ sequenceIndexes.add(i);
+ } else {
+ unsequenceIndexes.add(i);
}
- tsFileProcessor = workSequenceTsFileProcessor;
+ }
+
+ if (!sequenceIndexes.isEmpty()) {
+ insertBatchToTsFileProcessor(batchInsertPlan, sequenceIndexes, true, results);
+ }
+
+ if (!unsequenceIndexes.isEmpty()) {
+ insertBatchToTsFileProcessor(batchInsertPlan, unsequenceIndexes, false, results);
+ }
+ return results;
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ private void insertBatchToTsFileProcessor(BatchInsertPlan batchInsertPlan,
+ List indexes, boolean sequence, Integer[] results) {
+
+ TsFileProcessor tsFileProcessor = getOrCreateTsFileProcessor(sequence);
+ if (tsFileProcessor == null) {
+ for (int index : indexes) {
+ results[index] = TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode();
+ }
+ return;
+ }
+
+ boolean result = tsFileProcessor.insertBatch(batchInsertPlan, indexes, results);
+
+ // try to update the latest time of the device of this tsRecord
+ if (result && latestTimeForEachDevice.get(batchInsertPlan.getDeviceId()) < batchInsertPlan.getMaxTime()) {
+ latestTimeForEachDevice.put(batchInsertPlan.getDeviceId(), batchInsertPlan.getMaxTime());
+ }
+
+ // check memtable size and may asyncTryToFlush the work memtable
+ if (tsFileProcessor.shouldFlush()) {
+ logger.info("The memtable size {} reaches the threshold, async flush it to tsfile: {}",
+ tsFileProcessor.getWorkMemTableMemory(),
+ tsFileProcessor.getTsFileResource().getFile().getAbsolutePath());
+
+ if (tsFileProcessor.shouldClose()) {
+ moveOneWorkProcessorToClosingList(sequence);
} else {
- if (workUnSequenceTsFileProcessor == null) {
- // create a new TsfileProcessor
- workUnSequenceTsFileProcessor = createTsFileProcessor(false);
- unSequenceFileList.add(workUnSequenceTsFileProcessor.getTsFileResource());
- }
- tsFileProcessor = workUnSequenceTsFileProcessor;
+ tsFileProcessor.asyncFlush();
}
- } catch (DiskSpaceInsufficientException e) {
- logger.error(
- "disk space is insufficient when creating TsFile processor, change system mode to read-only",
- e);
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
+ }
+ }
+
+ private boolean insertToTsFileProcessor(InsertPlan insertPlan, boolean sequence) {
+ TsFileProcessor tsFileProcessor;
+ boolean result;
+
+ tsFileProcessor = getOrCreateTsFileProcessor(sequence);
+
+ if (tsFileProcessor == null) {
return false;
}
@@ -343,6 +448,37 @@ public class StorageGroupProcessor {
return result;
}
+ private TsFileProcessor getOrCreateTsFileProcessor(boolean sequence) {
+ TsFileProcessor tsFileProcessor = null;
+ try {
+ if (sequence) {
+ if (workSequenceTsFileProcessor == null) {
+ // create a new TsfileProcessor
+ workSequenceTsFileProcessor = createTsFileProcessor(true);
+ sequenceFileList.add(workSequenceTsFileProcessor.getTsFileResource());
+ }
+ tsFileProcessor = workSequenceTsFileProcessor;
+ } else {
+ if (workUnSequenceTsFileProcessor == null) {
+ // create a new TsfileProcessor
+ workUnSequenceTsFileProcessor = createTsFileProcessor(false);
+ unSequenceFileList.add(workUnSequenceTsFileProcessor.getTsFileResource());
+ }
+ tsFileProcessor = workUnSequenceTsFileProcessor;
+ }
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error(
+ "disk space is insufficient when creating TsFile processor, change system mode to read-only",
+ e);
+ IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
+ } catch (IOException e) {
+ logger.error("meet IOException when creating TsFileProcessor, change system mode to read-only",
+ e);
+ IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
+ }
+ return tsFileProcessor;
+ }
+
private TsFileProcessor createTsFileProcessor(boolean sequence)
throws IOException, DiskSpaceInsufficientException {
String baseDir;
@@ -359,15 +495,16 @@ public class StorageGroupProcessor {
if (sequence) {
return new TsFileProcessor(storageGroupName, new File(filePath),
- fileSchema, versionController, this::closeUnsealedTsFileProcessor,
+ schema, versionController, this::closeUnsealedTsFileProcessor,
this::updateLatestFlushTimeCallback, sequence);
} else {
return new TsFileProcessor(storageGroupName, new File(filePath),
- fileSchema, versionController, this::closeUnsealedTsFileProcessor,
+ schema, versionController, this::closeUnsealedTsFileProcessor,
() -> true, sequence);
}
}
+
/**
* only called by insert(), thread-safety should be ensured by caller
*/
@@ -456,8 +593,10 @@ public class StorageGroupProcessor {
}
// TODO need a read lock, please consider the concurrency with flush manager threads.
- public QueryDataSource query(String deviceId, String measurementId, QueryContext context) {
+ public QueryDataSource query(String deviceId, String measurementId, QueryContext context,
+ JobFileManager filePathsManager) {
insertLock.readLock().lock();
+ mergeLock.readLock().lock();
synchronized (lruForSensorUsedInQuery) {
if (lruForSensorUsedInQuery.size() >= MAX_CACHE_SENSORS) {
lruForSensorUsedInQuery.removeFirst();
@@ -469,9 +608,17 @@ public class StorageGroupProcessor {
deviceId, measurementId, context);
List unseqResources = getFileReSourceListForQuery(unSequenceFileList,
deviceId, measurementId, context);
- return new QueryDataSource(new Path(deviceId, measurementId), seqResources, unseqResources);
+ QueryDataSource dataSource = new QueryDataSource(new Path(deviceId, measurementId), seqResources, unseqResources);
+ // used files should be added before mergeLock is unlocked, or they may be deleted by
+ // running merge
+ // is null only in tests
+ if (filePathsManager != null) {
+ filePathsManager.addUsedFilesForGivenJob(context.getJobId(), dataSource);
+ }
+ return dataSource;
} finally {
insertLock.readLock().unlock();
+ mergeLock.readLock().unlock();
}
}
@@ -512,7 +659,7 @@ public class StorageGroupProcessor {
private List getFileReSourceListForQuery(List tsFileResources,
String deviceId, String measurementId, QueryContext context) {
- MeasurementSchema mSchema = fileSchema.getMeasurementSchema(measurementId);
+ MeasurementSchema mSchema = schema.getMeasurementSchema(measurementId);
TSDataType dataType = mSchema.getType();
List tsfileResourcesForQuery = new ArrayList<>();
@@ -557,6 +704,7 @@ public class StorageGroupProcessor {
public void delete(String deviceId, String measurementId, long timestamp) throws IOException {
// TODO: how to avoid partial deletion?
writeLock();
+ mergeLock.writeLock().lock();
// record files which are updated so that we can roll back them in case of exception
List updatedModFiles = new ArrayList<>();
@@ -584,7 +732,6 @@ public class StorageGroupProcessor {
Path fullPath = new Path(deviceId, measurementId);
Deletion deletion = new Deletion(fullPath, versionController.nextVersion(), timestamp);
if (mergingModification != null) {
- //TODO check me when implementing the merge process.
mergingModification.write(deletion);
updatedModFiles.add(mergingModification);
}
@@ -600,6 +747,7 @@ public class StorageGroupProcessor {
throw new IOException(e);
} finally {
writeUnlock();
+ mergeLock.writeLock().unlock();
}
}
@@ -674,6 +822,134 @@ public class StorageGroupProcessor {
}
}
+ public void merge(boolean fullMerge) {
+ writeLock();
+ try {
+ if (isMerging) {
+ if (logger.isInfoEnabled()) {
+ logger.info("{} Last merge is ongoing, currently consumed time: {}ms", storageGroupName,
+ (System.currentTimeMillis() - mergeStartTime));
+ }
+ return;
+ }
+ if (unSequenceFileList.isEmpty() || sequenceFileList.isEmpty()) {
+ logger.info("{} no files to be merged", storageGroupName);
+ return;
+ }
+
+ long budget = IoTDBDescriptor.getInstance().getConfig().getMergeMemoryBudget();
+ MergeResource mergeResource = new MergeResource(sequenceFileList, unSequenceFileList);
+ IMergeFileSelector fileSelector = getMergeFileSelector(budget, mergeResource);
+ try {
+ List[] mergeFiles = fileSelector.select();
+ if (mergeFiles.length == 0) {
+ logger.info("{} cannot select merge candidates under the budget {}", storageGroupName,
+ budget);
+ return;
+ }
+ // avoid pending tasks holds the metadata and streams
+ mergeResource.clear();
+ String taskName = storageGroupName + "-" + System.currentTimeMillis();
+ // do not cache metadata until true candidates are chosen, or too much metadata will be
+ // cached during selection
+ mergeResource.setCacheDeviceMeta(true);
+
+ MergeTask mergeTask = new MergeTask(mergeResource, storageGroupSysDir.getPath(),
+ this::mergeEndAction, taskName, fullMerge, fileSelector.getConcurrentMergeNum(), storageGroupName);
+ mergingModification = new ModificationFile(storageGroupSysDir + File.separator + MERGING_MODIFICAITON_FILE_NAME);
+ MergeManager.getINSTANCE().submitMainTask(mergeTask);
+ if (logger.isInfoEnabled()) {
+ logger.info("{} submits a merge task {}, merging {} seqFiles, {} unseqFiles",
+ storageGroupName, taskName, mergeFiles[0].size(), mergeFiles[1].size());
+ }
+ isMerging = true;
+ mergeStartTime = System.currentTimeMillis();
+
+ } catch (MergeException | IOException e) {
+ logger.error("{} cannot select file for merge", storageGroupName, e);
+ }
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ private IMergeFileSelector getMergeFileSelector(long budget, MergeResource resource) {
+ MergeFileStrategy strategy = IoTDBDescriptor.getInstance().getConfig().getMergeFileStrategy();
+ switch (strategy) {
+ case MAX_FILE_NUM:
+ return new MaxFileMergeFileSelector(resource, budget);
+ case MAX_SERIES_NUM:
+ return new MaxSeriesMergeFileSelector(resource, budget);
+ default:
+ throw new UnsupportedOperationException("Unknown MergeFileStrategy " + strategy);
+ }
+ }
+
+ protected void mergeEndAction(List seqFiles, List unseqFiles,
+ File mergeLog) {
+ logger.info("{} a merge task is ending...", storageGroupName);
+
+ if (unseqFiles.isEmpty()) {
+ // merge runtime exception arose, just end this merge
+ isMerging = false;
+ logger.info("{} a merge task abnormally ends", storageGroupName);
+ return;
+ }
+
+ mergeLock.writeLock().lock();
+ try {
+ unSequenceFileList.removeAll(unseqFiles);
+ } finally {
+ mergeLock.writeLock().unlock();
+ }
+
+ for (TsFileResource unseqFile : unseqFiles) {
+ unseqFile.getMergeQueryLock().writeLock().lock();
+ try {
+ unseqFile.remove();
+ } finally {
+ unseqFile.getMergeQueryLock().writeLock().unlock();
+ }
+ }
+
+ for (int i = 0; i < seqFiles.size(); i++) {
+ TsFileResource seqFile = seqFiles.get(i);
+ seqFile.getMergeQueryLock().writeLock().lock();
+ mergeLock.writeLock().lock();
+ try {
+ logger.debug("{} is updating the {} merged file's modification file", storageGroupName, i);
+ try {
+ // remove old modifications and write modifications generated during merge
+ seqFile.removeModFile();
+ if (mergingModification != null) {
+ for (Modification modification : mergingModification.getModifications()) {
+ seqFile.getModFile().write(modification);
+ }
+ }
+ } catch (IOException e) {
+ logger.error("{} cannot clean the ModificationFile of {} after merge", storageGroupName,
+ seqFile.getFile(), e);
+ }
+ if (i == seqFiles.size() - 1) {
+ try {
+ if (mergingModification != null) {
+ mergingModification.remove();
+ mergingModification = null;
+ }
+ } catch (IOException e) {
+ logger.error("{} cannot remove merging modification ", storageGroupName, e);
+ }
+ isMerging = false;
+ }
+ } finally {
+ mergeLog.delete();
+ seqFile.getMergeQueryLock().writeLock().unlock();
+ mergeLock.writeLock().unlock();
+ }
+ }
+ logger.info("{} a merge task ends", storageGroupName);
+ }
+
public TsFileProcessor getWorkSequenceTsFileProcessor() {
return workSequenceTsFileProcessor;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 34dcbac55609a8b260ef9dcf6207bd518ec7ee14..5d237d19f7c4817b26c10527681c25d47cab64b9 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -44,16 +44,18 @@ import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor.CloseTsFile
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.qp.constant.DatetimeUtils;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.rescon.MemTablePool;
+import org.apache.iotdb.rpc.TSStatusType;
import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +66,7 @@ public class TsFileProcessor {
private RestorableTsFileIOWriter writer;
- private FileSchema fileSchema;
+ private Schema schema;
private final String storageGroupName;
@@ -109,13 +111,13 @@ public class TsFileProcessor {
private long totalMemTableSize;
- TsFileProcessor(String storageGroupName, File tsfile, FileSchema fileSchema,
+ TsFileProcessor(String storageGroupName, File tsfile, Schema schema,
VersionController versionController,
CloseTsFileCallBack closeTsFileCallback,
Supplier updateLatestFlushTimeCallback, boolean sequence)
throws IOException {
this.storageGroupName = storageGroupName;
- this.fileSchema = fileSchema;
+ this.schema = schema;
this.tsFileResource = new TsFileResource(tsfile, this);
this.versionController = versionController;
this.writer = new RestorableTsFileIOWriter(tsfile);
@@ -134,13 +136,7 @@ public class TsFileProcessor {
public boolean insert(InsertPlan insertPlan) {
if (workMemTable == null) {
- // TODO change the impl of getAvailableMemTable to non-blocking
workMemTable = MemTablePool.getInstance().getAvailableMemTable(this);
-
- // no empty memtable, return failure
- if (workMemTable == null) {
- return false;
- }
}
if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
@@ -165,6 +161,37 @@ public class TsFileProcessor {
return true;
}
+ public boolean insertBatch(BatchInsertPlan batchInsertPlan, List indexes,
+ Integer[] results) {
+ if (workMemTable == null) {
+ workMemTable = MemTablePool.getInstance().getAvailableMemTable(this);
+ }
+
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
+ try {
+ getLogNode().write(batchInsertPlan);
+ } catch (IOException e) {
+ logger.error("write WAL failed", e);
+ for (int index: indexes) {
+ results[index] = TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode();
+ }
+ return false;
+ }
+ }
+
+ tsFileResource.updateStartTime(batchInsertPlan.getDeviceId(), batchInsertPlan.getMinTime());
+
+ //for sequence tsfile, we update the endTime only when the file is prepared to be closed.
+ //for unsequence tsfile, we have to update the endTime for each insertion.
+ if (!sequence) {
+ tsFileResource.updateEndTime(batchInsertPlan.getDeviceId(), batchInsertPlan.getMaxTime());
+ }
+
+ // insert insertPlan to the work memtable
+ workMemTable.insertBatch(batchInsertPlan, indexes);
+ return true;
+ }
+
/**
* Delete data which belongs to the timeseries `deviceId.measurementId` and the timestamp of which
* <= 'timestamp' in the deletion.
@@ -372,7 +399,7 @@ public class TsFileProcessor {
// signal memtable only may appear when calling asyncClose()
if (!memTableToFlush.isSignalMemTable()) {
- MemTableFlushTask flushTask = new MemTableFlushTask(memTableToFlush, fileSchema, writer,
+ MemTableFlushTask flushTask = new MemTableFlushTask(memTableToFlush, schema, writer,
storageGroupName);
try {
writer.mark();
@@ -439,7 +466,7 @@ public class TsFileProcessor {
long closeStartTime = System.currentTimeMillis();
tsFileResource.serialize();
- writer.endFile(fileSchema);
+ writer.endFile(schema);
// remove this processor from Closing list in StorageGroupProcessor,
// mark the TsFileResource closed, no need writer anymore
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index fae36f2e81522e6ff2757f0691a627c2dbe2847f..3a7af7ec5d2b7788535093c9d93ac4c976eec541 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -30,6 +30,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
@@ -40,6 +44,7 @@ public class TsFileResource {
private File file;
public static final String RESOURCE_SUFFIX = ".resource";
+ public static final String TEMP_SUFFIX = ".temp";
/**
* device -> start time
@@ -63,23 +68,24 @@ public class TsFileResource {
*/
private List chunkMetaDatas;
-
/**
* Mem chunk data. Only be set in a temporal TsFileResource in a query process.
*/
private ReadOnlyMemChunk readOnlyMemChunk;
+ private ReentrantReadWriteLock mergeQueryLock = new ReentrantReadWriteLock();
+
public TsFileResource(File file) {
this.file = file;
- this.startTimeMap = new HashMap<>();
+ this.startTimeMap = new ConcurrentHashMap<>();
this.endTimeMap = new HashMap<>();
this.closed = true;
}
public TsFileResource(File file, TsFileProcessor processor) {
this.file = file;
- this.startTimeMap = new HashMap<>();
- this.endTimeMap = new HashMap<>();
+ this.startTimeMap = new ConcurrentHashMap<>();
+ this.endTimeMap = new ConcurrentHashMap<>();
this.processor = processor;
}
@@ -106,7 +112,7 @@ public class TsFileResource {
public void serialize() throws IOException {
try (OutputStream outputStream = new BufferedOutputStream(
- new FileOutputStream(file + RESOURCE_SUFFIX))) {
+ new FileOutputStream(file + RESOURCE_SUFFIX + TEMP_SUFFIX))) {
ReadWriteIOUtils.write(this.startTimeMap.size(), outputStream);
for (Entry entry : this.startTimeMap.entrySet()) {
ReadWriteIOUtils.write(entry.getKey(), outputStream);
@@ -118,6 +124,10 @@ public class TsFileResource {
ReadWriteIOUtils.write(entry.getValue(), outputStream);
}
}
+ File src = new File(file + RESOURCE_SUFFIX + TEMP_SUFFIX);
+ File dest = new File(file + RESOURCE_SUFFIX);
+ dest.delete();
+ FileUtils.moveFile(src, dest);
}
public void deSerialize() throws IOException {
@@ -221,11 +231,44 @@ public class TsFileResource {
return processor;
}
- public void updateTime(String deviceId, long time) {
- startTimeMap.putIfAbsent(deviceId, time);
- Long endTime = endTimeMap.get(deviceId);
- if (endTime == null || endTime < time) {
- endTimeMap.put(deviceId, time);
+ public ReentrantReadWriteLock getMergeQueryLock() {
+ return mergeQueryLock;
+ }
+
+ public void removeModFile() throws IOException {
+ getModFile().remove();
+ modFile = null;
+ }
+
+ public void remove() {
+ file.delete();
+ new File(file.getPath() + RESOURCE_SUFFIX).delete();
+ new File(file.getPath() + ModificationFile.FILE_SUFFIX).delete();
+ }
+
+ @Override
+ public String toString() {
+ return file.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
}
+ TsFileResource that = (TsFileResource) o;
+ return Objects.equals(file, that.file);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(file);
+ }
+
+ public void setClosed(boolean closed) {
+ this.closed = closed;
}
}
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/MetadataDemo.java b/server/src/main/java/org/apache/iotdb/db/exception/MergeException.java
similarity index 61%
rename from jdbc/src/test/java/org/apache/iotdb/jdbc/demo/MetadataDemo.java
rename to server/src/main/java/org/apache/iotdb/db/exception/MergeException.java
index 41c22d3805ce8295466a9e14d1346b43dd4f1854..2980cfb9183822ada9a462582ad85bef7eaa44cd 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/MetadataDemo.java
+++ b/server/src/main/java/org/apache/iotdb/db/exception/MergeException.java
@@ -16,21 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.jdbc.demo;
-import java.sql.SQLException;
+package org.apache.iotdb.db.exception;
-public class MetadataDemo {
+public class MergeException extends Exception{
- public static void main(String[] args) throws ClassNotFoundException, SQLException {
- // Class.forName(TsfileJDBCConfig.JDBC_DRIVER_NAME);
- // Connection connection = null;
- // try {
- // connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
- // DatabaseMetaData databaseMetaData = connection.getMetaData();
- // } finally {
- // connection.close();
- // }
+ public MergeException() {
}
-}
\ No newline at end of file
+ public MergeException(String message) {
+ super(message);
+ }
+
+ public MergeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public MergeException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java b/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
index ac71b97dba32fd62d512023f72bfd009a609a663..64d10d517c13cb4c14a20d6b418ec88743290f67 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
@@ -376,14 +376,12 @@ public class MGraph implements Serializable {
}
/**
- *
* @return storage group name -> the series number
*/
Map countSeriesNumberInEachStorageGroup() throws PathErrorException {
Map res = new HashMap<>();
Set storageGroups = this.getAllStorageGroup();
for (String sg : storageGroups) {
- this.getNumSchemaMapForOneFileNode(sg);
MNode node = mtree.getNodeByPath(sg);
res.put(sg, node.getLeafCount());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java
index a6aefd6c62cfa9f51ee6b0e0657e6e50f5e74344..d9654bc07185744965b550162799de7d23d5fde0 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -114,6 +115,12 @@ public interface IQueryProcessExecutor {
*/
boolean insert(InsertPlan insertPlan) throws ProcessorException;
+ /**
+ * execute batch insert plan
+ * @return result of each row
+ */
+ Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws ProcessorException;
+
boolean judgePathExists(Path fullPath);
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
index 6d5ee597a7afd80d015e8e039dbaafe5d3541975..a73691575d21969904918e261e02859c40254575 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
@@ -46,6 +46,7 @@ import org.apache.iotdb.db.qp.logical.sys.AuthorOperator.AuthorType;
import org.apache.iotdb.db.qp.logical.sys.MetadataOperator;
import org.apache.iotdb.db.qp.logical.sys.PropertyOperator;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
@@ -200,20 +201,18 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor {
@Override
- public boolean insert(InsertPlan insertPlan)
- throws ProcessorException {
-
+ public boolean insert(InsertPlan insertPlan) throws ProcessorException {
try {
String[] measurementList = insertPlan.getMeasurements();
String deviceId = insertPlan.getDeviceId();
- MNode node = mManager.getNodeByDeviceIdFromCache(insertPlan.getDeviceId());
+ MNode node = mManager.getNodeByDeviceIdFromCache(deviceId);
String[] values = insertPlan.getValues();
TSDataType[] dataTypes = new TSDataType[measurementList.length];
for (int i = 0; i < measurementList.length; i++) {
if (!node.hasChild(measurementList[i])) {
throw new ProcessorException(
- String.format("Current deviceId[%s] does not contains measurement:%s",
+ String.format("Current deviceId[%s] does not contain measurement:%s",
deviceId, measurementList[i]));
}
MNode measurementNode = node.getChild(measurementList[i]);
@@ -234,6 +233,32 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor {
}
}
+ @Override
+ public Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws ProcessorException {
+ try {
+ String[] measurementList = batchInsertPlan.getMeasurements();
+ String deviceId = batchInsertPlan.getDeviceId();
+ MNode node = mManager.getNodeByDeviceIdFromCache(deviceId);
+
+ for (String s : measurementList) {
+ if (!node.hasChild(s)) {
+ throw new ProcessorException(
+ String.format("Current deviceId[%s] does not contain measurement:%s",
+ deviceId, s));
+ }
+ MNode measurementNode = node.getChild(s);
+ if (!measurementNode.isLeaf()) {
+ throw new ProcessorException(
+ String.format("Current Path is not leaf node. %s.%s", deviceId, s));
+ }
+ }
+ return storageEngine.insertBatch(batchInsertPlan);
+
+ } catch (PathErrorException | StorageEngineException e) {
+ throw new ProcessorException(e);
+ }
+ }
+
@Override
public List getAllPaths(String originPath) throws MetadataErrorException {
return MManager.getInstance().getPaths(originPath);
@@ -352,7 +377,7 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor {
storageEngine.deleteAllDataFilesInOneStorageGroup(deleteStorageGroup);
}
break;
- case SET_FILE_LEVEL:
+ case SET_STORAGE_GROUP:
mManager.setStorageLevelToMTree(path.getFullPath());
break;
default:
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
index 498b429ef50434834efb2a6aeca585da685d0776..f18df8782f539312c64c25a3b4d38a85a27900a5 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java
@@ -67,7 +67,7 @@ public abstract class Operator {
*/
public enum OperatorType {
SFW, JOIN, UNION, FILTER, GROUPBY, ORDERBY, LIMIT, SELECT, SEQTABLESCAN, HASHTABLESCAN,
- MERGEJOIN, FILEREAD, NULL, TABLESCAN, UPDATE, INSERT, DELETE, BASIC_FUNC, QUERY, MERGEQUERY,
+ MERGEJOIN, FILEREAD, NULL, TABLESCAN, UPDATE, INSERT, BATCHINSERT, DELETE, BASIC_FUNC, QUERY, MERGEQUERY,
AGGREGATION, AUTHOR, FROM, FUNC, LOADDATA, METADATA, PROPERTY, INDEX, INDEXQUERY, FILL,
SET_STORAGE_GROUP, CREATE_TIMESERIES, DELETE_TIMESERIES, CREATE_USER, DELETE_USER, MODIFY_PASSWORD,
GRANT_USER_PRIVILEGE, REVOKE_USER_PRIVILEGE, GRANT_USER_ROLE, REVOKE_USER_ROLE, CREATE_ROLE,
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/MetadataOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/MetadataOperator.java
index 4f2c6bd21a6f5bf762abaa637779f42944370d3a..2a80f6b387f99483911a11672422c9922c82e057 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/MetadataOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/MetadataOperator.java
@@ -46,7 +46,7 @@ public class MetadataOperator extends RootOperator {
super(tokenIntType);
namespaceType = type;
switch (type) {
- case SET_FILE_LEVEL:
+ case SET_STORAGE_GROUP:
operatorType = OperatorType.SET_STORAGE_GROUP;
break;
case ADD_PATH:
@@ -113,7 +113,7 @@ public class MetadataOperator extends RootOperator {
}
public enum NamespaceType {
- ADD_PATH, DELETE_PATH, SET_FILE_LEVEL;
+ ADD_PATH, DELETE_PATH, SET_STORAGE_GROUP;
/**
* deserialize short number.
@@ -128,7 +128,7 @@ public class MetadataOperator extends RootOperator {
case 1:
return DELETE_PATH;
case 2:
- return SET_FILE_LEVEL;
+ return SET_STORAGE_GROUP;
default:
return null;
}
@@ -145,7 +145,7 @@ public class MetadataOperator extends RootOperator {
return 0;
case DELETE_PATH:
return 1;
- case SET_FILE_LEVEL:
+ case SET_STORAGE_GROUP:
return 2;
default:
return -1;
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index b60599f87007c5e5cf0e3029e5a102fa6e269b27..4288539484b2f8375bed787bc65de31bc796ff98 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
import org.apache.iotdb.db.qp.logical.Operator;
+import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -118,6 +119,10 @@ public abstract class PhysicalPlan {
plan = new DeletePlan();
plan.deserializeFrom(buffer);
break;
+ case BATCHINSERT:
+ plan = new BatchInsertPlan();
+ plan.deserializeFrom(buffer);
+ break;
default:
throw new IOException("unrecognized log type " + type);
}
@@ -126,7 +131,7 @@ public abstract class PhysicalPlan {
}
public enum PhysicalPlanType {
- INSERT, DELETE
+ INSERT, DELETE, BATCHINSERT
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java
new file mode 100644
index 0000000000000000000000000000000000000000..ce907ed941073ad2fd685baeb631519a8fed4ff7
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java
@@ -0,0 +1,276 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.qp.physical.crud;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
+import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+import org.apache.iotdb.db.utils.QueryDataSetUtils;
+import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+
+public class BatchInsertPlan extends PhysicalPlan {
+
+ private String deviceId;
+ private String[] measurements;
+ private TSDataType[] dataTypes;
+
+ private long[] times;
+ private ByteBuffer timeBuffer;
+
+ private Object[] columns;
+ private ByteBuffer valueBuffer;
+
+ private int rowCount = 0;
+
+ // cached values
+ private Long maxTime = null;
+ private Long minTime = null;
+ private List paths;
+
+ public BatchInsertPlan() {
+ super(false, OperatorType.BATCHINSERT);
+ }
+
+ public BatchInsertPlan(String deviceId, List measurements) {
+ super(false, OperatorType.BATCHINSERT);
+ this.deviceId = deviceId;
+ setMeasurements(measurements);
+ }
+
+ public BatchInsertPlan(String deviceId, String[] measurements, List dataTypes) {
+ super(false, OperatorType.BATCHINSERT);
+ this.deviceId = deviceId;
+ this.measurements = measurements;
+ setDataTypes(dataTypes);
+ }
+
+
+ @Override
+ public List getPaths() {
+ if (paths != null) {
+ return paths;
+ }
+ List ret = new ArrayList<>();
+ for (String m : measurements) {
+ ret.add(new Path(deviceId, m));
+ }
+ paths = ret;
+ return ret;
+ }
+
+ @Override
+ public void serializeTo(ByteBuffer buffer) {
+ int type = PhysicalPlanType.BATCHINSERT.ordinal();
+ buffer.put((byte) type);
+
+ putString(buffer, deviceId);
+
+ buffer.putInt(measurements.length);
+ for (String m : measurements) {
+ putString(buffer, m);
+ }
+
+ for (TSDataType dataType: dataTypes) {
+ buffer.putShort(dataType.serialize());
+ }
+
+ buffer.putInt(times.length);
+
+ if (timeBuffer == null) {
+ for (long time: times) {
+ buffer.putLong(time);
+ }
+ } else {
+ buffer.put(timeBuffer.array());
+ timeBuffer = null;
+ }
+
+ if (valueBuffer == null) {
+ for (int i = 0; i < measurements.length; i++) {
+ TSDataType dataType = dataTypes[i];
+ switch (dataType) {
+ case INT32:
+ int[] intValues = (int[]) columns[i];
+ for (int index = 0; index < rowCount; index++) {
+ buffer.putInt(intValues[index]);
+ }
+ break;
+ case INT64:
+ long[] longValues = (long[]) columns[i];
+ for (int index = 0; index < rowCount; index++) {
+ buffer.putLong(longValues[index]);
+ }
+ break;
+ case FLOAT:
+ float[] floatValues = (float[]) columns[i];
+ for (int index = 0; index < rowCount; index++) {
+ buffer.putFloat(floatValues[index]);
+ }
+ break;
+ case DOUBLE:
+ double[] doubleValues = (double[]) columns[i];
+ for (int index = 0; index < rowCount; index++) {
+ buffer.putDouble(doubleValues[index]);
+ }
+ break;
+ case BOOLEAN:
+ boolean[] boolValues = (boolean[]) columns[i];
+ for (int index = 0; index < rowCount; index++) {
+ buffer.put(BytesUtils.boolToByte(boolValues[index]));
+ }
+ break;
+ case TEXT:
+ Binary[] binaryValues = (Binary[]) columns[i];
+ for (int index = 0; index < rowCount; index++) {
+ buffer.putInt(binaryValues[index].getLength());
+ buffer.put(binaryValues[index].getValues());
+ }
+ break;
+ default:
+ throw new UnSupportedDataTypeException(
+ String.format("Data type %s is not supported.", dataType));
+ }
+ }
+ } else {
+ buffer.put(valueBuffer.array());
+ valueBuffer = null;
+ }
+ }
+
+ public void setTimeBuffer(ByteBuffer timeBuffer) {
+ this.timeBuffer = timeBuffer;
+ this.timeBuffer.position(0);
+ }
+
+ public void setValueBuffer(ByteBuffer valueBuffer) {
+ this.valueBuffer = valueBuffer;
+ this.timeBuffer.position(0);
+ }
+
+ @Override
+ public void deserializeFrom(ByteBuffer buffer) {
+ this.deviceId = readString(buffer);
+
+ int measurementSize = buffer.getInt();
+ this.measurements = new String[measurementSize];
+ for (int i = 0; i < measurementSize; i++) {
+ measurements[i] = readString(buffer);
+ }
+
+ this.dataTypes = new TSDataType[measurementSize];
+ for (int i = 0; i < measurementSize; i++) {
+ dataTypes[i] = TSDataType.deserialize(buffer.getShort());
+ }
+
+ int rows = buffer.getInt();
+ this.times = new long[rows];
+ QueryDataSetUtils.readTimesFromBuffer(buffer, rows);
+
+ QueryDataSetUtils.readValuesFromBuffer(buffer, dataTypes, measurementSize, rows);
+ }
+
+
+ public String getDeviceId() {
+ return deviceId;
+ }
+
+ public void setDeviceId(String deviceId) {
+ this.deviceId = deviceId;
+ }
+
+ public String[] getMeasurements() {
+ return measurements;
+ }
+
+ public void setMeasurements(List measurements) {
+ this.measurements = new String[measurements.size()];
+ measurements.toArray(this.measurements);
+ }
+
+ public void setMeasurements(String[] measurements) {
+ this.measurements = measurements;
+ }
+
+ public TSDataType[] getDataTypes() {
+ return dataTypes;
+ }
+
+ public void setDataTypes(List dataTypes) {
+ this.dataTypes = new TSDataType[dataTypes.size()];
+ for (int i = 0; i < dataTypes.size(); i++) {
+ this.dataTypes[i] = TSDataType.values()[dataTypes.get(i)];
+ }
+ }
+
+ public Object[] getColumns() {
+ return columns;
+ }
+
+ public long getMinTime() {
+ if (minTime != null) {
+ return minTime;
+ }
+ minTime = Long.MAX_VALUE;
+ for (Long time: times) {
+ if (time < minTime) {
+ minTime = time;
+ }
+ }
+ return minTime;
+ }
+
+ public long getMaxTime() {
+ if (maxTime != null) {
+ return maxTime;
+ }
+ long maxTime = Long.MIN_VALUE;
+ for (Long time: times) {
+ if (time > maxTime) {
+ maxTime = time;
+ }
+ }
+ return maxTime;
+ }
+
+ public long[] getTimes() {
+ return times;
+ }
+
+ public void setTimes(long[] times) {
+ this.times = times;
+ }
+
+ public int getRowCount() {
+ return rowCount;
+ }
+
+ public void setRowCount(int size) {
+ this.rowCount = size;
+ }
+
+ public void setColumns(Object[] columns) {
+ this.columns = columns;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/MetadataPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/MetadataPlan.java
index 263ea533c87331f2e1a19e6be4020276533e7fce..cae6f63c5ae9653ca0fbd962d17aa1cbf0f67ca4 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/MetadataPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/MetadataPlan.java
@@ -55,19 +55,24 @@ public class MetadataPlan extends PhysicalPlan {
this.encoding = encoding;
this.props = props;
this.deletePathList = deletePathList;
- switch (namespaceType) {
- case SET_FILE_LEVEL:
- setOperatorType(Operator.OperatorType.SET_STORAGE_GROUP);
- break;
- case ADD_PATH:
- setOperatorType(Operator.OperatorType.CREATE_TIMESERIES);
- break;
- case DELETE_PATH:
- setOperatorType(Operator.OperatorType.DELETE_TIMESERIES);
- break;
- default:
- break;
- }
+ setOperatorType(namespaceType);
+ }
+
+ public MetadataPlan(MetadataOperator.NamespaceType namespaceType, Path path, TSDataType dataType,
+ TSEncoding encoding) {
+ super(false, Operator.OperatorType.METADATA);
+ this.namespaceType = namespaceType;
+ this.path = path;
+ this.dataType = dataType;
+ this.encoding = encoding;
+ setOperatorType(namespaceType);
+ }
+
+ public MetadataPlan(MetadataOperator.NamespaceType namespaceType, Path path) {
+ super(false, Operator.OperatorType.METADATA);
+ this.namespaceType = namespaceType;
+ this.path = path;
+ setOperatorType(namespaceType);
}
public Path getPath() {
@@ -171,4 +176,20 @@ public class MetadataPlan extends PhysicalPlan {
.hash(getNamespaceType(), getPath(), getDataType(), getCompressor(), getEncoding(),
getProps(), getDeletePathList());
}
+
+ private void setOperatorType(MetadataOperator.NamespaceType namespaceType) {
+ switch (namespaceType) {
+ case SET_STORAGE_GROUP:
+ setOperatorType(Operator.OperatorType.SET_STORAGE_GROUP);
+ break;
+ case ADD_PATH:
+ setOperatorType(Operator.OperatorType.CREATE_TIMESERIES);
+ break;
+ case DELETE_PATH:
+ setOperatorType(Operator.OperatorType.DELETE_TIMESERIES);
+ break;
+ default:
+ break;
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
index 13be7d33794cc1384efffc83872364dda1eaef50..fb288f750364af778025363ae678077c85c1ae02 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
@@ -425,7 +425,7 @@ public class LogicalGenerator {
private void analyzeMetadataSetFileLevel(AstNode astNode) {
MetadataOperator metadataOperator = new MetadataOperator(
SQLConstant.TOK_METADATA_SET_FILE_LEVEL,
- MetadataOperator.NamespaceType.SET_FILE_LEVEL);
+ MetadataOperator.NamespaceType.SET_STORAGE_GROUP);
Path path = parsePath(astNode.getChild(0).getChild(0));
metadataOperator.setPath(path);
initializedOperator = metadataOperator;
@@ -533,10 +533,6 @@ public class LogicalGenerator {
if (filterOperator.getTokenIntType() == LESSTHAN) {
time = time - 1;
}
- // time must greater than 0 now
- if (time <= 0) {
- throw new LogicalOperatorException("delete Time:" + time + ", time must > 0");
- }
return time;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index 29d1bc9d8dcd69c64dfc57b47c8018b27fdaed52..b41a1efcb162383bc18420fc5d02d5b8fbfde905 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -90,9 +90,6 @@ public class PhysicalGenerator {
case DELETE:
DeleteOperator delete = (DeleteOperator) operator;
paths = delete.getSelectedPaths();
- if (delete.getTime() <= 0) {
- throw new LogicalOperatorException("For Delete command, time must greater than 0.");
- }
return new DeletePlan(delete.getTime(), paths);
case INSERT:
InsertOperator insert = (InsertOperator) operator;
@@ -101,9 +98,6 @@ public class PhysicalGenerator {
throw new LogicalOperatorException(
"For Insert command, cannot specified more than one seriesPath:" + paths);
}
- if (insert.getTime() <= 0) {
- throw new LogicalOperatorException("For Insert command, time must greater than 0.");
- }
return new InsertPlan(paths.get(0).getFullPath(), insert.getTime(),
insert.getMeasurementList(),
insert.getValueList());
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
index 8d33137e6a928b2cf75f2d28a3a5913e1eaa6d2a..76dad2f1757d9725355c5d45296509134c1dd1e3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -50,23 +51,23 @@ public class FileReaderManager implements IService {
* the key of closedFileReaderMap is the file path and the value of closedFileReaderMap
* is the corresponding reader.
*/
- private ConcurrentHashMap closedFileReaderMap;
+ private ConcurrentHashMap closedFileReaderMap;
/**
* the key of unclosedFileReaderMap is the file path and the value of unclosedFileReaderMap
* is the corresponding reader.
*/
- private ConcurrentHashMap unclosedFileReaderMap;
+ private ConcurrentHashMap unclosedFileReaderMap;
/**
* the key of closedFileReaderMap is the file path and the value of closedFileReaderMap
* is the file's reference count.
*/
- private ConcurrentHashMap closedReferenceMap;
+ private ConcurrentHashMap closedReferenceMap;
/**
* the key of unclosedFileReaderMap is the file path and the value of unclosedFileReaderMap
* is the file's reference count.
*/
- private ConcurrentHashMap unclosedReferenceMap;
+ private ConcurrentHashMap unclosedReferenceMap;
private ScheduledExecutorService executorService;
@@ -97,9 +98,9 @@ public class FileReaderManager implements IService {
}, 0, examinePeriod, TimeUnit.MILLISECONDS);
}
- private void clearMap(Map readerMap,
- Map refMap) {
- for (Map.Entry entry : readerMap.entrySet()) {
+ private void clearMap(Map readerMap,
+ Map refMap) {
+ for (Map.Entry entry : readerMap.entrySet()) {
TsFileSequenceReader reader = entry.getValue();
AtomicInteger refAtom = refMap.get(entry.getKey());
@@ -120,71 +121,73 @@ public class FileReaderManager implements IService {
* exists, just get it from closedFileReaderMap or unclosedFileReaderMap depending on isClosing .
* Otherwise a new reader will be created and cached.
*
- * @param filePath the path of the file, of which the reader is desired.
+ * @param tsFile the path of the file, of which the reader is desired.
* @param isClosed whether the corresponding file still receives insertions or not.
* @return the reader of the file specified by filePath.
* @throws IOException when reader cannot be created.
*/
- public synchronized TsFileSequenceReader get(String filePath, boolean isClosed)
+ public synchronized TsFileSequenceReader get(TsFileResource tsFile, boolean isClosed)
throws IOException {
- Map readerMap = !isClosed ? unclosedFileReaderMap
+ Map readerMap = !isClosed ? unclosedFileReaderMap
: closedFileReaderMap;
- if (!readerMap.containsKey(filePath)) {
+ if (!readerMap.containsKey(tsFile)) {
if (readerMap.size() >= MAX_CACHED_FILE_SIZE) {
logger.warn("Query has opened {} files !", readerMap.size());
}
- TsFileSequenceReader tsFileReader = !isClosed ? new UnClosedTsFileReader(filePath)
- : new TsFileSequenceReader(filePath);
+ TsFileSequenceReader tsFileReader = !isClosed ? new UnClosedTsFileReader(tsFile.getFile().getPath())
+ : new TsFileSequenceReader(tsFile.getFile().getPath());
- readerMap.put(filePath, tsFileReader);
+ readerMap.put(tsFile, tsFileReader);
return tsFileReader;
}
- return readerMap.get(filePath);
+ return readerMap.get(tsFile);
}
/**
* Increase the reference count of the reader specified by filePath. Only when the reference count
* of a reader equals zero, the reader can be closed and removed.
*/
- public synchronized void increaseFileReaderReference(String filePath, boolean isClosed) {
+ public synchronized void increaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
// TODO : this should be called in get()
if (!isClosed) {
- unclosedReferenceMap.computeIfAbsent(filePath, k -> new AtomicInteger()).getAndIncrement();
+ unclosedReferenceMap.computeIfAbsent(tsFile, k -> new AtomicInteger()).getAndIncrement();
} else {
- closedReferenceMap.computeIfAbsent(filePath, k -> new AtomicInteger()).getAndIncrement();
+ closedReferenceMap.computeIfAbsent(tsFile, k -> new AtomicInteger()).getAndIncrement();
}
+ tsFile.getMergeQueryLock().readLock().lock();
}
/**
* Decrease the reference count of the reader specified by filePath. This method is latch-free.
* Only when the reference count of a reader equals zero, the reader can be closed and removed.
*/
- public synchronized void decreaseFileReaderReference(String filePath, boolean isClosed) {
- if (!isClosed && unclosedReferenceMap.containsKey(filePath)) {
- unclosedReferenceMap.get(filePath).getAndDecrement();
- } else if (closedReferenceMap.containsKey(filePath)){
- closedReferenceMap.get(filePath).getAndDecrement();
+ public synchronized void decreaseFileReaderReference(TsFileResource tsFile, boolean isClosed) {
+ if (!isClosed && unclosedReferenceMap.containsKey(tsFile)) {
+ unclosedReferenceMap.get(tsFile).getAndDecrement();
+ } else if (closedReferenceMap.containsKey(tsFile)){
+ closedReferenceMap.get(tsFile).getAndDecrement();
}
+ tsFile.getMergeQueryLock().readLock().unlock();
}
/**
* This method is used when the given file path is deleted.
*/
- public synchronized void closeFileAndRemoveReader(String filePath)
+ public synchronized void closeFileAndRemoveReader(TsFileResource tsFile)
throws IOException {
- if (unclosedFileReaderMap.containsKey(filePath)) {
- unclosedReferenceMap.remove(filePath);
- unclosedFileReaderMap.get(filePath).close();
- unclosedFileReaderMap.remove(filePath);
+ if (unclosedFileReaderMap.containsKey(tsFile)) {
+ unclosedReferenceMap.remove(tsFile);
+ unclosedFileReaderMap.get(tsFile).close();
+ unclosedFileReaderMap.remove(tsFile);
}
- if (closedFileReaderMap.containsKey(filePath)) {
- closedReferenceMap.remove(filePath);
- closedFileReaderMap.get(filePath).close();
- closedFileReaderMap.remove(filePath);
+ if (closedFileReaderMap.containsKey(tsFile)) {
+ closedReferenceMap.remove(tsFile);
+ closedFileReaderMap.get(tsFile).close();
+ closedFileReaderMap.remove(tsFile);
}
}
@@ -193,12 +196,12 @@ public class FileReaderManager implements IService {
* integration tests will not conflict with each other.
*/
public synchronized void closeAndRemoveAllOpenedReaders() throws IOException {
- for (Map.Entry entry : closedFileReaderMap.entrySet()) {
+ for (Map.Entry entry : closedFileReaderMap.entrySet()) {
entry.getValue().close();
closedReferenceMap.remove(entry.getKey());
closedFileReaderMap.remove(entry.getKey());
}
- for (Map.Entry entry : unclosedFileReaderMap.entrySet()) {
+ for (Map.Entry entry : unclosedFileReaderMap.entrySet()) {
entry.getValue().close();
unclosedReferenceMap.remove(entry.getKey());
unclosedFileReaderMap.remove(entry.getKey());
@@ -208,9 +211,9 @@ public class FileReaderManager implements IService {
/**
* This method is only for unit tests.
*/
- public synchronized boolean contains(String filePath, boolean isClosed) {
- return (isClosed && closedFileReaderMap.containsKey(filePath))
- || (!isClosed && unclosedFileReaderMap.containsKey(filePath));
+ public synchronized boolean contains(TsFileResource tsFile, boolean isClosed) {
+ return (isClosed && closedFileReaderMap.containsKey(tsFile))
+ || (!isClosed && unclosedFileReaderMap.containsKey(tsFile));
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
index f102e4448df26bc22c02ac45f301c6d54ddd278b..d8901f790fe436323afd4521108302a379b4e07b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.query.control;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -34,8 +35,8 @@ public class JobFileManager {
/**
* Map>
*/
- private ConcurrentHashMap> sealedFilePathsMap;
- private ConcurrentHashMap> unsealedFilePathsMap;
+ private ConcurrentHashMap> sealedFilePathsMap;
+ private ConcurrentHashMap> unsealedFilePathsMap;
public JobFileManager() {
sealedFilePathsMap = new ConcurrentHashMap<>();
@@ -59,14 +60,12 @@ public class JobFileManager {
//sequence data
for(TsFileResource tsFileResource : dataSource.getSeqResources()){
- String path = tsFileResource.getFile().getPath();
- addFilePathToMap(jobId, path, tsFileResource.isClosed());
+ addFilePathToMap(jobId, tsFileResource, tsFileResource.isClosed());
}
//unsequence data
for(TsFileResource tsFileResource : dataSource.getUnseqResources()){
- String path = tsFileResource.getFile().getPath();
- addFilePathToMap(jobId, path, tsFileResource.isClosed());
+ addFilePathToMap(jobId, tsFileResource, tsFileResource.isClosed());
}
}
@@ -75,14 +74,20 @@ public class JobFileManager {
* this jdbc request must be cleared and thus the usage reference must be decreased.
*/
void removeUsedFilesForGivenJob(long jobId) {
- for (String filePath : sealedFilePathsMap.get(jobId)) {
- FileReaderManager.getInstance().decreaseFileReaderReference(filePath, true);
+ Set tsFiles = sealedFilePathsMap.get(jobId);
+ if (tsFiles != null) {
+ for (TsFileResource tsFile : sealedFilePathsMap.get(jobId)) {
+ FileReaderManager.getInstance().decreaseFileReaderReference(tsFile, true);
}
sealedFilePathsMap.remove(jobId);
- for (String filePath : unsealedFilePathsMap.get(jobId)) {
- FileReaderManager.getInstance().decreaseFileReaderReference(filePath, false);
+ }
+ tsFiles = unsealedFilePathsMap.get(jobId);
+ if (tsFiles != null) {
+ for (TsFileResource tsFile : unsealedFilePathsMap.get(jobId)) {
+ FileReaderManager.getInstance().decreaseFileReaderReference(tsFile, false);
}
unsealedFilePathsMap.remove(jobId);
+ }
}
/**
@@ -91,13 +96,13 @@ public class JobFileManager {
* so sealedFilePathsMap.get(jobId)
or unsealedFilePathsMap.get(jobId)
* must not return null.
*/
- void addFilePathToMap(long jobId, String filePath, boolean isSealed) {
- ConcurrentHashMap> pathMap = !isSealed ? unsealedFilePathsMap :
+ void addFilePathToMap(long jobId, TsFileResource tsFile, boolean isClosed) {
+ ConcurrentHashMap> pathMap = !isClosed ? unsealedFilePathsMap :
sealedFilePathsMap;
//TODO this is not an atomic operation, is there concurrent problem?
- if (!pathMap.get(jobId).contains(filePath)) {
- pathMap.get(jobId).add(filePath);
- FileReaderManager.getInstance().increaseFileReaderReference(filePath, isSealed);
+ if (!pathMap.get(jobId).contains(tsFile)) {
+ pathMap.get(jobId).add(tsFile);
+ FileReaderManager.getInstance().increaseFileReaderReference(tsFile, isClosed);
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index be81daf8a63dd045236b9677326f433cd72c43b4..e05943e9bf6aae62f2119635710e9fa9df25deae 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -18,12 +18,7 @@
*/
package org.apache.iotdb.db.query.control;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
@@ -40,56 +35,15 @@ import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
* QueryResourceManager manages resource (file streams) used by each query job, and assign Ids to the jobs.
* During the life cycle of a query, the following methods must be called in strict order:
* 1. assignJobId - get an Id for the new job.
- * 2. beginQueryOfGivenQueryPaths - remind StorageEngine that some files are being used
- * 3. (if using filter)beginQueryOfGivenExpression
- * - remind StorageEngine that some files are being used
- * 4. getQueryDataSource - open files for the job or reuse existing readers.
- * 5. endQueryForGivenJob - putBack the resource used by this job.
+ * 2. getQueryDataSource - open files for the job or reuse existing readers.
+ * 3. endQueryForGivenJob - release the resource used by this job.
*
*/
public class QueryResourceManager {
- /**
- * Map<jobId, Map<deviceId, List<token>>>.
- *
- *
- * Key of queryTokensMap is job id, value of queryTokensMap is a deviceId-tokenList map, key of
- * the deviceId-tokenList map is device id, value of deviceId-tokenList map is a list of tokens.
- *
- *
- *
- * For example, during a query process Q1, given a query sql select device_1.sensor_1,
- * device_1.sensor_2, device_2.sensor_1, device_2.sensor_2, we will invoke
- * StorageEngine.getInstance().beginQuery(device_1)
and
- * StorageEngine.getInstance().beginQuery(device_2)
both once. Although there
- * exists four paths, but the unique devices are only `device_1` and `device_2`. When invoking
- * StorageEngine.getInstance().beginQuery(device_1)
, it returns result token `1`.
- * Similarly,
- * StorageEngine.getInstance().beginQuery(device_2)
returns result token `2`.
- *
- * In the meanwhile, another query process Q2 aroused by other client is triggered, whose sql
- * statement is same to Q1. Although StorageEngine.getInstance().beginQuery(device_1)
- *
- * and
- * StorageEngine.getInstance().beginQuery(device_2)
will be invoked again, it
- * returns result token `3` and `4` .
- *
- * StorageEngine.getInstance().endQueryForGivenJob(device_1, 1)
and
- * StorageEngine.getInstance().endQueryForGivenJob(device_2, 2)
must be invoked no matter how
- * query process Q1 exits normally or abnormally. So is Q2,
- * StorageEngine.getInstance().endQueryForGivenJob(device_1, 3)
and
- * StorageEngine.getInstance().endQueryForGivenJob(device_2, 4)
must be invoked
- *
- * Last but no least, to ensure the correctness of insert process and query process of IoTDB,
- * StorageEngine.getInstance().beginQuery()
and
- * StorageEngine.getInstance().endQueryForGivenJob()
must be executed rightly.
- *
- */
- private ConcurrentHashMap>> queryTokensMap;
private JobFileManager filePathsManager;
private AtomicLong maxJobId;
private QueryResourceManager() {
- queryTokensMap = new ConcurrentHashMap<>();
filePathsManager = new JobFileManager();
maxJobId = new AtomicLong(0);
}
@@ -104,86 +58,24 @@ public class QueryResourceManager {
*/
public long assignJobId() {
long jobId = maxJobId.incrementAndGet();
- queryTokensMap.computeIfAbsent(jobId, x -> new ConcurrentHashMap<>());
filePathsManager.addJobId(jobId);
return jobId;
}
- /**
- * Begin query and set query tokens of queryPaths. This method is used for projection
- * calculation.
- */
- public void beginQueryOfGivenQueryPaths(long jobId, List queryPaths)
- throws StorageEngineException {
- Set deviceIdSet = new HashSet<>();
- queryPaths.forEach(path -> deviceIdSet.add(path.getDevice()));
-
- for (String deviceId : deviceIdSet) {
- putQueryTokenForCurrentRequestThread(jobId, deviceId,
- StorageEngine.getInstance().beginQuery(deviceId));
- }
- }
-
- /**
- * Begin query and set query tokens of all paths in expression. This method is used in filter
- * calculation.
- */
- public void beginQueryOfGivenExpression(long jobId, IExpression expression)
- throws StorageEngineException {
- Set deviceIdSet = new HashSet<>();
- getUniquePaths(expression, deviceIdSet);
- for (String deviceId : deviceIdSet) {
- putQueryTokenForCurrentRequestThread(jobId, deviceId,
- StorageEngine.getInstance().beginQuery(deviceId));
- }
- }
-
- /**
- * Begin query and set query tokens of all filter paths in expression. This method is used in
- * filter calculation.
- * @param remoteDeviceIdSet device id set which can not handle locally
- * Note : the method is for cluster
- */
- public void beginQueryOfGivenExpression(long jobId, IExpression expression,
- Set remoteDeviceIdSet) throws StorageEngineException {
- Set deviceIdSet = new HashSet<>();
- getUniquePaths(expression, deviceIdSet);
- deviceIdSet.removeAll(remoteDeviceIdSet);
- for (String deviceId : deviceIdSet) {
- putQueryTokenForCurrentRequestThread(jobId, deviceId,
- StorageEngine.getInstance().beginQuery(deviceId));
- }
- }
-
public QueryDataSource getQueryDataSource(Path selectedPath,
QueryContext context) throws StorageEngineException {
SingleSeriesExpression singleSeriesExpression = new SingleSeriesExpression(selectedPath, null);
- QueryDataSource queryDataSource = StorageEngine
- .getInstance().query(singleSeriesExpression, context);
-
- // add used files to current thread request cached map
- filePathsManager.addUsedFilesForGivenJob(context.getJobId(), queryDataSource);
-
- return queryDataSource;
+ return StorageEngine
+ .getInstance().query(singleSeriesExpression, context, filePathsManager);
}
/**
* Whenever the jdbc request is closed normally or abnormally, this method must be invoked. All
* query tokens created by this jdbc request must be cleared.
*/
- public void endQueryForGivenJob(long jobId) throws StorageEngineException {
- if (queryTokensMap.get(jobId) == null) {
- // no resource need to be released.
- return;
- }
- for (Map.Entry> entry : queryTokensMap.get(jobId).entrySet()) {
- for (int token : entry.getValue()) {
- StorageEngine.getInstance().endQuery(entry.getKey(), token);
- }
- }
- queryTokensMap.remove(jobId);
+ public void endQueryForGivenJob(long jobId) {
// remove usage of opened file paths of current thread
filePathsManager.removeUsedFilesForGivenJob(jobId);
}
@@ -198,10 +90,6 @@ public class QueryResourceManager {
}
}
- private void putQueryTokenForCurrentRequestThread(long jobId, String deviceId, int queryToken) {
- queryTokensMap.get(jobId).computeIfAbsent(deviceId, x -> new ArrayList<>()).add(queryToken);
- }
-
private static class QueryTokenManagerHelper {
private static final QueryResourceManager INSTANCE = new QueryResourceManager();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
index 2993a7da7aa1c23d65f1993541574b77e5141ea5..d5771f3650c117a1b33161b691118d9cf21a4373 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
@@ -73,9 +73,6 @@ public class GroupByWithValueFilterDataSet extends GroupByEngineDataSet {
throws StorageEngineException, PathErrorException, ProcessorException, IOException {
initAggreFuction(aggres);
- QueryResourceManager.getInstance().beginQueryOfGivenExpression(context.getJobId(), expression);
- QueryResourceManager
- .getInstance().beginQueryOfGivenQueryPaths(context.getJobId(), selectedSeries);
this.timestampGenerator = new EngineTimeGenerator(expression, context);
this.allDataReaderList = new ArrayList<>();
for (Path path : selectedSeries) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 47ff47ad6b84a8cae33ebbef2a74bf37b23686bf..3043d09a8cf52471e29a53f5c0122dea70d4d70a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -76,7 +76,6 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
throws StorageEngineException, PathErrorException, ProcessorException, IOException {
initAggreFuction(aggres);
// init reader
- QueryResourceManager.getInstance().beginQueryOfGivenQueryPaths(jobId, selectedSeries);
if (expression != null) {
timeFilter = ((GlobalTimeExpression) expression).getFilter();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
index 6b31fe9682e4d99a2327d8af848f4c1d86777f4c..994ffa6467a01afc8b402d755816926af7799182 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
@@ -86,8 +86,6 @@ public class AggregateEngineExecutor {
if (expression != null) {
timeFilter = ((GlobalTimeExpression) expression).getFilter();
}
- QueryResourceManager
- .getInstance().beginQueryOfGivenQueryPaths(context.getJobId(), selectedSeries);
List readersOfSequenceData = new ArrayList<>();
List readersOfUnSequenceData = new ArrayList<>();
@@ -229,14 +227,16 @@ public class AggregateEngineExecutor {
} else {
// cal by pageData
BatchData batchData = sequenceReader.nextBatch();
- if (lastBatchTimeStamp > batchData.currentTime()) {
- // the chunk is end.
- isChunkEnd = true;
- } else {
- // current page and last page are in the same chunk.
- lastBatchTimeStamp = batchData.currentTime();
+ if (batchData.length() > 0) {
+ if (lastBatchTimeStamp > batchData.currentTime()) {
+ // the chunk is end.
+ isChunkEnd = true;
+ } else {
+ // current page and last page are in the same chunk.
+ lastBatchTimeStamp = batchData.currentTime();
+ }
+ function.calculateValueFromPageData(batchData, unSequenceReader);
}
- function.calculateValueFromPageData(batchData, unSequenceReader);
}
if (isChunkEnd) {
break;
@@ -258,9 +258,6 @@ public class AggregateEngineExecutor {
*/
public QueryDataSet executeWithValueFilter(QueryContext context)
throws StorageEngineException, PathErrorException, IOException, ProcessorException {
- QueryResourceManager
- .getInstance().beginQueryOfGivenQueryPaths(context.getJobId(), selectedSeries);
- QueryResourceManager.getInstance().beginQueryOfGivenExpression(context.getJobId(), expression);
EngineTimeGenerator timestampGenerator = new EngineTimeGenerator(expression, context);
List readersOfSelectedSeries = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
index 0279f9af57526e525e246bfae0deec82bf31cc27..c2bd99605be2457857f3b59219a698e4d380019d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java
@@ -56,8 +56,6 @@ public class EngineExecutor {
*/
public QueryDataSet executeWithoutValueFilter(QueryContext context)
throws StorageEngineException, IOException {
- QueryResourceManager.getInstance()
- .beginQueryOfGivenQueryPaths(context.getJobId(), queryExpression.getSelectedSeries());
Filter timeFilter = null;
if (queryExpression.hasQueryFilter()) {
@@ -94,11 +92,6 @@ public class EngineExecutor {
*/
public QueryDataSet executeWithValueFilter(QueryContext context) throws StorageEngineException, IOException {
- QueryResourceManager.getInstance()
- .beginQueryOfGivenQueryPaths(context.getJobId(), queryExpression.getSelectedSeries());
- QueryResourceManager.getInstance()
- .beginQueryOfGivenExpression(context.getJobId(), queryExpression.getExpression());
-
EngineTimeGenerator timestampGenerator = new EngineTimeGenerator(
queryExpression.getExpression(), context);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
index 20f9b006115f59c9f8fdd767e27f94f6cd37df20..3ac598bd0909b16cfae863f5422df15fb6ae463b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
@@ -58,7 +58,6 @@ public class FillEngineExecutor {
*/
public QueryDataSet execute(QueryContext context)
throws StorageEngineException, PathErrorException, IOException {
- QueryResourceManager.getInstance().beginQueryOfGivenQueryPaths(jobId, selectedSeries);
List fillList = new ArrayList<>();
List dataTypeList = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/IPointReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/IPointReader.java
index cca6968c2db644337ca0814fd7c0f34265a6822b..40a10e0fa424291b1f08d190eac628e2d63e72e6 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/IPointReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/IPointReader.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.utils.TimeValuePair;
public interface IPointReader {
+ // FIXME: this means hasCurrent actually
boolean hasNext() throws IOException;
TimeValuePair next() throws IOException;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/CachedDiskChunkReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/CachedDiskChunkReader.java
new file mode 100644
index 0000000000000000000000000000000000000000..6f09219ad8af9d9775d25eafbf835f3d22498fa7
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/CachedDiskChunkReader.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.reader.chunkRelated;
+
+import java.io.IOException;
+import org.apache.iotdb.db.query.reader.IPointReader;
+import org.apache.iotdb.db.utils.TimeValuePair;
+import org.apache.iotdb.db.utils.TimeValuePairUtils;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
+
+public class CachedDiskChunkReader implements IPointReader {
+
+ private ChunkReader chunkReader;
+ private BatchData data;
+ private TimeValuePair prev;
+ private TimeValuePair current;
+
+ public CachedDiskChunkReader(ChunkReader chunkReader) {
+ this.chunkReader = chunkReader;
+ this.prev =
+ TimeValuePairUtils.getEmptyTimeValuePair(chunkReader.getChunkHeader().getDataType());
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ if (data != null && data.hasNext()) {
+ return true;
+ }
+ while (chunkReader.hasNextBatch()) {
+ data = chunkReader.nextBatch();
+ if (data.hasNext()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public TimeValuePair next() throws IOException {
+ TimeValuePairUtils.setCurrentTimeValuePair(data, prev);
+ data.next();
+ if (data.hasNext()) {
+ TimeValuePairUtils.setCurrentTimeValuePair(data, current());
+ } else {
+ while (chunkReader.hasNextBatch()) {
+ data = chunkReader.nextBatch();
+ if (data.hasNext()) {
+ TimeValuePairUtils.setCurrentTimeValuePair(data, current());
+ break;
+ }
+ }
+ }
+ return prev;
+ }
+
+ @Override
+ public TimeValuePair current() {
+ if (current == null) {
+ this.current =
+ TimeValuePairUtils.getEmptyTimeValuePair(chunkReader.getChunkHeader().getDataType());
+ TimeValuePairUtils.setCurrentTimeValuePair(data, current);
+ }
+ return current;
+ }
+
+ @Override
+ public void close() {
+ this.chunkReader.close();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/DiskChunkReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/DiskChunkReader.java
index d2bae96d8213356f8600c6f0a4a6c50ae1bbda52..e272ffbeaeb2138d3404861bcc8737d8d4c157d0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/DiskChunkReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/DiskChunkReader.java
@@ -66,6 +66,7 @@ public class DiskChunkReader implements IPointReader {
@Override
public TimeValuePair current() {
+ // FIXME: if data.hasNext() = false and this method is called...
return TimeValuePairUtils.getCurrentTimeValuePair(data);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileIterateReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileIterateReader.java
index b0a3f0453da30f5d2785ad0bce3da2f2293769d6..2477076e2baf92d32347f02a66d730f490bdabcc 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileIterateReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileIterateReader.java
@@ -28,7 +28,7 @@ import org.apache.iotdb.db.query.reader.chunkRelated.MemChunkReader;
import org.apache.iotdb.db.query.reader.universal.IterateReader;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
+import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
@@ -114,8 +114,8 @@ public class UnSealedTsFileIterateReader extends IterateReader {
// prepare chunkLoader
TsFileSequenceReader unClosedTsFileReader = FileReaderManager.getInstance()
- .get(unSealedTsFile.getFile().getPath(), false);
- ChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader);
+ .get(unSealedTsFile, false);
+ IChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader);
// init fileSeriesReader
FileSeriesReader fileSeriesReader;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderByTimestamp.java b/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderByTimestamp.java
index f9fdd2709b0870e977a56e49bf44d6c5bc5e2045..86d6d76da6bc247ec5bfec0205f9e41a677ef8ea 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderByTimestamp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderByTimestamp.java
@@ -24,7 +24,7 @@ import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.IReaderByTimestamp;
import org.apache.iotdb.db.query.reader.chunkRelated.MemChunkReaderByTimestamp;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
+import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderByTimestamp;
@@ -64,8 +64,8 @@ public class UnSealedTsFileReaderByTimestamp implements IReaderByTimestamp {
// create FileSeriesReaderByTimestamp for data which has been flushed to disk
TsFileSequenceReader unClosedTsFileReader = FileReaderManager.getInstance()
- .get(unsealedTsFile.getFile().getPath(), false);
- ChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader);
+ .get(unsealedTsFile, false);
+ IChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader);
unSealedTsFileDiskReaderByTs = new FileSeriesReaderByTimestamp(chunkLoader,
unsealedTsFile.getChunkMetaDatas());
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/CachedUnseqResourceMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/CachedUnseqResourceMergeReader.java
new file mode 100644
index 0000000000000000000000000000000000000000..15525a2020f9f74c1c197292d1aaaad7324c9169
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/CachedUnseqResourceMergeReader.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.reader.resourceRelated;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.iotdb.db.query.reader.chunkRelated.CachedDiskChunkReader;
+import org.apache.iotdb.db.query.reader.universal.CachedPriorityMergeReader;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter;
+
+public class CachedUnseqResourceMergeReader extends CachedPriorityMergeReader {
+
+ public CachedUnseqResourceMergeReader(List chunks, TSDataType dataType)
+ throws IOException {
+ super(dataType);
+ int priorityValue = 1;
+ for (Chunk chunk : chunks) {
+ ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk);
+ addReaderWithPriority(new CachedDiskChunkReader(chunkReader), priorityValue++);
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java
index 9ba9128c05f5fbcca9c98ae85a61ce17673cfd00..b72dd63210ce968c9d49a4c2f8bcb5e6a3270b2c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java
@@ -34,7 +34,7 @@ import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
+import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
@@ -159,8 +159,7 @@ public class SeqResourceIterateReader extends IterateReader {
QueryContext context) throws IOException {
// prepare metaDataList
List