diff --git a/README.md b/README.md index 11074c8b118a792fd449044afc0a172eab59eadf..f62258238f212800dc791604efd6db324413f970 100644 --- a/README.md +++ b/README.md @@ -96,8 +96,8 @@ then by default `IOTDB_HOME` is the direct parent directory of `sbin/start-serve (or that of `sbin\start-server.bat` on Windows). * if `IOTDB_CLI_HOME` is not explicitly assigned, -then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-client.sh` on -Unix/OS X (or that of `sbin\start-client.bat` on Windows). +then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-cli.sh` on +Unix/OS X (or that of `sbin\start-cli.bat` on Windows). If you are not the first time that building IoTDB, remember deleting the following files: @@ -199,10 +199,10 @@ After build, the IoTDB client will be at the folder "client/target/iotdb-client- ``` # Unix/OS X -> $IOTDB_CLI_HOME/sbin/start-client.sh -h -p -u +> $IOTDB_CLI_HOME/sbin/start-cli.sh -h -p -u # Windows -> $IOTDB_CLI_HOME\sbin\start-client.bat -h -p -u +> $IOTDB_CLI_HOME\sbin\start-cli.bat -h -p -u ``` > NOTE: In the system, we set a default user in IoTDB named 'root'. The default password for 'root' is 'root'. You can use this default user if you are making the first try or you didn't create users by yourself. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 16330850588aa287f28c96cf5aa89009a687604f..10745b267eea395ad50a0500855ba8da8f8c9f1d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -130,7 +130,7 @@ If you use the previous unofficial version 0.7.0. It is incompatible with 0.8.0. * IOTDB-44 Error message in server log when select timeseries * IOTDB-49 Authorizer module outputs too many debug log info * IOTDB-50 DataSetWithoutTimeGenerator's initHeap behaves wrongly -* IOTDB-52 Client doesn't support aggregate +* IOTDB-52 Cli doesn't support aggregate * IOTDB-54 Predicates doesn't take effect * IOTDB-67 ValueDecoder reading new page bug * IOTDB-70 Disconnect from server when logging in fails @@ -147,7 +147,7 @@ If you use the previous unofficial version 0.7.0. It is incompatible with 0.8.0. * IOTDB-103 Does not give a hint when encountering unsupported data types * IOTDB-104 MManager is incorrectly recovered when system reboots * IOTDB-108 Mistakes in documents -* IOTDB-110 Clients inserts data normally even if there is no space left on the disk +* IOTDB-110 Clis inserts data normally even if there is no space left on the disk * IOTDB-118 When the disk space is full, the storage group is created successfully * IOTDB-121 A bug of query on value columns * IOTDB-128 Probably a bug in iotdb official website diff --git a/client/pom.xml b/client/pom.xml index 07da5c62323642b2d49b2b68d7dbe9b989da6fb8..ef1ea30ae9a655c71d9fb87015b746665d77fbca 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -29,7 +29,6 @@ iotdb-client IoTDB Client - A Client tool. false ${cli.test.skip} diff --git a/client/src/assembly/resources/sbin/start-client.bat b/client/src/assembly/resources/sbin/start-client.bat index b7f2dbb1691500ca280e9016665ef75fb112ca76..2642dfd9860e13478468b72b713c8f99bb32bc15 100755 --- a/client/src/assembly/resources/sbin/start-client.bat +++ b/client/src/assembly/resources/sbin/start-client.bat @@ -28,7 +28,7 @@ pushd %~dp0.. if NOT DEFINED IOTDB_CLI_HOME set IOTDB_CLI_HOME=%CD% popd -if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cli.client.Client +if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.client.Client if NOT DEFINED JAVA_HOME goto :err @REM ----------------------------------------------------------------------------- diff --git a/client/src/assembly/resources/sbin/start-client.sh b/client/src/assembly/resources/sbin/start-client.sh index 70e85037479569de7214ca26793ee367b07b0c34..4591159ddf783170aaae7711f6c4e86127904506 100755 --- a/client/src/assembly/resources/sbin/start-client.sh +++ b/client/src/assembly/resources/sbin/start-client.sh @@ -24,7 +24,7 @@ if [ -z "${IOTDB_CLI_HOME}" ]; then fi -MAIN_CLASS=org.apache.iotdb.cli.client.Client +MAIN_CLASS=org.apache.iotdb.client.Client CLASSPATH="" diff --git a/client/src/assembly/resources/tools/export-csv.bat b/client/src/assembly/resources/tools/export-csv.bat index a5c1ce26e9384aa2a0a40980d3319b51c113543d..f2432b8b6ce4d6c781be5bd2e2e0630528535145 100644 --- a/client/src/assembly/resources/tools/export-csv.bat +++ b/client/src/assembly/resources/tools/export-csv.bat @@ -28,7 +28,7 @@ pushd %~dp0.. if NOT DEFINED IOTDB_CLI_HOME set IOTDB_CLI_HOME=%CD% popd -if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cli.tool.ExportCsv +if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ExportCsv if NOT DEFINED JAVA_HOME goto :err @REM ----------------------------------------------------------------------------- diff --git a/client/src/assembly/resources/tools/export-csv.sh b/client/src/assembly/resources/tools/export-csv.sh index 940e2416e2f59406bbffb0fb2b70e5ed69e455f0..375e461361214a03b0767792c35887f817545440 100755 --- a/client/src/assembly/resources/tools/export-csv.sh +++ b/client/src/assembly/resources/tools/export-csv.sh @@ -47,7 +47,7 @@ for f in ${IOTDB_CLI_HOME}/lib/*.jar; do CLASSPATH=${CLASSPATH}":"$f done -MAIN_CLASS=org.apache.iotdb.cli.tool.ExportCsv +MAIN_CLASS=org.apache.iotdb.tool.ExportCsv "$JAVA" -DIOTDB_CLI_HOME=${IOTDB_CLI_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" exit $? \ No newline at end of file diff --git a/client/src/assembly/resources/tools/import-csv.bat b/client/src/assembly/resources/tools/import-csv.bat index 1d4cab7e1a82d7c8793d8215fe6fff781b7b64fe..ea514eae46acb077f70d46206c1fbd492855071a 100644 --- a/client/src/assembly/resources/tools/import-csv.bat +++ b/client/src/assembly/resources/tools/import-csv.bat @@ -28,7 +28,7 @@ pushd %~dp0.. if NOT DEFINED IOTDB_CLI_HOME set IOTDB_CLI_HOME=%CD% popd -if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cli.tool.ImportCsv +if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ImportCsv if NOT DEFINED JAVA_HOME goto :err @REM ----------------------------------------------------------------------------- diff --git a/client/src/assembly/resources/tools/import-csv.sh b/client/src/assembly/resources/tools/import-csv.sh index 80b700b7b97dd57413baa7c84fcf154de7587f9e..413dc4273431770023a4eaf6762681e4f189dc3f 100755 --- a/client/src/assembly/resources/tools/import-csv.sh +++ b/client/src/assembly/resources/tools/import-csv.sh @@ -47,7 +47,7 @@ for f in ${IOTDB_CLI_HOME}/lib/*.jar; do CLASSPATH=${CLASSPATH}":"$f done -MAIN_CLASS=org.apache.iotdb.cli.tool.ImportCsv +MAIN_CLASS=org.apache.iotdb.tool.ImportCsv "$JAVA" -DIOTDB_CLI_HOME=${IOTDB_CLI_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@" exit $? \ No newline at end of file diff --git a/client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java b/client/src/main/java/org/apache/iotdb/client/AbstractClient.java similarity index 80% rename from client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java rename to client/src/main/java/org/apache/iotdb/client/AbstractClient.java index 3cbf4522eec470491abf107b397388eec98d7e7e..c35cb0801608ca834a84920d5ecebf42ab248e8f 100644 --- a/client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java +++ b/client/src/main/java/org/apache/iotdb/client/AbstractClient.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.client; +package org.apache.iotdb.client; import java.io.PrintStream; import java.sql.ResultSet; @@ -38,65 +38,64 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.iotdb.cli.exception.ArgsErrorException; -import org.apache.iotdb.cli.tool.ImportCsv; +import org.apache.iotdb.exception.ArgsErrorException; +import org.apache.iotdb.tool.ImportCsv; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.iotdb.jdbc.IoTDBDatabaseMetadata; import org.apache.iotdb.jdbc.IoTDBMetadataResultSet; import org.apache.iotdb.jdbc.IoTDBQueryResultSet; -import org.apache.iotdb.jdbc.IoTDBSQLException; import org.apache.iotdb.service.rpc.thrift.ServerProperties; import org.apache.thrift.TException; public abstract class AbstractClient { - protected static final String HOST_ARGS = "h"; - protected static final String HOST_NAME = "host"; - - protected static final String HELP_ARGS = "help"; - - protected static final String PORT_ARGS = "p"; - protected static final String PORT_NAME = "port"; - - protected static final String PASSWORD_ARGS = "pw"; - protected static final String PASSWORD_NAME = "password"; - - protected static final String USERNAME_ARGS = "u"; - protected static final String USERNAME_NAME = "username"; - - protected static final String EXECUTE_ARGS = "e"; - protected static final String EXECUTE_NAME = "execute"; - - protected static final String ISO8601_ARGS = "disableISO8601"; - protected static final List AGGREGRATE_TIME_LIST = new ArrayList<>(); - protected static final String MAX_PRINT_ROW_COUNT_ARGS = "maxPRC"; - protected static final String MAX_PRINT_ROW_COUNT_NAME = "maxPrintRowCount"; - protected static final String SET_MAX_DISPLAY_NUM = "set max_display_num"; - protected static final String SET_TIMESTAMP_DISPLAY = "set time_display_type"; - protected static final String SHOW_TIMESTAMP_DISPLAY = "show time_display_type"; - protected static final String SET_TIME_ZONE = "set time_zone"; - protected static final String SHOW_TIMEZONE = "show time_zone"; - protected static final String SET_FETCH_SIZE = "set fetch_size"; - protected static final String SHOW_FETCH_SIZE = "show fetch_size"; - protected static final String HELP = "help"; - protected static final String IOTDB_CLI_PREFIX = "IoTDB"; - protected static final String SCRIPT_HINT = "./start-client.sh(start-client.bat if Windows)"; - protected static final String QUIT_COMMAND = "quit"; - protected static final String EXIT_COMMAND = "exit"; - protected static final String SHOW_METADATA_COMMAND = "show timeseries"; - protected static final int MAX_HELP_CONSOLE_WIDTH = 88; - protected static final String TIMESTAMP_STR = "Time"; - protected static final int ISO_DATETIME_LEN = 35; - protected static final String IMPORT_CMD = "import"; + static final String HOST_ARGS = "h"; + static final String HOST_NAME = "host"; + + static final String HELP_ARGS = "help"; + + static final String PORT_ARGS = "p"; + static final String PORT_NAME = "port"; + + static final String PASSWORD_ARGS = "pw"; + private static final String PASSWORD_NAME = "password"; + + static final String USERNAME_ARGS = "u"; + static final String USERNAME_NAME = "username"; + + private static final String EXECUTE_ARGS = "e"; + private static final String EXECUTE_NAME = "execute"; + + static final String ISO8601_ARGS = "disableISO8601"; + static final List AGGREGRATE_TIME_LIST = new ArrayList<>(); + static final String MAX_PRINT_ROW_COUNT_ARGS = "maxPRC"; + private static final String MAX_PRINT_ROW_COUNT_NAME = "maxPrintRowCount"; + static final String SET_MAX_DISPLAY_NUM = "set max_display_num"; + static final String SET_TIMESTAMP_DISPLAY = "set time_display_type"; + static final String SHOW_TIMESTAMP_DISPLAY = "show time_display_type"; + static final String SET_TIME_ZONE = "set time_zone"; + static final String SHOW_TIMEZONE = "show time_zone"; + static final String SET_FETCH_SIZE = "set fetch_size"; + static final String SHOW_FETCH_SIZE = "show fetch_size"; + private static final String HELP = "help"; + static final String IOTDB_CLI_PREFIX = "IoTDB"; + static final String SCRIPT_HINT = "./start-client.sh(start-client.bat if Windows)"; + static final String QUIT_COMMAND = "quit"; + static final String EXIT_COMMAND = "exit"; + static final String SHOW_METADATA_COMMAND = "show timeseries"; + static final int MAX_HELP_CONSOLE_WIDTH = 88; + static final String TIMESTAMP_STR = "Time"; + static final int ISO_DATETIME_LEN = 35; + private static final String IMPORT_CMD = "import"; private static final String NEED_NOT_TO_PRINT_TIMESTAMP = "AGGREGATION"; private static final String DEFAULT_TIME_FORMAT = "default"; - protected static String timeFormat = DEFAULT_TIME_FORMAT; - protected static int maxPrintRowCount = 1000; - protected static int fetchSize = 10000; - protected static int maxTimeLength = ISO_DATETIME_LEN; - protected static int maxValueLength = 15; - protected static boolean isQuit = false; - protected static String TIMESTAMP_PRECISION = "ms"; + private static String timeFormat = DEFAULT_TIME_FORMAT; + static int maxPrintRowCount = 1000; + private static int fetchSize = 10000; + static int maxTimeLength = ISO_DATETIME_LEN; + static int maxValueLength = 15; + private static boolean isQuit = false; + static String TIMESTAMP_PRECISION = "ms"; /** * control the width of columns for 'show timeseries path' and 'show storage group'. @@ -138,22 +137,22 @@ public abstract class AbstractClient { * *

*/ - protected static int[] maxValueLengthForShow = new int[]{75, 45, 8, 8}; - protected static String formatTime = "%" + maxTimeLength + "s|"; - protected static String formatValue = "%" + maxValueLength + "s|"; + private static int[] maxValueLengthForShow = new int[]{75, 45, 8, 8}; + static String formatTime = "%" + maxTimeLength + "s|"; + private static String formatValue = "%" + maxValueLength + "s|"; private static final int DIVIDING_LINE_LENGTH = 40; - protected static String host = "127.0.0.1"; - protected static String port = "6667"; - protected static String username; - protected static String password; - protected static String execute; - protected static boolean hasExecuteSQL = false; + static String host = "127.0.0.1"; + static String port = "6667"; + static String username; + static String password; + static String execute; + static boolean hasExecuteSQL = false; - protected static boolean printToConsole = true; + private static boolean printToConsole = true; - protected static Set keywordSet = new HashSet<>(); + static Set keywordSet = new HashSet<>(); - protected static ServerProperties properties = null; + static ServerProperties properties = null; private static boolean printHeader = false; private static int displayCnt = 0; @@ -165,7 +164,7 @@ public abstract class AbstractClient { */ private static boolean showException = false; - protected static void init() { + static void init() { keywordSet.add("-" + HOST_ARGS); keywordSet.add("-" + HELP_ARGS); keywordSet.add("-" + PORT_ARGS); @@ -177,14 +176,14 @@ public abstract class AbstractClient { } /** - * CLI result output. + * Client result output. * * @param res result set * @param printToConsole print to console * @param zoneId time-zone ID * @throws SQLException SQLException */ - public static void output(ResultSet res, boolean printToConsole, ZoneId zoneId) + private static void output(ResultSet res, boolean printToConsole, ZoneId zoneId) throws SQLException { int cnt = 0; boolean printTimestamp = true; @@ -225,15 +224,14 @@ public abstract class AbstractClient { } } - println(StringUtils.repeat('-', DIVIDING_LINE_LENGTH)); printCount(isShow, res, cnt); } - protected static String getTimestampPrecision() { + private static String getTimestampPrecision() { return TIMESTAMP_PRECISION; } - protected static void printCount(boolean isShow, ResultSet res, int cnt) throws SQLException { + private static void printCount(boolean isShow, ResultSet res, int cnt) throws SQLException { if (isShow) { int type = res.getType(); if (type == IoTDBMetadataResultSet.MetadataType.STORAGE_GROUP.ordinal()) { // storage group @@ -247,7 +245,7 @@ public abstract class AbstractClient { } } - protected static void printRow(boolean printTimestamp, int colCount, + private static void printRow(boolean printTimestamp, int colCount, ResultSetMetaData resultSetMetaData, boolean isShow, ResultSet res, ZoneId zoneId) throws SQLException { // Output Labels @@ -263,7 +261,7 @@ public abstract class AbstractClient { } } - protected static void printHeader(boolean printTimestamp, int colCount, + private static void printHeader(boolean printTimestamp, int colCount, ResultSetMetaData resultSetMetaData, boolean isShow) throws SQLException { if (!printHeader) { printBlockLine(printTimestamp, colCount, resultSetMetaData, isShow); @@ -273,7 +271,7 @@ public abstract class AbstractClient { } } - protected static void printShow(int colCount, ResultSet res) throws SQLException { + private static void printShow(int colCount, ResultSet res) throws SQLException { print("|"); for (int i = 1; i <= colCount; i++) { formatValue = "%" + maxValueLengthForShow[i - 1] + "s|"; @@ -282,7 +280,7 @@ public abstract class AbstractClient { println(); } - protected static void printRowData(boolean printTimestamp, ResultSet res, ZoneId zoneId, + private static void printRowData(boolean printTimestamp, ResultSet res, ZoneId zoneId, ResultSetMetaData resultSetMetaData, int colCount) throws SQLException { if (displayCnt < maxPrintRowCount) { // NOTE displayCnt only works on queried data results @@ -298,7 +296,7 @@ public abstract class AbstractClient { } } - protected static void printColumnData(ResultSetMetaData resultSetMetaData, ResultSet res, int i, + private static void printColumnData(ResultSetMetaData resultSetMetaData, ResultSet res, int i, ZoneId zoneId) throws SQLException { boolean flag = false; for (String timeStr : AGGREGRATE_TIME_LIST) { @@ -319,7 +317,7 @@ public abstract class AbstractClient { } } - protected static Options createOptions() { + static Options createOptions() { Options options = new Options(); Option help = new Option(HELP_ARGS, false, "Display help information(optional)"); help.setRequired(false); @@ -420,7 +418,7 @@ public abstract class AbstractClient { } } - protected static String checkRequiredArg(String arg, String name, CommandLine commandLine, + static String checkRequiredArg(String arg, String name, CommandLine commandLine, boolean isRequired, String defaultValue) throws ArgsErrorException { String str = commandLine.getOptionValue(arg); @@ -442,7 +440,7 @@ public abstract class AbstractClient { return str; } - protected static void setTimeFormat(String newTimeFormat) { + static void setTimeFormat(String newTimeFormat) { switch (newTimeFormat.trim().toLowerCase()) { case "long": case "number": @@ -475,7 +473,7 @@ public abstract class AbstractClient { } } - protected static void setMaxDisplayNumber(String maxDisplayNum) { + static void setMaxDisplayNumber(String maxDisplayNum) { long tmp = Long.parseLong(maxDisplayNum.trim()); if (tmp > Integer.MAX_VALUE || tmp < 0) { maxPrintRowCount = Integer.MAX_VALUE; @@ -484,7 +482,7 @@ public abstract class AbstractClient { } } - protected static void printBlockLine(boolean printTimestamp, int colCount, + private static void printBlockLine(boolean printTimestamp, int colCount, ResultSetMetaData resultSetMetaData, boolean isShowTs) throws SQLException { StringBuilder blockLine = new StringBuilder(); @@ -512,7 +510,7 @@ public abstract class AbstractClient { println(blockLine); } - protected static void printName(boolean printTimestamp, int colCount, + private static void printName(boolean printTimestamp, int colCount, ResultSetMetaData resultSetMetaData, boolean isShowTs) throws SQLException { print("|"); @@ -533,7 +531,7 @@ public abstract class AbstractClient { println(); } - protected static String[] removePasswordArgs(String[] args) { + static String[] removePasswordArgs(String[] args) { int index = -1; for (int i = 0; i < args.length; i++) { if (args[i].equals("-" + PASSWORD_ARGS)) { @@ -548,7 +546,7 @@ public abstract class AbstractClient { return args; } - protected static String[] processExecuteArgs(String[] args) { + static String[] processExecuteArgs(String[] args) { int index = -1; for (int i = 0; i < args.length; i++) { if (args[i].equals("-" + EXECUTE_ARGS)) { @@ -574,7 +572,7 @@ public abstract class AbstractClient { } } - protected static void displayLogo(String version) { + static void displayLogo(String version) { println(" _____ _________ ______ ______ \n" + "|_ _| | _ _ ||_ _ `.|_ _ \\ \n" + " | | .--.|_/ | | \\_| | | `. \\ | |_) | \n" @@ -584,13 +582,13 @@ public abstract class AbstractClient { + " \n"); } - protected static void echoStarting(){ + static void echoStarting(){ println("---------------------"); println("Starting IoTDB Client"); println("---------------------"); } - protected static OperationResult handleInputCmd(String cmd, IoTDBConnection connection) { + static OperationResult handleInputCmd(String cmd, IoTDBConnection connection) { String specialCmd = cmd.toLowerCase().trim(); if (QUIT_COMMAND.equals(specialCmd) || EXIT_COMMAND.equals(specialCmd)) { @@ -647,7 +645,7 @@ public abstract class AbstractClient { return OperationResult.NO_OPER; } - protected static void showHelp() { + private static void showHelp() { println(" \t\t\t execute your sql statment"); println(String.format(" %s\t\t show how many timeseries are in iotdb", SHOW_METADATA_COMMAND)); @@ -665,7 +663,7 @@ public abstract class AbstractClient { SET_MAX_DISPLAY_NUM)); } - protected static void showMetaData(IoTDBConnection connection) { + private static void showMetaData(IoTDBConnection connection) { try { println(((IoTDBDatabaseMetadata) connection.getMetaData()).getMetadataInJson()); } catch (SQLException e) { @@ -674,7 +672,7 @@ public abstract class AbstractClient { } } - protected static void setTimestampDisplay(String specialCmd, String cmd) { + private static void setTimestampDisplay(String specialCmd, String cmd) { String[] values = specialCmd.split("="); if (values.length != 2) { println(String.format("Time display format error, please input like %s=ISO8601", @@ -691,7 +689,7 @@ public abstract class AbstractClient { println("Time display type has set to " + cmd.split("=")[1].trim()); } - protected static void setTimeZone(String specialCmd, String cmd, IoTDBConnection connection) { + private static void setTimeZone(String specialCmd, String cmd, IoTDBConnection connection) { String[] values = specialCmd.split("="); if (values.length != 2) { println( @@ -708,7 +706,7 @@ public abstract class AbstractClient { println("Time zone has set to " + values[1].trim()); } - protected static void setFetchSize(String specialCmd, String cmd) { + private static void setFetchSize(String specialCmd, String cmd) { String[] values = specialCmd.split("="); if (values.length != 2) { println(String @@ -725,7 +723,7 @@ public abstract class AbstractClient { println("Fetch size has set to " + values[1].trim()); } - protected static void setMaxDisplaNum(String specialCmd, String cmd) { + private static void setMaxDisplaNum(String specialCmd, String cmd) { String[] values = specialCmd.split("="); if (values.length != 2) { println(String.format("Max display number format error, please input like %s = 10000", @@ -742,7 +740,7 @@ public abstract class AbstractClient { println("Max display number has set to " + values[1].trim()); } - protected static void showTimeZone(IoTDBConnection connection) { + private static void showTimeZone(IoTDBConnection connection) { try { println("Current time zone: " + connection.getTimeZone()); } catch (Exception e) { @@ -751,7 +749,7 @@ public abstract class AbstractClient { } } - protected static void importCmd(String specialCmd, String cmd, IoTDBConnection connection) { + private static void importCmd(String specialCmd, String cmd, IoTDBConnection connection) { String[] values = specialCmd.split(" "); if (values.length != 2) { println("Please input like: import /User/myfile. " @@ -772,12 +770,10 @@ public abstract class AbstractClient { } } - protected static void executeQuery(IoTDBConnection connection, String cmd) { - Statement statement = null; + private static void executeQuery(IoTDBConnection connection, String cmd) { long startTime = System.currentTimeMillis(); - try { + try (Statement statement = connection.createStatement();) { ZoneId zoneId = ZoneId.of(connection.getTimeZone()); - statement = connection.createStatement(); statement.setFetchSize(fetchSize); boolean hasResultSet = statement.execute(cmd.trim()); if (hasResultSet) { @@ -790,15 +786,6 @@ public abstract class AbstractClient { } catch (Exception e) { println("Msg: " + e.getMessage()); handleException(e); - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - println("Cannot close statement because: " + e.getMessage()); - handleException(e); - } - } } long costTime = System.currentTimeMillis() - startTime; println(String.format("It costs %.3fs", costTime / 1000.0)); @@ -808,27 +795,27 @@ public abstract class AbstractClient { STOP_OPER, CONTINUE_OPER, NO_OPER } - protected static void printf(String format, Object... args) { + private static void printf(String format, Object... args) { SCREEN_PRINTER.printf(format, args); } - protected static void print(String msg) { + static void print(String msg) { SCREEN_PRINTER.print(msg); } - protected static void println() { + private static void println() { SCREEN_PRINTER.println(); } - protected static void println(String msg) { + static void println(String msg) { SCREEN_PRINTER.println(msg); } - protected static void println(Object obj) { + private static void println(Object obj) { SCREEN_PRINTER.println(obj); } - protected static void handleException(Exception e) { + static void handleException(Exception e) { if (showException) { e.printStackTrace(SCREEN_PRINTER); } diff --git a/client/src/main/java/org/apache/iotdb/cli/client/Client.java b/client/src/main/java/org/apache/iotdb/client/Client.java similarity index 97% rename from client/src/main/java/org/apache/iotdb/cli/client/Client.java rename to client/src/main/java/org/apache/iotdb/client/Client.java index 8e865032e223ff3e9695b50560712072540c9c61..9678abcc3b2a33d5017517d6ed9e38f04359852a 100644 --- a/client/src/main/java/org/apache/iotdb/cli/client/Client.java +++ b/client/src/main/java/org/apache/iotdb/client/Client.java @@ -16,12 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.client; +package org.apache.iotdb.client; import java.io.IOException; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.Arrays; import jline.console.ConsoleReader; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -29,17 +28,20 @@ import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.iotdb.cli.exception.ArgsErrorException; +import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.thrift.TException; +/** + * usage: -h 127.0.0.1 -p 6667 -u root -pw root + */ public class Client extends AbstractClient { private static CommandLine commandLine; /** - * IoTDB CLI main function. + * IoTDB Client main function. * * @param args launch arguments * @throws ClassNotFoundException ClassNotFoundException diff --git a/client/src/main/java/org/apache/iotdb/cli/client/WinClient.java b/client/src/main/java/org/apache/iotdb/client/WinClient.java similarity index 97% rename from client/src/main/java/org/apache/iotdb/cli/client/WinClient.java rename to client/src/main/java/org/apache/iotdb/client/WinClient.java index b66f6586187c16d8888e3768755faa07d223a666..a174e10379eb7805d00cfa2ce3d0679cdf359b18 100644 --- a/client/src/main/java/org/apache/iotdb/cli/client/WinClient.java +++ b/client/src/main/java/org/apache/iotdb/client/WinClient.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.client; +package org.apache.iotdb.client; import java.io.Console; import java.sql.DriverManager; @@ -28,7 +28,7 @@ import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.iotdb.cli.exception.ArgsErrorException; +import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.thrift.TException; @@ -71,7 +71,7 @@ public class WinClient extends AbstractClient { private static String readPassword() { Console c = System.console(); - if (c == null) { // IN ECLIPSE IDE + if (c == null) { // IN ECLIENTPSE IDE print(IOTDB_CLI_PREFIX + "> please input password: "); Scanner scanner = new Scanner(System.in); return scanner.nextLine(); diff --git a/client/src/main/java/org/apache/iotdb/cli/exception/ArgsErrorException.java b/client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java similarity index 96% rename from client/src/main/java/org/apache/iotdb/cli/exception/ArgsErrorException.java rename to client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java index 66f35f1e2b91fd7972e6c3acdf4e14e56fda922b..f3c7c2b383d7da6b322eb8fe59c8ce2359045681 100644 --- a/client/src/main/java/org/apache/iotdb/cli/exception/ArgsErrorException.java +++ b/client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.exception; +package org.apache.iotdb.exception; public class ArgsErrorException extends Exception { diff --git a/client/src/main/java/org/apache/iotdb/cli/tool/AbstractCsvTool.java b/client/src/main/java/org/apache/iotdb/tool/AbstractCsvTool.java similarity index 98% rename from client/src/main/java/org/apache/iotdb/cli/tool/AbstractCsvTool.java rename to client/src/main/java/org/apache/iotdb/tool/AbstractCsvTool.java index 5045f7ad212fd471db54419bde04c2eebead3468..bd625b866a53f77f2c37dd337e6843631622a0fa 100644 --- a/client/src/main/java/org/apache/iotdb/cli/tool/AbstractCsvTool.java +++ b/client/src/main/java/org/apache/iotdb/tool/AbstractCsvTool.java @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.tool; +package org.apache.iotdb.tool; import java.io.IOException; import java.time.ZoneId; import jline.console.ConsoleReader; import org.apache.commons.cli.CommandLine; -import org.apache.iotdb.cli.exception.ArgsErrorException; +import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.iotdb.jdbc.IoTDBSQLException; import org.apache.thrift.TException; diff --git a/client/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java b/client/src/main/java/org/apache/iotdb/tool/ExportCsv.java similarity index 98% rename from client/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java rename to client/src/main/java/org/apache/iotdb/tool/ExportCsv.java index fd358d92c26b1f7795ae9bb8f019113511bb3f27..702d4b210f36dcd0595ee3b00891d7d69165c2be 100644 --- a/client/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java +++ b/client/src/main/java/org/apache/iotdb/tool/ExportCsv.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.cli.tool; +package org.apache.iotdb.tool; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -41,8 +41,8 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.iotdb.cli.client.AbstractClient; -import org.apache.iotdb.cli.exception.ArgsErrorException; +import org.apache.iotdb.client.AbstractClient; +import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.thrift.TException; diff --git a/client/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java b/client/src/main/java/org/apache/iotdb/tool/ImportCsv.java similarity index 99% rename from client/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java rename to client/src/main/java/org/apache/iotdb/tool/ImportCsv.java index 498af0c2893c17082d54d162985b35e0a2854a1c..81669505243556d8915f57b3a96d3b1b17f12850 100644 --- a/client/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java +++ b/client/src/main/java/org/apache/iotdb/tool/ImportCsv.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.tool; +package org.apache.iotdb.tool; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -46,13 +46,11 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; -import org.apache.iotdb.cli.exception.ArgsErrorException; +import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.jdbc.Constant; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.thrift.TException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * read a CSV formatted data File and insert all the data into IoTDB. diff --git a/client/src/test/java/org/apache/iotdb/cli/client/AbstractClientIT.java b/client/src/test/java/org/apache/iotdb/client/AbstractClientIT.java similarity index 74% rename from client/src/test/java/org/apache/iotdb/cli/client/AbstractClientIT.java rename to client/src/test/java/org/apache/iotdb/client/AbstractClientIT.java index 5f302b2c3c216d7a3e3ecdc98573c2fbba53c914..e9c6f15f9f29fac1d80480e8341b0d86a9630861 100644 --- a/client/src/test/java/org/apache/iotdb/cli/client/AbstractClientIT.java +++ b/client/src/test/java/org/apache/iotdb/client/AbstractClientIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.client; +package org.apache.iotdb.client; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -27,8 +27,8 @@ import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.iotdb.cli.client.AbstractClient.OperationResult; -import org.apache.iotdb.cli.exception.ArgsErrorException; +import org.apache.iotdb.client.AbstractClient.OperationResult; +import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.iotdb.jdbc.IoTDBDatabaseMetadata; import org.junit.After; @@ -143,30 +143,46 @@ public class AbstractClientIT { @Test public void testHandleInputInputCmd() { - assertEquals(OperationResult.STOP_OPER, AbstractClient.handleInputCmd(AbstractClient.EXIT_COMMAND, connection)); - assertEquals(OperationResult.STOP_OPER, AbstractClient.handleInputCmd(AbstractClient.QUIT_COMMAND, connection)); - - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(AbstractClient.SHOW_METADATA_COMMAND, connection)); - - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=", AbstractClient.SET_TIMESTAMP_DISPLAY), connection)); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=xxx", AbstractClient.SET_TIMESTAMP_DISPLAY), connection)); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=default", AbstractClient.SET_TIMESTAMP_DISPLAY), connection)); + assertEquals(OperationResult.STOP_OPER, AbstractClient + .handleInputCmd(AbstractClient.EXIT_COMMAND, connection)); + assertEquals(OperationResult.STOP_OPER, AbstractClient + .handleInputCmd(AbstractClient.QUIT_COMMAND, connection)); + + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(AbstractClient.SHOW_METADATA_COMMAND, connection)); + + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=", AbstractClient.SET_TIMESTAMP_DISPLAY), connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=xxx", AbstractClient.SET_TIMESTAMP_DISPLAY), connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=default", AbstractClient.SET_TIMESTAMP_DISPLAY), connection)); testSetTimeFormat(); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=", AbstractClient.SET_MAX_DISPLAY_NUM), connection)); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=xxx", AbstractClient.SET_MAX_DISPLAY_NUM),connection)); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=1", AbstractClient.SET_MAX_DISPLAY_NUM), connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=", AbstractClient.SET_MAX_DISPLAY_NUM), connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=xxx", AbstractClient.SET_MAX_DISPLAY_NUM),connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=1", AbstractClient.SET_MAX_DISPLAY_NUM), connection)); testSetMaxDisplayNumber(); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(AbstractClient.SHOW_TIMEZONE, connection)); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(AbstractClient.SHOW_TIMESTAMP_DISPLAY, connection)); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(AbstractClient.SHOW_FETCH_SIZE, connection)); - - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=", AbstractClient.SET_TIME_ZONE), connection)); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=+08:00", AbstractClient.SET_TIME_ZONE), connection)); - - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=", AbstractClient.SET_FETCH_SIZE), connection)); - assertEquals(OperationResult.CONTINUE_OPER, AbstractClient.handleInputCmd(String.format("%s=111", AbstractClient.SET_FETCH_SIZE), connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(AbstractClient.SHOW_TIMEZONE, connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(AbstractClient.SHOW_TIMESTAMP_DISPLAY, connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(AbstractClient.SHOW_FETCH_SIZE, connection)); + + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=", AbstractClient.SET_TIME_ZONE), connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=+08:00", AbstractClient.SET_TIME_ZONE), connection)); + + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=", AbstractClient.SET_FETCH_SIZE), connection)); + assertEquals(OperationResult.CONTINUE_OPER, AbstractClient + .handleInputCmd(String.format("%s=111", AbstractClient.SET_FETCH_SIZE), connection)); } private void testSetTimeFormat() { diff --git a/client/src/test/java/org/apache/iotdb/cli/client/AbstractScript.java b/client/src/test/java/org/apache/iotdb/client/AbstractScript.java similarity index 98% rename from client/src/test/java/org/apache/iotdb/cli/client/AbstractScript.java rename to client/src/test/java/org/apache/iotdb/client/AbstractScript.java index 4cd1a403b5336f54d66289c802d22b244f900b90..79c5daf19e819c80a6a95d134a807ede86d25817 100644 --- a/client/src/test/java/org/apache/iotdb/cli/client/AbstractScript.java +++ b/client/src/test/java/org/apache/iotdb/client/AbstractScript.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.client; +package org.apache.iotdb.client; import static org.junit.Assert.assertEquals; diff --git a/client/src/test/java/org/apache/iotdb/cli/client/StartClientScriptIT.java b/client/src/test/java/org/apache/iotdb/client/StartClientScriptIT.java similarity index 98% rename from client/src/test/java/org/apache/iotdb/cli/client/StartClientScriptIT.java rename to client/src/test/java/org/apache/iotdb/client/StartClientScriptIT.java index 85fdcd827e2e65bb4e26b48a719bb93f4b9ffd07..28a291701ebf66a8a2f3f0a3b23f8bc6defea4e0 100644 --- a/client/src/test/java/org/apache/iotdb/cli/client/StartClientScriptIT.java +++ b/client/src/test/java/org/apache/iotdb/client/StartClientScriptIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.client; +package org.apache.iotdb.client; import java.io.File; import java.io.IOException; diff --git a/client/src/test/java/org/apache/iotdb/cli/tool/ExportCsvTestIT.java b/client/src/test/java/org/apache/iotdb/tool/ExportCsvTestIT.java similarity index 96% rename from client/src/test/java/org/apache/iotdb/cli/tool/ExportCsvTestIT.java rename to client/src/test/java/org/apache/iotdb/tool/ExportCsvTestIT.java index de30750a22152e480a337de3618e09d7cd73a4fa..7d0fd202f759795ae2715525e9b84cc698992203 100644 --- a/client/src/test/java/org/apache/iotdb/cli/tool/ExportCsvTestIT.java +++ b/client/src/test/java/org/apache/iotdb/tool/ExportCsvTestIT.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.tool; +package org.apache.iotdb.tool; import java.io.File; import java.io.IOException; -import org.apache.iotdb.cli.client.AbstractScript; +import org.apache.iotdb.client.AbstractScript; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/client/src/test/java/org/apache/iotdb/cli/tool/ImportCsvTestIT.java b/client/src/test/java/org/apache/iotdb/tool/ImportCsvTestIT.java similarity index 96% rename from client/src/test/java/org/apache/iotdb/cli/tool/ImportCsvTestIT.java rename to client/src/test/java/org/apache/iotdb/tool/ImportCsvTestIT.java index 8bbe91bf56ad265177568ef7ae140dc7a23d24d2..95863b492710c3e3b110eb72687a43811f731a0b 100644 --- a/client/src/test/java/org/apache/iotdb/cli/tool/ImportCsvTestIT.java +++ b/client/src/test/java/org/apache/iotdb/tool/ImportCsvTestIT.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.cli.tool; +package org.apache.iotdb.tool; import java.io.File; import java.io.IOException; -import org.apache.iotdb.cli.client.AbstractScript; +import org.apache.iotdb.client.AbstractScript; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/docs/Development.md b/docs/Development.md index ee00bef054a4c1e3b7c132a002ff0a30e2b18708..d3d026d51e3c69124534fc923ccd867159b141f0 100644 --- a/docs/Development.md +++ b/docs/Development.md @@ -74,8 +74,8 @@ you can just choose menu "import" -> "Maven" -> "Existing Maven Projects". ## Debugging IoTDB The main class of IoTDB server is `org.apache.iotdb.db.service.IoTDB`. -The main class of IoTDB client is `org.apache.iotdb.cli.client.Client` -(or `org.apache.iotdb.cli.client.WinClient` on Win OS). +The main class of IoTDB cli is `org.apache.iotdb.client.Client` +(or `org.apache.iotdb.client.WinClient` on Win OS). You can run/debug IoTDB by using the two classes as the entrance. diff --git a/docs/Documentation-CHN/QuickStart.md b/docs/Documentation-CHN/QuickStart.md index 9fa9a16462a7582b7e060a4a6568923676456bc2..4d189d03d100f7583015e963d17e647eb3a6fe40 100755 --- a/docs/Documentation-CHN/QuickStart.md +++ b/docs/Documentation-CHN/QuickStart.md @@ -88,12 +88,12 @@ IoTDB为用户提供多种与服务器交互的方式，您可以选择使用Cli Linux系统与MacOS系统启动命令如下： ``` -> $IOTDB_HOME/sbin/start-client.sh -h 127.0.0.1 -p 6667 -u root +> $IOTDB_HOME/sbin/start-cli.sh -h 127.0.0.1 -p 6667 -u root ``` Windows系统启动命令如下： ``` -> $IOTDB_HOME\sbin\start-client.bat -h 127.0.0.1 -p 6667 -u root +> $IOTDB_HOME\sbin\start-cli.bat -h 127.0.0.1 -p 6667 -u root ``` 回车后输入root用户的密码，即可成功启动客户端。启动后出现如图提示即为启动成功。 @@ -286,7 +286,7 @@ IoTDB> SELECT * FROM root.ln.wf01.wt01 +-----------------------+--------------------------+-----------------------------+ ``` -输入quit或exit可退出Client结束本次会话，Client输出quit normally表示退出成功，操作语句与返回结果如下： +输入quit或exit可退出Cli结束本次会话，Cli输出quit normally表示退出成功，操作语句与返回结果如下： ``` IoTDB> quit quit normally diff --git a/docs/Documentation-CHN/UserGuide/1-Overview/3-Scenario.md b/docs/Documentation-CHN/UserGuide/1-Overview/3-Scenario.md index 95176b33a840045c1f02d2088842497e0038bce8..79fe86e82dbaab7b690662a85a4148aa5457d113 100644 --- a/docs/Documentation-CHN/UserGuide/1-Overview/3-Scenario.md +++ b/docs/Documentation-CHN/UserGuide/1-Overview/3-Scenario.md @@ -57,7 +57,7 @@ 为了保证机械手的监控数据能够及时监控和分析，公司需要收集这些机械手传感器信息，将其发送至可以连接外部网络的服务器上，而后将原始数据信息上传到数据中心进行复杂的计算和分析。 -此时，可以采用IoTDB套件中的IoTDB、IoTDB-CLI工具、TsFileSync工具和Hadoop/Spark集成组件等。将IoTDB服务器安装在工厂连接外网的服务器上，用户接收机械手传输的数据并将数据上传到数据中心。将IoTDB-CLI工具安装在每一个连接工厂内网的机械手上，用于将传感器产生的实时数据上传到工厂内部服务器。再使用TsFileSync工具将原始数据上传到数据中心。此外还需要部署Hadoop/Spark集群用于数据中心端的数据存储和分析。如图1.6中间场景所示。 +此时，可以采用IoTDB套件中的IoTDB、IoTDB-Client工具、TsFileSync工具和Hadoop/Spark集成组件等。将IoTDB服务器安装在工厂连接外网的服务器上，用户接收机械手传输的数据并将数据上传到数据中心。将IoTDB-Client工具安装在每一个连接工厂内网的机械手上，用于将传感器产生的实时数据上传到工厂内部服务器。再使用TsFileSync工具将原始数据上传到数据中心。此外还需要部署Hadoop/Spark集群用于数据中心端的数据存储和分析。如图1.6中间场景所示。 @@ -71,7 +71,7 @@ 为了能够实时接收汽车传感器所采集的物联网数据，公司需要在车辆行驶的过程中将传感器数据通过窄带物联网实时发送至数据中心，而后在数据中心的服务器上进行复杂的计算和分析。 -此时，可以采用IoTDB套件中的IoTDB、IoTDB-CLI和Hadoop/Spark集成组件等。将IoTDB-CLI工具安装在每一辆车联网内的车辆上，使用IoTDB-JDBC工具将数据直接传回数据中心的服务器。 +此时，可以采用IoTDB套件中的IoTDB、IoTDB-Client和Hadoop/Spark集成组件等。将IoTDB-Client工具安装在每一辆车联网内的车辆上，使用IoTDB-JDBC工具将数据直接传回数据中心的服务器。 此外还需要部署Hadoop/Spark集群用于数据中心端的数据存储和分析。如图1.8所示。 diff --git a/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md b/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md new file mode 100644 index 0000000000000000000000000000000000000000..15f77d90e1a69c6875a2da796db013be8402e69f --- /dev/null +++ b/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md @@ -0,0 +1,64 @@ + + +# Chapter6: JDBC API + +## Status Code + +在最新版本中引入了**状态码**这一概念。例如，因为IoTDB需要在写入数据之前首先注册时间序列，一种可能的解决方案是： + +``` +try { + writeData(); +} catch (SQLException e) { + // the most case is that the time series does not exist + if (e.getMessage().contains("exist")) { + //However, using the content of the error message is not so efficient + registerTimeSeries(); + //write data once again + writeData(); + } +} + +``` + +利用状态码，我们就可以不必写诸如`if (e.getErrorMessage().contains("exist"))`的代码，只需要使用`e.getStatusType().getCode() == TSStatusType.TIME_SERIES_NOT_EXIST_ERROR.getStatusCode()`。 + +这里是状态码和相对应信息的列表： + +|状态码|状态类型|状态信息| +|:---|:---|:---| +|200|SUCCESS_STATUS|| +|201|STILL_EXECUTING_STATUS|| +|202|INVALID_HANDLE_STATUS|| +|301|TIMESERIES_NOT_EXIST_ERROR|时间序列不存在| +|302|UNSUPPORTED_FETCH_METADATA_OPERATION_ERROR|不支持的获取元数据操作| +|303|FETCH_METADATA_ERROR|获取元数据失败| +|304|CHECK_FILE_LEVEL_ERROR|检查文件层级错误| +|400|EXECUTE_STATEMENT_ERROR|执行语句错误| +|401|SQL_PARSE_ERROR|SQL语句分析错误| +|402|GENERATE_TIME_ZONE_ERROR|生成时区错误| +|403|SET_TIME_ZONE_ERROR|设置时区错误| +|500|INTERNAL_SERVER_ERROR|服务器内部错误| +|600|WRONG_LOGIN_PASSWORD_ERROR|用户名或密码错误| +|601|NOT_LOGIN_ERROR|没有登录| +|602|NO_PERMISSION_ERROR|没有操作权限| +|603|UNINITIALIZED_AUTH_ERROR|授权人未初始化| diff --git a/docs/Documentation-CHN/UserGuide/7-Session API/1-Session API.md b/docs/Documentation-CHN/UserGuide/7-Session API/1-Session API.md new file mode 100644 index 0000000000000000000000000000000000000000..9b21474072f7ca30c7a77511a8de3b52ad44a305 --- /dev/null +++ b/docs/Documentation-CHN/UserGuide/7-Session API/1-Session API.md @@ -0,0 +1,105 @@ + + +# 第7章: Session API + +# 使用方式 + +## 依赖 + +* JDK >= 1.8 +* Maven >= 3.0 + +## 安装到本地 maven 库 + +In root directory: +> mvn clean install -pl session -am -Dmaven.test.skip=true + +## 在 maven 中使用 session 接口 + +``` + + + org.apache.iotdb + iotdb-session + 0.9.0-SNAPSHOT + + +``` + +## Session 接口使用示例 + + +```Java +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.record.RowBatch; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; + +public class SessionExample { + + public static void main(String[] args) throws ClassNotFoundException, IoTDBSessionException { + Session session = new Session("127.0.0.1", 6667, "root", "root"); + session.open(); + + session.setStorageGroup("root.sg1"); + session.createTimeseriesResp("root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE); + session.createTimeseriesResp("root.sg1.d1.s2", TSDataType.INT64, TSEncoding.RLE); + session.createTimeseriesResp("root.sg1.d1.s3", TSDataType.INT64, TSEncoding.RLE); + + Schema schema = new Schema(); + schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); + schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); + schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE)); + + RowBatch rowBatch = schema.createRowBatch("root.sg1.d1", 100); + + long[] timestamps = rowBatch.timestamps; + Object[] values = rowBatch.values; + + for (long time = 0; time < 30000; time++) { + int row = rowBatch.batchSize++; + timestamps[row] = time; + for (int i = 0; i < 3; i++) { + long[] sensor = (long[]) values[i]; + sensor[row] = time; + } + if (rowBatch.batchSize == rowBatch.getMaxBatchSize()) { + session.insertBatch(rowBatch); + rowBatch.reset(); + } + } + + if (rowBatch.batchSize != 0) { + session.insertBatch(rowBatch); + rowBatch.reset(); + } + session.close(); + } +} +``` + +> The code is in example/session/src/main/java/org/apache/iotdb/session/SessionExample.java \ No newline at end of file diff --git a/docs/Documentation-CHN/UserGuide/7-TsFile/1-Installation.md b/docs/Documentation-CHN/UserGuide/8-TsFile/1-Installation.md similarity index 96% rename from docs/Documentation-CHN/UserGuide/7-TsFile/1-Installation.md rename to docs/Documentation-CHN/UserGuide/8-TsFile/1-Installation.md index 1813bb63df807f0582d127bcc619ac5128092cdb..77409d34ddf6562e9ebc55299f4542645608a649 100644 --- a/docs/Documentation-CHN/UserGuide/7-TsFile/1-Installation.md +++ b/docs/Documentation-CHN/UserGuide/8-TsFile/1-Installation.md @@ -19,6 +19,6 @@ --> -# Chaper 7: TsFile +# 第8章: TsFile Coming Soon. \ No newline at end of file diff --git a/docs/Documentation-CHN/UserGuide/7-TsFile/2-Usage.md b/docs/Documentation-CHN/UserGuide/8-TsFile/2-Usage.md similarity index 96% rename from docs/Documentation-CHN/UserGuide/7-TsFile/2-Usage.md rename to docs/Documentation-CHN/UserGuide/8-TsFile/2-Usage.md index 1813bb63df807f0582d127bcc619ac5128092cdb..77409d34ddf6562e9ebc55299f4542645608a649 100644 --- a/docs/Documentation-CHN/UserGuide/7-TsFile/2-Usage.md +++ b/docs/Documentation-CHN/UserGuide/8-TsFile/2-Usage.md @@ -19,6 +19,6 @@ --> -# Chaper 7: TsFile +# 第8章: TsFile Coming Soon. \ No newline at end of file diff --git a/docs/Documentation-CHN/UserGuide/7-TsFile/3-Hierarchy.md b/docs/Documentation-CHN/UserGuide/8-TsFile/3-Hierarchy.md similarity index 96% rename from docs/Documentation-CHN/UserGuide/7-TsFile/3-Hierarchy.md rename to docs/Documentation-CHN/UserGuide/8-TsFile/3-Hierarchy.md index 1813bb63df807f0582d127bcc619ac5128092cdb..77409d34ddf6562e9ebc55299f4542645608a649 100644 --- a/docs/Documentation-CHN/UserGuide/7-TsFile/3-Hierarchy.md +++ b/docs/Documentation-CHN/UserGuide/8-TsFile/3-Hierarchy.md @@ -19,6 +19,6 @@ --> -# Chaper 7: TsFile +# 第8章: TsFile Coming Soon. \ No newline at end of file diff --git a/docs/Documentation-CHN/UserGuide/8-System Tools/1-Sync.md b/docs/Documentation-CHN/UserGuide/9-System Tools/1-Sync.md similarity index 99% rename from docs/Documentation-CHN/UserGuide/8-System Tools/1-Sync.md rename to docs/Documentation-CHN/UserGuide/9-System Tools/1-Sync.md index dbe5eb9a0800f3f26bc8bf62cf854a700870cf2a..cddf0c173911d91aca39e9126384c2872ee97915 100644 --- a/docs/Documentation-CHN/UserGuide/8-System Tools/1-Sync.md +++ b/docs/Documentation-CHN/UserGuide/9-System Tools/1-Sync.md @@ -19,7 +19,7 @@ --> -# 同步工具 +# 第9章: 同步工具 - [同步工具](#同步工具) diff --git a/docs/Documentation-CHN/UserGuide/8-System Tools/2-Memory Estimation Tool.md b/docs/Documentation-CHN/UserGuide/9-System Tools/2-Memory Estimation Tool.md similarity index 99% rename from docs/Documentation-CHN/UserGuide/8-System Tools/2-Memory Estimation Tool.md rename to docs/Documentation-CHN/UserGuide/9-System Tools/2-Memory Estimation Tool.md index 60526702424cd44bba68f74ad52fc3258523628f..692d30bbdf411d581c2044914ad1ac9399d7bc5c 100644 --- a/docs/Documentation-CHN/UserGuide/8-System Tools/2-Memory Estimation Tool.md +++ b/docs/Documentation-CHN/UserGuide/9-System Tools/2-Memory Estimation Tool.md @@ -19,7 +19,7 @@ --> -# 内存预估工具 +# 第9章: 内存预估工具 diff --git a/docs/Documentation-CHN/UserGuide/8-Tools-Cli.md b/docs/Documentation-CHN/UserGuide/9-Tools-Cli.md similarity index 76% rename from docs/Documentation-CHN/UserGuide/8-Tools-Cli.md rename to docs/Documentation-CHN/UserGuide/9-Tools-Cli.md index 0b00442a3a5029674eb9a0fe40ad61a53b03bb48..f4e0c1e309b18eb4abb82517059688bf24e4f0f5 100644 --- a/docs/Documentation-CHN/UserGuide/8-Tools-Cli.md +++ b/docs/Documentation-CHN/UserGuide/9-Tools-Cli.md @@ -29,7 +29,7 @@ # Cli / Shell工具 -IOTDB为用户提供CLI/Shell工具用于启动客户端和服务端程序。下面介绍每个CLI/Shell工具的运行方式和相关参数。 +IOTDB为用户提供Client/Shell工具用于启动客户端和服务端程序。下面介绍每个Client/Shell工具的运行方式和相关参数。 > \$IOTDB\_HOME表示IoTDB的安装目录所在路径。 ## Cli / Shell运行方式 @@ -38,12 +38,12 @@ IOTDB为用户提供CLI/Shell工具用于启动客户端和服务端程序。下 Linux系统与MacOS系统启动命令如下： ``` - Shell > ./sbin/start-client.sh -h 127.0.0.1 -p 6667 -u root -pw root + Shell > ./sbin/start-cli.sh -h 127.0.0.1 -p 6667 -u root -pw root ``` Windows系统启动命令如下： ``` - Shell > \sbin\start-client.bat -h 127.0.0.1 -p 6667 -u root -pw root + Shell > \sbin\start-cli.bat -h 127.0.0.1 -p 6667 -u root -pw root ``` 回车后即可成功启动客户端。启动后出现如图提示即为启动成功。 ``` @@ -79,22 +79,22 @@ IoTDB> Linux系统与MacOS系统启动命令如下： ``` - Shell >./sbin/start-client.sh -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10 + Shell >./sbin/start-cli.sh -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10 ``` Windows系统启动命令如下： ``` - Shell > \sbin\start-client.bat -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10 + Shell > \sbin\start-cli.bat -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10 ``` ## Cli / Shell的-e参数 当您想要通过脚本的方式通过Cli / Shell对IoTDB进行批量操作时，可以使用-e参数。通过使用该参数，您可以在不进入客户端输入模式的情况下操作IoTDB。 为了避免SQL语句和其他参数混淆，现在只支持-e参数作为最后的参数使用。 -针对CLI/Shell工具的-e参数用法如下： +针对Client/Shell工具的-e参数用法如下： ``` - Shell > ./sbin/start-client.sh -h {host} -p {port} -u {user} -pw {password} -e {sql for iotdb} + Shell > ./sbin/start-cli.sh -h {host} -p {port} -u {user} -pw {password} -e {sql for iotdb} ``` 为了更好的解释-e参数的使用，可以参考下面的例子。 @@ -109,7 +109,7 @@ Windows系统启动命令如下： 4.查询验证数据是否插入成功 -那么通过使用CLI/Shell工具的-e参数，可以采用如下的脚本： +那么通过使用Client/Shell工具的-e参数，可以采用如下的脚本： ``` # !/bin/bash @@ -119,12 +119,12 @@ port=6667 user=root pass=root -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "set storage group to root.demo" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "create timeseries root.demo.s1 WITH DATATYPE=INT32, ENCODING=RLE" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(1,10)" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(2,11)" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(3,12)" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "select s1 from root.demo" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "set storage group to root.demo" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "create timeseries root.demo.s1 WITH DATATYPE=INT32, ENCODING=RLE" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(1,10)" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(2,11)" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(3,12)" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "select s1 from root.demo" ``` 打印出来的结果显示在下图，通过这种方式进行的操作与客户端的输入模式以及通过JDBC进行操作结果是一致的。 diff --git a/docs/Documentation-CHN/UserGuide/8-Tools-Grafana.md b/docs/Documentation-CHN/UserGuide/9-Tools-Grafana.md similarity index 100% rename from docs/Documentation-CHN/UserGuide/8-Tools-Grafana.md rename to docs/Documentation-CHN/UserGuide/9-Tools-Grafana.md diff --git a/docs/Documentation-CHN/UserGuide/8-Tools-Hadoop.md b/docs/Documentation-CHN/UserGuide/9-Tools-Hadoop.md similarity index 100% rename from docs/Documentation-CHN/UserGuide/8-Tools-Hadoop.md rename to docs/Documentation-CHN/UserGuide/9-Tools-Hadoop.md diff --git a/docs/Documentation-CHN/UserGuide/8-Tools-spark.md b/docs/Documentation-CHN/UserGuide/9-Tools-spark.md similarity index 100% rename from docs/Documentation-CHN/UserGuide/8-Tools-spark.md rename to docs/Documentation-CHN/UserGuide/9-Tools-spark.md diff --git a/docs/Documentation/Frequently asked questions.md b/docs/Documentation/Frequently asked questions.md index b359c67ee4a7ba20c3306c220ca3ec3bdb2b9e29..096e4decfa1d86f3920926ad7eeb00b53acb4700 100644 --- a/docs/Documentation/Frequently asked questions.md +++ b/docs/Documentation/Frequently asked questions.md @@ -31,7 +31,7 @@ - Can I use Hadoop and Spark to read TsFile in IoTDB? - How does IoTDB handle duplicate points? - How can I tell what type of the specific timeseries? - - How can I change IoTDB's CLI time display format? + - How can I change IoTDB's Client time display format? # Frequently Asked Questions @@ -43,19 +43,19 @@ There are several ways to identify the version of IoTDB that you are using: * Launch IoTDB's Command Line Interface: ``` -> ./start-client.sh -p 6667 -pw root -u root -h localhost +> ./start-cli.sh -p 6667 -pw root -u root -h localhost _____ _________ ______ ______ |_ _| | _ _ ||_ _ `.|_ _ \ | | .--.|_/ | | \_| | | `. \ | |_) | | | / .'`\ \ | | | | | | | __'. _| |_| \__. | _| |_ _| |_.' /_| |__) | -|_____|'.__.' |_____| |______.'|_______/ version 0.7.0 +|_____|'.__.' |_____| |______.'|_______/ version x.x.x ``` * Check pom.xml file: ``` -0.7.0 +0.9.0-SNAPSHOT ``` * Use JDBC API: @@ -137,11 +137,11 @@ Otherwise, you can also use wildcard in timeseries path: IoTDB> show timeseries root.fit.d1.* ``` -## How can I change IoTDB's CLI time display format? +## How can I change IoTDB's Client time display format? -The default IoTDB's CLI time display format is human readable (e.g. ```1970-01-01T08:00:00.001```), if you want to display time in timestamp type or other readable format, add parameter ```-disableIS08601``` in start command: +The default IoTDB's Client time display format is human readable (e.g. ```1970-01-01T08:00:00.001```), if you want to display time in timestamp type or other readable format, add parameter ```-disableIS08601``` in start command: ``` -> $IOTDB_CLI_HOME/sbin/start-client.sh -h 127.0.0.1 -p 6667 -u root -pw root -disableIS08601 +> $IOTDB_CLI_HOME/sbin/start-cli.sh -h 127.0.0.1 -p 6667 -u root -pw root -disableIS08601 ``` diff --git a/docs/Documentation/QuickStart.md b/docs/Documentation/QuickStart.md index 3d5e579901a9b8fdf59d834246021a6a89a3242e..d36489a44fcb28559952638ddca8becac1145b54 100755 --- a/docs/Documentation/QuickStart.md +++ b/docs/Documentation/QuickStart.md @@ -30,7 +30,7 @@ - Configure - Start - Start Server - - Start Client + - Start Cli - Have a try - Stop Server @@ -89,8 +89,8 @@ then by default `IOTDB_HOME` is the direct parent directory of `sbin/start-serve (or that of `sbin\start-server.bat` on Windows). * if `IOTDB_CLI_HOME` is not explicitly assigned, -then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-client.sh` on -Unix/OS X (or that of `sbin\start-client.bat` on Windows). +then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-cli.sh` on +Unix/OS X (or that of `sbin\start-cli.bat` on Windows). If you are not the first time that building IoTDB, remember deleting the following files: @@ -170,9 +170,9 @@ After that we start the server. Running the startup script: > $IOTDB_HOME\sbin\start-server.bat ``` -### Start Client +### Start Cli -Now let's trying to read and write some data from IoTDB using our Client. To start the client, you need to explicit the server's IP and PORT as well as the USER_NAME and PASSWORD. +Now let's trying to read and write some data from IoTDB using our Cli. To start the client, you need to explicit the server's IP and PORT as well as the USER_NAME and PASSWORD. ``` # You can first build cli project @@ -182,10 +182,10 @@ Now let's trying to read and write some data from IoTDB using our Client. To sta > mvn clean package -pl client -am -Dmaven.test.skip=true # Unix/OS X -> $IOTDB_CLI_HOME/sbin/start-client.sh -h -p -u +> $IOTDB_CLI_HOME/sbin/start-cli.sh -h -p -u # Windows -> $IOTDB_CLI_HOME\sbin\start-client.bat -h -p -u +> $IOTDB_CLI_HOME\sbin\start-cli.bat -h -p -u ``` > NOTE: In the system, we set a default user in IoTDB named 'root'. The default password for 'root' is 'root'. You can use this default user if you are making the first try or you didn't create users by yourself. diff --git a/docs/Documentation/UserGuide/0-Content.md b/docs/Documentation/UserGuide/0-Content.md index 1b245d2135fb2a5c3c24b614304511e0a3f825c2..07aa1cf618e8196529a1a164410687fd411b6992 100644 --- a/docs/Documentation/UserGuide/0-Content.md +++ b/docs/Documentation/UserGuide/0-Content.md @@ -49,10 +49,13 @@ * 2-Reference # Chapter 6: JDBC API * 1-JDBC API -# Chapter 7: TsFile +* 2-Status Code +# Chapter 7: Session API +* 1-Session API +# Chapter 8: TsFile * 1-Installation * 2-Usage * 3-Hierarchy -# Chapter 8: System Tools +# Chapter 9: System Tools * 1-Sync * 2-Memory Estimation Tool diff --git a/docs/Documentation/UserGuide/1-Overview/3-Scenario.md b/docs/Documentation/UserGuide/1-Overview/3-Scenario.md index 879c5cbbceb9d2e952276c20c0fd2abbae464305..bac5ec91b8007eb47f9c0e9d2229c28fb75e12a6 100644 --- a/docs/Documentation/UserGuide/1-Overview/3-Scenario.md +++ b/docs/Documentation/UserGuide/1-Overview/3-Scenario.md @@ -61,7 +61,7 @@ In order to ensure that the data of the robot can be monitored and analyzed in t -At this point, IoTDB, IoTDB-CLI tools, TsFileSync tools, and Hadoop/Spark integration components in the IoTDB suite can be used. IoTDB-CLI tool is installed on the robot and each of them is connected to the LAN of the factory. When sensors generate real-time data, the data will be uploaded to the server in the factory. The IoTDB server and TsFileSync is installed on the server connected to the external network. Once triggered, the data on the server will be upload to the data center. In addition, Hadoop/Spark clusters need to be deployed for data storage and analysis on the data center side. As shown in Figure 1.6. Figure 1.7 shows the architecture at this time. +At this point, IoTDB, IoTDB-Client tools, TsFileSync tools, and Hadoop/Spark integration components in the IoTDB suite can be used. IoTDB-Client tool is installed on the robot and each of them is connected to the LAN of the factory. When sensors generate real-time data, the data will be uploaded to the server in the factory. The IoTDB server and TsFileSync is installed on the server connected to the external network. Once triggered, the data on the server will be upload to the data center. In addition, Hadoop/Spark clusters need to be deployed for data storage and analysis on the data center side. As shown in Figure 1.6. Figure 1.7 shows the architecture at this time. @@ -71,7 +71,7 @@ A car company installed sensors on its cars to collect monitoring information su In order to receive the IoT data collected by the car sensor in real time, the company needs to send the sensor data to the data center in real time through the narrowband IoT while the vehicle is running. Thus, they can perform complex calculations and analysis on the server in the data center. -At this point, IoTDB, IoTDB-CLI, and Hadoop/Spark integration components in the IoTDB suite can be used. IoTDB-CLI tool is installed on each car and use IoTDB-JDBC tool to send data directly back to the server in the data center. +At this point, IoTDB, IoTDB-Client, and Hadoop/Spark integration components in the IoTDB suite can be used. IoTDB-Client tool is installed on each car and use IoTDB-JDBC tool to send data directly back to the server in the data center. In addition, Hadoop/Spark clusters need to be deployed for data storage and analysis on the data center side. As shown in Figure 1.8. diff --git a/docs/Documentation/UserGuide/3-Operation Manual/3-Data Import.md b/docs/Documentation/UserGuide/3-Operation Manual/3-Data Import.md index 78740162affcb4d8fc034a2d90473c89073518d0..542e8ed4b08170e8595cb8097c1622387107e092 100644 --- a/docs/Documentation/UserGuide/3-Operation Manual/3-Data Import.md +++ b/docs/Documentation/UserGuide/3-Operation Manual/3-Data Import.md @@ -28,7 +28,7 @@ This feature is not supported in version 0.7.0. ### Import Real-time Data -IoTDB provides users with a variety of ways to insert real-time data, such as directly inputting [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1) in [Cli/Shell tools](/#/Tools/Cli), or using [Java JDBC](/#/Documents/0.8.0/chap6/sec1) to perform single or batch execution of [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1). +IoTDB provides users with a variety of ways to insert real-time data, such as directly inputting [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1) in [Client/Shell tools](/#/Tools/Client), or using [Java JDBC](/#/Documents/0.8.0/chap6/sec1) to perform single or batch execution of [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1). This section mainly introduces the use of [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1) for real-time data import in the scenario. See Section 5.1 for a detailed syntax of [INSERT SQL statement](/#/Documents/0.8.0/chap5/sec1). diff --git a/docs/Documentation/UserGuide/4-Deployment and Management/1-Deployment.md b/docs/Documentation/UserGuide/4-Deployment and Management/1-Deployment.md index 13960415f8388577fd23f594839394cc2bdc3ef8..5d6ebf7f8e64d290f1e6b2e9f801ccaa45cb9a09 100644 --- a/docs/Documentation/UserGuide/4-Deployment and Management/1-Deployment.md +++ b/docs/Documentation/UserGuide/4-Deployment and Management/1-Deployment.md @@ -104,8 +104,8 @@ then by default `IOTDB_HOME` is the direct parent directory of `sbin/start-serve (or that of `sbin\start-server.bat` on Windows). * if `IOTDB_CLI_HOME` is not explicitly assigned, -then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-client.sh` on -Unix/OS X (or that of `sbin\start-client.bat` on Windows). +then by default `IOTDB_CLI_HOME` is the direct parent directory of `sbin/start-cli.sh` on +Unix/OS X (or that of `sbin\start-cli.bat` on Windows). If you are not the first time that building IoTDB, remember deleting the following files: diff --git a/docs/Documentation/UserGuide/4-Deployment and Management/7-Build and use IoTDB by Dockerfile.md b/docs/Documentation/UserGuide/4-Deployment and Management/7-Build and use IoTDB by Dockerfile.md index 8b1f2a0f3a6fadc008437f11733b9fb8086db126..1924d1145104fe7a1cdf1ae2c9c78e4f4e63c893 100644 --- a/docs/Documentation/UserGuide/4-Deployment and Management/7-Build and use IoTDB by Dockerfile.md +++ b/docs/Documentation/UserGuide/4-Deployment and Management/7-Build and use IoTDB by Dockerfile.md @@ -66,16 +66,16 @@ suppose the IP is . 4. If you just want to have a try by using iotdb-cli, you can: ``` $ docker exec -it /bin/bash -$ (now you have enter the container): /cli/sbin/start-client.sh -h localhost -p 6667 -u root -pw root +$ (now you have enter the container): /cli/sbin/start-cli.sh -h localhost -p 6667 -u root -pw root ``` Or, run a new docker container as the client: ``` -$ docker run -it iotdb:base /cli/sbin/start-client.sh -h -p 6667 -u root -pw root +$ docker run -it iotdb:base /cli/sbin/start-cli.sh -h -p 6667 -u root -pw root ``` Or, if you have a iotdb-cli locally (e.g., you have compiled the source code by `mvn package`), and suppose your work_dir is cli/bin, then you can just run: ``` -$ start-client.sh -h localhost -p 6667 -u root -pw root +$ start-cli.sh -h localhost -p 6667 -u root -pw root ``` 5. If you want to write codes to insert data and query data, please add the following dependence: ```xml diff --git a/docs/Documentation/UserGuide/6-JDBC API/1-JDBC API.md b/docs/Documentation/UserGuide/6-JDBC API/1-JDBC API.md index e194a35bffac22c0391f3f781c6690ce7ca8e27d..d1c865f7f7e107f492db147bc4d4ee0bca3192f6 100644 --- a/docs/Documentation/UserGuide/6-JDBC API/1-JDBC API.md +++ b/docs/Documentation/UserGuide/6-JDBC API/1-JDBC API.md @@ -19,7 +19,7 @@ --> -# Chaper6: JDBC API +# Chapter6: JDBC API # Usage @@ -57,6 +57,8 @@ This chapter provides an example of how to open a database connection, execute a Requires that you include the packages containing the JDBC classes needed for database programming. +**NOTE: For faster insertion, the insertBatch() in Session is recommended.** + ```Java import java.sql.*; import org.apache.iotdb.jdbc.IoTDBSQLException; diff --git a/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md b/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md new file mode 100644 index 0000000000000000000000000000000000000000..ea62c35776b7a5b93691a52d9fc42e52ccd4a26c --- /dev/null +++ b/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md @@ -0,0 +1,64 @@ + + +# Chapter6: JDBC API + +## Status Code + +**Status Code** is introduced in the latest version. For example, as IoTDB requires registering the time series first before writing data, a kind of solution is: + +``` +try { + writeData(); +} catch (SQLException e) { + // the most case is that the time series does not exist + if (e.getMessage().contains("exist")) { + //However, using the content of the error message is not so efficient + registerTimeSeries(); + //write data once again + writeData(); + } +} + +``` + +With Status Code, instead of writing codes like `if (e.getErrorMessage().contains("exist"))`, we can simply use `e.getStatusType().getCode() == TSStatusType.TIME_SERIES_NOT_EXIST_ERROR.getStatusCode()`. + +Here is a list of Status Code and related message: + +|Status Code|Status Type|Status Message| +|:---|:---|:---| +|200|SUCCESS_STATUS|| +|201|STILL_EXECUTING_STATUS|| +|202|INVALID_HANDLE_STATUS|| +|301|TIMESERIES_NOT_EXIST_ERROR|Timeseries does not exist| +|302|UNSUPPORTED_FETCH_METADATA_OPERATION_ERROR|Unsupported fetch metadata operation| +|303|FETCH_METADATA_ERROR|Failed to fetch metadata| +|304|CHECK_FILE_LEVEL_ERROR|Meet error while checking file level| +|400|EXECUTE_STATEMENT_ERROR|Execute statement error| +|401|SQL_PARSE_ERROR|Meet error while parsing SQL| +|402|GENERATE_TIME_ZONE_ERROR|Meet error while generating time zone| +|403|SET_TIME_ZONE_ERROR|Meet error while setting time zone| +|500|INTERNAL_SERVER_ERROR|Internal server error| +|600|WRONG_LOGIN_PASSWORD_ERROR|Username or password is wrong| +|601|NOT_LOGIN_ERROR|Has not logged in| +|602|NO_PERMISSION_ERROR|No permissions for this operation| +|603|UNINITIALIZED_AUTH_ERROR|Uninitialized authorizer| diff --git a/docs/Documentation/UserGuide/7-Session API/1-Session API.md b/docs/Documentation/UserGuide/7-Session API/1-Session API.md new file mode 100644 index 0000000000000000000000000000000000000000..b2cd10cf21b05ed1cc36a4809f86490f6377e1e5 --- /dev/null +++ b/docs/Documentation/UserGuide/7-Session API/1-Session API.md @@ -0,0 +1,115 @@ + + +# Chapter7: Session API + +# Usage + +## Dependencies + +* JDK >= 1.8 +* Maven >= 3.0 + +## How to package only client module + +In root directory: +> mvn clean package -pl client -am -Dmaven.test.skip=true + +## How to install in local maven repository + +In root directory: +> mvn clean install -pl client -am -Dmaven.test.skip=true + +## Using IoTDB Session with Maven + +``` + + + org.apache.iotdb + iotdb-client + 0.9.0-SNAPSHOT + + +``` + + +## Examples with Session + +This chapter provides an example of how to open an IoTDB session, execute a batch insertion. + +Requires that you include the packages containing the Client classes needed for database programming. + +```Java +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.record.RowBatch; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; + +public class SessionExample { + + public static void main(String[] args) throws ClassNotFoundException, IoTDBSessionException { + Session session = new Session("127.0.0.1", 6667, "root", "root"); + session.open(); + + session.setStorageGroup("root.sg1"); + session.createTimeseriesResp("root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE); + session.createTimeseriesResp("root.sg1.d1.s2", TSDataType.INT64, TSEncoding.RLE); + session.createTimeseriesResp("root.sg1.d1.s3", TSDataType.INT64, TSEncoding.RLE); + + Schema schema = new Schema(); + schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); + schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); + schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE)); + + RowBatch rowBatch = schema.createRowBatch("root.sg1.d1", 100); + + long[] timestamps = rowBatch.timestamps; + Object[] values = rowBatch.values; + + for (long time = 0; time < 30000; time++) { + int row = rowBatch.batchSize++; + timestamps[row] = time; + for (int i = 0; i < 3; i++) { + long[] sensor = (long[]) values[i]; + sensor[row] = time; + } + if (rowBatch.batchSize == rowBatch.getMaxBatchSize()) { + session.insertBatch(rowBatch); + rowBatch.reset(); + } + } + + if (rowBatch.batchSize != 0) { + session.insertBatch(rowBatch); + rowBatch.reset(); + } + + session.close(); + } +} +``` + +> The code is in example/session/src/main/java/org/apache/iotdb/session/SessionExample.java \ No newline at end of file diff --git a/docs/Documentation/UserGuide/7-TsFile/1-Installation.md b/docs/Documentation/UserGuide/8-TsFile/1-Installation.md similarity index 99% rename from docs/Documentation/UserGuide/7-TsFile/1-Installation.md rename to docs/Documentation/UserGuide/8-TsFile/1-Installation.md index 9c3c9ae950b3daf4d55d91cca1dab26366e6b99d..269d524c8c97f4931fab19361c573eae54e5e057 100644 --- a/docs/Documentation/UserGuide/7-TsFile/1-Installation.md +++ b/docs/Documentation/UserGuide/8-TsFile/1-Installation.md @@ -19,7 +19,7 @@ --> -# Chapter 7: TsFile +# Chapter 8: TsFile ## Installation diff --git a/docs/Documentation/UserGuide/7-TsFile/2-Usage.md b/docs/Documentation/UserGuide/8-TsFile/2-Usage.md similarity index 96% rename from docs/Documentation/UserGuide/7-TsFile/2-Usage.md rename to docs/Documentation/UserGuide/8-TsFile/2-Usage.md index eed28a5b954af6e4528be121f81c6ec0dea60c2f..ae8bb14f8f3dd1858791571bd7fdb19b384f13da 100644 --- a/docs/Documentation/UserGuide/7-TsFile/2-Usage.md +++ b/docs/Documentation/UserGuide/8-TsFile/2-Usage.md @@ -19,7 +19,7 @@ --> -# Chapter 7: TsFile +# Chapter 8: TsFile ## Usage @@ -85,12 +85,12 @@ A TsFile can be generated by following three steps and the complete code will be ``` * With pre-defined schema ``` - public TsFileWriter(File file, FileSchema schema) throws IOException + public TsFileWriter(File file, Schema schema) throws IOException ``` This one is for using the HDFS file system. `TsFileOutput` can be an instance of class `HDFSOutput`. ``` - public TsFileWriter(TsFileOutput output, FileSchema schema) throws IOException + public TsFileWriter(TsFileOutput output, Schema schema) throws IOException ``` **Parameters:** @@ -100,15 +100,15 @@ A TsFile can be generated by following three steps and the complete code will be * Second, add measurements - Or you can make an instance of class `FileSchema` first and pass this to the constructor of class `TsFileWriter` + Or you can make an instance of class `Schema` first and pass this to the constructor of class `TsFileWriter` - The class `FileSchema` contains a map whose key is the name of one measurement schema, and the value is the schema itself. + The class `Schema` contains a map whose key is the name of one measurement schema, and the value is the schema itself. Here are the interfaces: ``` - // Create an empty FileSchema or from an existing map - public FileSchema() - public FileSchema(Map measurements) + // Create an empty Schema or from an existing map + public Schema() + public Schema(Map measurements) // Use this two interfaces to add measurements public void registerMeasurement(MeasurementSchema descriptor) @@ -259,7 +259,7 @@ import java.io.File; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.write.TsFileWriter; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.apache.iotdb.tsfile.write.record.RowBatch; /** @@ -275,7 +275,7 @@ public class TsFileWriteWithRowBatch { f.delete(); } - FileSchema fileSchema = new FileSchema(); + Schema schema = new Schema(); // the number of rows to include in the row batch int rowNum = 1000000; @@ -284,15 +284,15 @@ public class TsFileWriteWithRowBatch { // add measurements into file schema (all with INT64 data type) for (int i = 0; i < sensorNum; i++) { - fileSchema.registerMeasurement( + schema.registerMeasurement( new MeasurementSchema("sensor_" + (i + 1), TSDataType.INT64, TSEncoding.TS_2DIFF)); } // add measurements into TSFileWriter - TsFileWriter tsFileWriter = new TsFileWriter(f, fileSchema); + TsFileWriter tsFileWriter = new TsFileWriter(f, schema); // construct the row batch - RowBatch rowBatch = fileSchema.createRowBatch("device_1"); + RowBatch rowBatch = schema.createRowBatch("device_1"); long[] timestamps = rowBatch.timestamps; Object[] values = rowBatch.values; diff --git a/docs/Documentation/UserGuide/7-TsFile/3-Hierarchy.md b/docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md similarity index 68% rename from docs/Documentation/UserGuide/7-TsFile/3-Hierarchy.md rename to docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md index 643417f54ec4810d0ebe27849479c7b8c99ece09..732c5e688e78da2b1acaefc5ecde125ea65f3c6d 100644 --- a/docs/Documentation/UserGuide/7-TsFile/3-Hierarchy.md +++ b/docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md @@ -7,9 +7,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,53 +18,63 @@ under the License. --> -# Chapter 7: TsFile + +# Chapter 8: TsFile + ## TsFile Hierarchy + Here is a brief introduction of the structure of a TsFile file. - + ## Variable Storage - * **Big Endian** - - * For Example, the `int` `0x8` will be stored as `00 00 00 08`, not `08 00 00 00` - - * **String with Variable Length** - - * The format is `int size` plus `String literal`. Size can be zero. - - * Size equals the number of bytes this string will take, and it may not equal to the length of the string. - - * For example "sensor_1" will be stored as `00 00 00 08` plus the encoding(ASCII) of "sensor_1". - - * Note that for the "Magic String"(file signature) "TsFilev0.8.0", the size(12) and encoding(ASCII) + +- **Big Endian** + + - For Example, the `int` `0x8` will be stored as `00 00 00 08`, not `08 00 00 00` +- **String with Variable Length** + - The format is `int size` plus `String literal`. Size can be zero. + - Size equals the number of bytes this string will take, and it may not equal to the length of the string. + - For example "sensor_1" will be stored as `00 00 00 08` plus the encoding(ASCII) of "sensor_1". + - Note that for the "Magic String"(file signature) "TsFilev0.8.0", the size(12) and encoding(ASCII) is fixed so there is no need to put the size before this string literal. - - * **Data Type Hardcode** - * 0: BOOLEAN - * 1: INT32 (`int`) - * 2: INT64 (`long`) - * 3: FLOAT - * 4: DOUBLE - * 5: TEXT (`String`) - * **Encoding Type Hardcode** - * 0: PLAIN - * 1: PLAIN_DICTIONARY - * 2: RLE - * 3: DIFF - * 4: TS_2DIFF - * 5: BITMAP - * 6: GORILLA - * 7: REGULAR - * **Compressing Type Hardcode** - * 0: UNCOMPRESSED - * 1: SNAPPY - - +- **Data Type Hardcode** + - 0: BOOLEAN + - 1: INT32 (`int`) + - 2: INT64 (`long`) + - 3: FLOAT + - 4: DOUBLE + - 5: TEXT (`String`) +- **Encoding Type Hardcode** + - 0: PLAIN + - 1: PLAIN_DICTIONARY + - 2: RLE + - 3: DIFF + - 4: TS_2DIFF + - 5: BITMAP + - 6: GORILLA + - 7: REGULAR +- **Compressing Type Hardcode** + - 0: UNCOMPRESSED + - 1: SNAPPY + - 2: GZIP + - 3: LZO + - 4: SDT + - 5: PAA + - 6: PLA +- **TsDigest Statistics Type Hardcode** + - 0: min_value + - 1: max_value + - 2: first_value + - 3: last_value + - 4: sum_value + ## TsFile Overview + Here is a graph about the TsFile structure. ![TsFile Breakdown](https://user-images.githubusercontent.com/40447846/61616997-6fad1300-ac9c-11e9-9c17-46785ebfbc88.png) ## Magic String + There is a 12 bytes magic string: `TsFilev0.8.0` @@ -87,6 +97,7 @@ The `ChunkGroup` has an array of `Chunk`, a following byte `0x00` as the marker, A `Chunk` represents a *sensor*. There is a byte `0x01` as the marker, following a `ChunkHeader` and an array of `Page`. ###### ChunkHeader +
@@ -136,6 +147,7 @@ PageHeader Structure ## Metadata ### TsDeviceMetaData + The first part of metadata is `TsDeviceMetaData`
@@ -148,6 +160,7 @@ The first part of metadata is `TsDeviceMetaData`
Then there is an array of `ChunkGroupMetaData` after `TsDeviceMetaData` + ### ChunkGroupMetaData
@@ -179,14 +192,57 @@ Then there is an array of `ChunkMetadata` for each `ChunkGroupMetadata`
Member DescriptionMember Type
-###### TsDigest +###### TsDigest (updated on 2019/8/27) + +Right now there are five statistics: `min_value, max_value, first_value, last_value, sum_value`. + +In v0.8.0, the storage format of statistics is a name-value pair. That is, `Map statistics`. The name is a string (remember the length is before the literal). But for the value, there is also an integer byteLength acting as the self description length of the following value because the value may be of various type. For example, if the `min_value` is an integer 0, then it will be stored as [9 "min_value" 4 0] in the TsFile. + +The figure below shows an example of `TsDigest.deserializeFrom(buffer)`. In v0.8.0, we will get -There are five statistics: `min, last, sum, first, max` +``` +Map statistics = { + "min_value" -> ByteBuffer of int value 0, + "last" -> ByteBuffer of int value 19, + "sum" -> ByteBuffer of double value 1093347116, + "first" -> ByteBuffer of int value 0, + "max_value" -> ByteBuffer of int value 99 +} +``` -The storage format is a name-value pair. The name is a string (remember the length is before the literal). +![TsDigest ByteBuffer Breakdown comparison](https://user-images.githubusercontent.com/33376433/63765352-664a4280-c8fb-11e9-869e-859edf6d00bb.png) -But for the value, there is also a size integer before the data even if it is not string. For example, if the `min` is 3, then it will be -stored as 3 "min" 4 3 in the TsFile. +In v0.9.0, the storage format is changed to an array for space and time efficiency. That is, `ByteBuffer[] statistics`. Each position of the array has a fixed association with a specific type of statistic, following the order defined in StatisticType: + +``` +enum StatisticType { + min_value, max_value, first_value, last_value, sum_value +} +``` + +Therefore, in the above example, we will get + +``` +ByteBuffer[] statistics = [ + ByteBuffer of int value 0, // associated with "min_value" + ByteBuffer of int value 99, // associated with "max_value" + ByteBuffer of int value 0, // associated with "first_value" + ByteBuffer of int value 19, // associated with "last_value" + ByteBuffer of double value 1093347116 // associated with "sum_value" +] +``` + +As another example in v0.9.0, when deserializing a TsDigest from buffer [3, 0,4,0, 1,4,99, 3,4,19], we get + +``` +ByteBuffer[] statistics = [ + ByteBuffer of int value 0, // associated with "min_value" + ByteBuffer of int value 99, // associated with "max_value" + null, // associated with "first_value" + ByteBuffer of int value 19, // associated with "last_value" + null // associated with "sum_value" +] +``` #### File Metadata @@ -207,6 +263,7 @@ After the array of `ChunkGroupMetadata`, here is the last part of the metadata. ##### DeviceIndexMetadata +
@@ -219,6 +276,7 @@ After the array of `ChunkGroupMetadata`, here is the last part of the metadata. ##### MeasurementSchema +
Member DescriptionMember Type
diff --git a/docs/Documentation/UserGuide/8-System Tools/1-Sync.md b/docs/Documentation/UserGuide/9-System Tools/1-Sync.md similarity index 99% rename from docs/Documentation/UserGuide/8-System Tools/1-Sync.md rename to docs/Documentation/UserGuide/9-System Tools/1-Sync.md index 741b36528edee82064802f3900a3a400cacd55b3..c41c34d44516c0ae5689ade0c365e629bf316c3a 100644 --- a/docs/Documentation/UserGuide/8-System Tools/1-Sync.md +++ b/docs/Documentation/UserGuide/9-System Tools/1-Sync.md @@ -19,7 +19,7 @@ --> -# Chapter 8: System Tools +# Chapter 9: System Tools ## Data Import diff --git a/docs/Documentation/UserGuide/8-System Tools/2-Memory Estimation Tool.md b/docs/Documentation/UserGuide/9-System Tools/2-Memory Estimation Tool.md similarity index 99% rename from docs/Documentation/UserGuide/8-System Tools/2-Memory Estimation Tool.md rename to docs/Documentation/UserGuide/9-System Tools/2-Memory Estimation Tool.md index e11aedbf05d30c522682e3785e1b8adb1467f7f0..a38793f8f2b2a94a617c60746e9c789fc4131ea7 100644 --- a/docs/Documentation/UserGuide/8-System Tools/2-Memory Estimation Tool.md +++ b/docs/Documentation/UserGuide/9-System Tools/2-Memory Estimation Tool.md @@ -19,7 +19,7 @@ --> -# Chapter 8: System Tools +# Chapter 9: System Tools # Memory Estimation Tool diff --git a/docs/Documentation/UserGuide/8-Tools-Cli.md b/docs/Documentation/UserGuide/9-Tools-Cli.md similarity index 80% rename from docs/Documentation/UserGuide/8-Tools-Cli.md rename to docs/Documentation/UserGuide/9-Tools-Cli.md index ff5df5419dceb21a07672b77d2fcfba0b653fd4a..adc42f6f9f12a4c1ff82e3f56d6f0400107423fc 100644 --- a/docs/Documentation/UserGuide/8-Tools-Cli.md +++ b/docs/Documentation/UserGuide/9-Tools-Cli.md @@ -44,12 +44,12 @@ the port number of the server running, set the specific IP and PORT at -h and -p The Linux and MacOS system startup commands are as follows: ``` - Shell > ./sbin/start-client.sh -h 127.0.0.1 -p 6667 -u root -pw root + Shell > ./sbin/start-cli.sh -h 127.0.0.1 -p 6667 -u root -pw root ``` The Windows system startup commands are as follows: ``` - Shell > \sbin\start-client.bat -h 127.0.0.1 -p 6667 -u root -pw root + Shell > \sbin\start-cli.bat -h 127.0.0.1 -p 6667 -u root -pw root ``` After using these commands, the client can be started successfully. The successful status will be as follows: @@ -86,12 +86,12 @@ Following is a client command which connects the host with IP The Linux and MacOS system startup commands are as follows: ``` - Shell > ./sbin/start-client.sh -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10 + Shell > ./sbin/start-cli.sh -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10 ``` The Windows system startup commands are as follows: ``` - Shell > \sbin\start-client.bat -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10 + Shell > \sbin\start-cli.bat -h 10.129.187.21 -p 6667 -u root -pw root -disableIS08601 -maxPRC 10 ``` ## Cli/shell tool with -e parameter @@ -102,7 +102,7 @@ In order to avoid confusion between statements and other parameters, the current The usage of -e parameter for Cli/shell is as follows: ``` - Shell > ./sbin/start-client.sh -h {host} -p {port} -u {user} -pw {password} -e {sql for iotdb} + Shell > ./sbin/start-cli.sh -h {host} -p {port} -u {user} -pw {password} -e {sql for iotdb} ``` In order to better explain the use of -e parameter, take following as an example. @@ -117,12 +117,12 @@ port=6667 user=root pass=root -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "set storage group to root.demo" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "create timeseries root.demo.s1 WITH DATATYPE=INT32, ENCODING=RLE" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(1,10)" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(2,11)" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(3,12)" -./sbin/start-client.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "select s1 from root.demo" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "set storage group to root.demo" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "create timeseries root.demo.s1 WITH DATATYPE=INT32, ENCODING=RLE" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(1,10)" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(2,11)" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "insert into root.demo(timestamp,s1) values(3,12)" +./sbin/start-cli.sh -h ${host} -p ${port} -u ${user} -pw ${pass} -e "select s1 from root.demo" ``` The print results are shown in the figure, which are consistent with the client and jdbc operations. diff --git a/docs/Documentation/UserGuide/8-Tools-Grafana.md b/docs/Documentation/UserGuide/9-Tools-Grafana.md similarity index 100% rename from docs/Documentation/UserGuide/8-Tools-Grafana.md rename to docs/Documentation/UserGuide/9-Tools-Grafana.md diff --git a/docs/Documentation/UserGuide/8-Tools-Hadoop.md b/docs/Documentation/UserGuide/9-Tools-Hadoop.md similarity index 100% rename from docs/Documentation/UserGuide/8-Tools-Hadoop.md rename to docs/Documentation/UserGuide/9-Tools-Hadoop.md diff --git a/docs/Documentation/UserGuide/8-Tools-spark.md b/docs/Documentation/UserGuide/9-Tools-spark.md similarity index 100% rename from docs/Documentation/UserGuide/8-Tools-spark.md rename to docs/Documentation/UserGuide/9-Tools-spark.md diff --git a/example/jdbc/pom.xml b/example/jdbc/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..065731f471b988f4e3b717b36dc346536d9f2bc1 --- /dev/null +++ b/example/jdbc/pom.xml @@ -0,0 +1,43 @@ + + + + + iotdb-examples + org.apache.iotdb + 0.9.0-SNAPSHOT + + 4.0.0 + jdbc-example + jdbc-example + + UTF-8 + 1.7 + 1.7 + + + + org.apache.iotdb + iotdb-jdbc + 0.9.0-SNAPSHOT + + + diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/StatementDemo.java b/example/jdbc/src/main/java/org/apache/iotdb/JDBCExample.java similarity index 52% rename from jdbc/src/test/java/org/apache/iotdb/jdbc/demo/StatementDemo.java rename to example/jdbc/src/main/java/org/apache/iotdb/JDBCExample.java index a122e8a00fd8238e4d0b363884c0e099f1ea9048..f67caf03f500bb296e81447942288f763be13dd1 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/StatementDemo.java +++ b/example/jdbc/src/main/java/org/apache/iotdb/JDBCExample.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.jdbc.demo; +package org.apache.iotdb; import java.sql.Connection; import java.sql.DriverManager; @@ -25,31 +25,24 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; -public class StatementDemo { +public class JDBCExample { public static void main(String[] args) throws ClassNotFoundException, SQLException { Class.forName("org.apache.iotdb.jdbc.IoTDBDriver"); - Connection connection = null; - try { - connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); - statement.execute("SET STORAGE GROUP TO root.ln.wf01.wt01"); - statement.execute( - "CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN, ENCODING=PLAIN"); - statement.execute( - "CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE"); - statement - .execute("insert into root.ln.wf01.wt01(timestamp,status) values(1509465600000,true)"); - statement - .execute("insert into root.ln.wf01.wt01(timestamp,status) values(1509465660000,true)"); - statement - .execute("insert into root.ln.wf01.wt01(timestamp,status) values(1509465720000,false)"); - statement.execute( - "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465600000,25.957603)"); - statement.execute( - "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465660000,24.359503)"); - statement.execute( - "insert into root.ln.wf01.wt01(timestamp,temperature) values(1509465720000,20.092794)"); + try (Connection connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + statement.execute("SET STORAGE GROUP TO root.sg1"); + statement.execute("CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT64, ENCODING=RLE"); + statement.execute("CREATE TIMESERIES root.sg1.d1.s2 WITH DATATYPE=INT64, ENCODING=RLE"); + statement.execute("CREATE TIMESERIES root.sg1.d1.s3 WITH DATATYPE=INT64, ENCODING=RLE"); + + for (int i = 0; i < 10; i++) { + for (int j = 0 ; j < 10; j++) { + statement.addBatch("insert into root.sg1.d1(timestamp, s1, s2, s3) values("+ (i * 10 + j) + "," + 1 + "," + 1 + "," + 1 + ")"); + } + statement.executeBatch(); + statement.clearBatch(); + } ResultSet resultSet = statement.executeQuery("select * from root"); ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); while (resultSet.next()) { @@ -57,12 +50,7 @@ public class StatementDemo { for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { builder.append(resultSet.getString(i)).append(","); } - System.out.println(builder); } - statement.close(); - - } finally { - connection.close(); } } } diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/PrepareStatementDemo.java b/example/jdbc/src/main/java/org/apache/iotdb/PrepareStatementDemo.java similarity index 69% rename from jdbc/src/test/java/org/apache/iotdb/jdbc/demo/PrepareStatementDemo.java rename to example/jdbc/src/main/java/org/apache/iotdb/PrepareStatementDemo.java index 3f0e3a0d5137efaf71c8c5c9784db23e0481acf3..5ab8d24c2e7e316ddac1136ae5bab146008eb83c 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/PrepareStatementDemo.java +++ b/example/jdbc/src/main/java/org/apache/iotdb/PrepareStatementDemo.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.jdbc.demo; +package org.apache.iotdb; import java.sql.Connection; import java.sql.DriverManager; @@ -30,12 +30,12 @@ public class PrepareStatementDemo { public static void main(String[] args) throws ClassNotFoundException, SQLException { Class.forName("org.apache.iotdb.jdbc.IoTDBDriver"); - Connection connection = null; - try { - connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - PreparedStatement preparedStatement = connection - .prepareStatement( - "insert into root.ln.wf01.wt01(timestamp,status,temperature) values(?,?,?)"); + try (Connection connection = DriverManager + .getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + PreparedStatement preparedStatement = connection + .prepareStatement( + "insert into root.ln.wf01.wt01(timestamp,status,temperature) values(?,?,?)")) { + preparedStatement.setLong(1, 1509465600000L); preparedStatement.setBoolean(2, true); preparedStatement.setFloat(3, 25.957603f); @@ -60,24 +60,22 @@ public class PrepareStatementDemo { preparedStatement.execute(); preparedStatement.clearParameters(); - preparedStatement.close(); - ResultSet resultSet = preparedStatement.executeQuery("select * from root"); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - while (resultSet.next()) { - StringBuilder builder = new StringBuilder(); + + try(ResultSet resultSet = preparedStatement.executeQuery("select * from root")) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(resultSet.getString(i)).append(","); + } + System.out.println(builder); + } for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - builder.append(resultSet.getString(i)).append(","); + System.out + .println(resultSetMetaData.getColumnType(i) + "-" + resultSetMetaData.getColumnName(i)); } - System.out.println(builder); - } - preparedStatement.close(); - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - System.out - .println(resultSetMetaData.getColumnType(i) + "-" + resultSetMetaData.getColumnName(i)); } - } finally { - connection.close(); } } diff --git a/example/pom.xml b/example/pom.xml index 31e2d3f9bcba3d156f600be5aa26fa41ec10d104..4750be31d473f65e86db72dd6f166853553b2244 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -36,6 +36,7 @@ kafka rocketmq + session tsfile diff --git a/example/rocketmq/src/main/java/org/apache/iotdb/example/Constant.java b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Constant.java similarity index 99% rename from example/rocketmq/src/main/java/org/apache/iotdb/example/Constant.java rename to example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Constant.java index 1a290554a08b3ab4ac42c241568a9009183d4162..72b08e446f0f4bce539e5f833dbd817998d306c0 100644 --- a/example/rocketmq/src/main/java/org/apache/iotdb/example/Constant.java +++ b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Constant.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.example; +package org.apache.iotdb.rocketmq; import org.apache.iotdb.jdbc.Config; diff --git a/example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQConsumer.java b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQConsumer.java similarity index 99% rename from example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQConsumer.java rename to example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQConsumer.java index 2fbda076873f8ca399246639a5b8efa9bfda2d2c..c9bcf91ad258afa9ed5025f013b4444a8d40f38d 100644 --- a/example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQConsumer.java +++ b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQConsumer.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.example; +package org.apache.iotdb.rocketmq; import java.sql.Connection; import java.sql.DriverManager; diff --git a/example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQProducer.java b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQProducer.java similarity index 98% rename from example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQProducer.java rename to example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQProducer.java index 2e231d95b28d6dfe09c745d6db39dc22172d68bd..fc44e9511c144d22c188c429ae3cda40c964553c 100644 --- a/example/rocketmq/src/main/java/org/apache/iotdb/example/RocketMQProducer.java +++ b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQProducer.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.example; +package org.apache.iotdb.rocketmq; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; diff --git a/example/rocketmq/src/main/java/org/apache/iotdb/example/Utils.java b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Utils.java similarity index 97% rename from example/rocketmq/src/main/java/org/apache/iotdb/example/Utils.java rename to example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Utils.java index c0a9eb1c24e3a79a2a966ced3386ee1802760fdc..e6b0ad1af0e8710f2ff2eb943dc19372fa3e059b 100644 --- a/example/rocketmq/src/main/java/org/apache/iotdb/example/Utils.java +++ b/example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/Utils.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.example; +package org.apache.iotdb.rocketmq; /** * @author lta diff --git a/example/session/pom.xml b/example/session/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..8b1b2152f9b05393ca0440e4235fb782fc1c581c --- /dev/null +++ b/example/session/pom.xml @@ -0,0 +1,43 @@ + + + + + iotdb-examples + org.apache.iotdb + 0.9.0-SNAPSHOT + + 4.0.0 + client-example + client-example + + UTF-8 + 1.7 + 1.7 + + + + org.apache.iotdb + iotdb-session + 0.9.0-SNAPSHOT + + + diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java new file mode 100644 index 0000000000000000000000000000000000000000..0becd798271edc40af8c47ccfffa6969d390ed1e --- /dev/null +++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb; + +import org.apache.iotdb.session.IoTDBSessionException; +import org.apache.iotdb.session.Session; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.record.RowBatch; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; + +public class SessionExample { + + public static void main(String[] args) throws IoTDBSessionException { + Session session = new Session("127.0.0.1", 6667, "root", "root"); + session.open(); + + session.setStorageGroup("root.sg1"); + session.createTimeseries("root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE); + session.createTimeseries("root.sg1.d1.s2", TSDataType.INT64, TSEncoding.RLE); + session.createTimeseries("root.sg1.d1.s3", TSDataType.INT64, TSEncoding.RLE); + + Schema schema = new Schema(); + schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE)); + schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE)); + schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE)); + + RowBatch rowBatch = schema.createRowBatch("root.sg1.d1", 100); + + long[] timestamps = rowBatch.timestamps; + Object[] values = rowBatch.values; + + for (long time = 0; time < 30000; time++) { + int row = rowBatch.batchSize++; + timestamps[row] = time; + for (int i = 0; i < 3; i++) { + long[] sensor = (long[]) values[i]; + sensor[row] = time; + } + if (rowBatch.batchSize == rowBatch.getMaxBatchSize()) { + session.insertBatch(rowBatch); + rowBatch.reset(); + } + } + + if (rowBatch.batchSize != 0) { + session.insertBatch(rowBatch); + rowBatch.reset(); + } + session.close(); + } +} \ No newline at end of file diff --git a/example/tsfile/pom.xml b/example/tsfile/pom.xml index 1dab0e0400f9e8b10a0977829368173a6945841c..7a99f0134a5b2646024b6ac9f921c2b3043bbc15 100644 --- a/example/tsfile/pom.xml +++ b/example/tsfile/pom.xml @@ -19,32 +19,22 @@ under the License. --> - - 4.0.0 - - org.apache.iotdb - iotdb-examples + + 4.0.0 + + org.apache.iotdb + iotdb-examples + 0.9.0-SNAPSHOT + ../pom.xml + + tsfile-example 0.9.0-SNAPSHOT - ../pom.xml - - org.apache.iotdb - tsfile-example - 0.9.0-SNAPSHOT - tsfile-example - - http://www.example.com - - - junit - junit - 4.11 - test - - - org.apache.iotdb - tsfile - 0.9.0-SNAPSHOT - - + tsfile-example + + + org.apache.iotdb + tsfile + 0.9.0-SNAPSHOT + + diff --git a/example/tsfile/readme.md b/example/tsfile/readme.md index 9615cdd05aa8de58362d175960b8e35fba66fbad..000cb7c2d21a75d9c12109fe522d6ec6218f8dae 100644 --- a/example/tsfile/readme.md +++ b/example/tsfile/readme.md @@ -81,4 +81,4 @@ The example is to show how to write and read a TsFile File. This class is to show the structure of a TsFile. ### Notice -  For detail, please refer to https://github.com/thulab/tsfile/wiki/Get-Started. +  For detail, please refer to https://github.com/apache/incubator-iotdb/blob/master/tsfile/README.md. diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java index 416ef742dc9f819ff3725549e677779d98c0e646..4bac3731c78cb9f728f5c38c74c148f1bbc20734 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteWithRowBatch.java @@ -23,9 +23,10 @@ import java.io.File; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.write.TsFileWriter; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.apache.iotdb.tsfile.write.record.RowBatch; + /** * An example of writing data with RowBatch to TsFile */ @@ -39,7 +40,7 @@ public class TsFileWriteWithRowBatch { f.delete(); } - FileSchema fileSchema = new FileSchema(); + Schema schema = new Schema(); // the number of rows to include in the row batch int rowNum = 1000000; @@ -48,15 +49,15 @@ public class TsFileWriteWithRowBatch { // add measurements into file schema (all with INT64 data type) for (int i = 0; i < sensorNum; i++) { - fileSchema.registerMeasurement( + schema.registerMeasurement( new MeasurementSchema("sensor_" + (i + 1), TSDataType.INT64, TSEncoding.TS_2DIFF)); } // add measurements into TSFileWriter - TsFileWriter tsFileWriter = new TsFileWriter(f, fileSchema); + TsFileWriter tsFileWriter = new TsFileWriter(f, schema); // construct the row batch - RowBatch rowBatch = fileSchema.createRowBatch("device_1"); + RowBatch rowBatch = schema.createRowBatch("device_1"); long[] timestamps = rowBatch.timestamps; Object[] values = rowBatch.values; diff --git a/hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java b/hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java index 3f23cf6c1ea77abeeeea8e8ce33709336147b65b..bfc2f8912fb7182cd78c4e935f7675f381ca69a9 100644 --- a/hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java +++ b/hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java @@ -30,7 +30,7 @@ import org.apache.iotdb.tsfile.hadoop.io.HDFSOutputStream; import org.apache.iotdb.tsfile.timeseries.basis.TsFile; import org.apache.iotdb.tsfile.write.exception.InvalidJsonSchemaException; import org.apache.iotdb.tsfile.write.exception.WriteProcessException; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,9 +42,9 @@ public class TSFRecordWriter extends RecordWriter { public TSFRecordWriter(Path path, JSONObject schema) throws InterruptedException, IOException { // construct the internalrecordwriter - FileSchema fileSchema = null; + Schema fileSchema = null; try { - fileSchema = new FileSchema(schema); + fileSchema = new Schema(schema); } catch (InvalidJsonSchemaException e) { throw new InterruptedException(String.format("Construct the tsfile schema failed"), e); } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java index c366c0c984b32d33cafff6c1c59f60cdb9ade013..85270e1a267d29020e129e316b85847d323ea3af 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java @@ -18,11 +18,6 @@ */ package org.apache.iotdb.jdbc; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; -import java.net.SocketException; import java.sql.Array; import java.sql.Blob; import java.sql.CallableStatement; @@ -40,21 +35,12 @@ import java.sql.Savepoint; import java.sql.Statement; import java.sql.Struct; import java.time.ZoneId; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.Executor; -import org.apache.iotdb.service.rpc.thrift.ServerProperties; -import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq; -import org.apache.iotdb.service.rpc.thrift.TSGetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; -import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; -import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; +import org.apache.iotdb.rpc.IoTDBRPCException; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TCompactProtocol; @@ -65,8 +51,8 @@ import org.slf4j.LoggerFactory; public class IoTDBConnection implements Connection { - Logger logger = LoggerFactory.getLogger(IoTDBConnection.class); - private final List supportedProtocols = new LinkedList<>(); + private static final Logger logger = LoggerFactory.getLogger(IoTDBConnection.class); + private final TSProtocolVersion protocolVersion = TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1; public TSIService.Iface client = null; public TS_SessionHandle sessionHandle = null; private IoTDBConnectionParams params; @@ -87,8 +73,6 @@ public class IoTDBConnection implements Connection { } params = Utils.parseUrl(url, info); - supportedProtocols.add(TSProtocolVersion.TSFILE_SERVICE_PROTOCOL_V1); - openTransport(); if(Config.rpcThriftCompressionEnable) { client = new TSIService.Client(new TCompactProtocol(transport)); @@ -99,15 +83,10 @@ public class IoTDBConnection implements Connection { // open client session openSession(); // Wrap the client with a thread-safe proxy to serialize the RPC calls - client = newSynchronizedClient(client); + client = RpcUtils.newSynchronizedClient(client); autoCommit = false; } - public static TSIService.Iface newSynchronizedClient(TSIService.Iface client) { - return (TSIService.Iface) Proxy.newProxyInstance(IoTDBConnection.class.getClassLoader(), - new Class[]{TSIService.Iface.class}, new SynchronizedHandler(client)); - } - @Override public boolean isWrapperFor(Class arg0) throws SQLException { throw new SQLException("Method not supported"); @@ -416,18 +395,13 @@ public class IoTDBConnection implements Connection { private void openTransport() throws TTransportException { transport = new TSocket(params.getHost(), params.getPort(), Config.connectionTimeoutInMs); - try { - transport.getSocket().setKeepAlive(true); - } catch (SocketException e) { - logger.error("Cannot set socket keep alive because: ", e); - } if (!transport.isOpen()) { transport.open(); } } private void openSession() throws SQLException { - TSOpenSessionReq openReq = new TSOpenSessionReq(TSProtocolVersion.TSFILE_SERVICE_PROTOCOL_V1); + TSOpenSessionReq openReq = new TSOpenSessionReq(TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1); openReq.setUsername(params.getUsername()); openReq.setPassword(params.getPassword()); @@ -437,14 +411,16 @@ public class IoTDBConnection implements Connection { // validate connection try { - Utils.verifySuccess(openResp.getStatus()); - } catch (IoTDBSQLException e) { + RpcUtils.verifySuccess(openResp.getStatus()); + } catch (IoTDBRPCException e) { // failed to connect, disconnect from the server transport.close(); - throw e; + throw new IoTDBSQLException(e.getMessage()); } - if (!supportedProtocols.contains(openResp.getServerProtocolVersion())) { - throw new TException("Unsupported TsFile protocol"); + if (protocolVersion.getValue() != openResp.getServerProtocolVersion().getValue()) { + throw new TException(String + .format("Protocol not supported, Client version is {}, but Server version is {}", + protocolVersion.getValue(), openResp.getServerProtocolVersion().getValue())); } setProtocol(openResp.getServerProtocolVersion()); sessionHandle = openResp.getSessionHandle(); @@ -476,7 +452,7 @@ public class IoTDBConnection implements Connection { client = new TSIService.Client(new TBinaryProtocol(transport)); } openSession(); - client = newSynchronizedClient(client); + client = RpcUtils.newSynchronizedClient(client); flag = true; break; } @@ -497,14 +473,22 @@ public class IoTDBConnection implements Connection { } TSGetTimeZoneResp resp = client.getTimeZone(); - Utils.verifySuccess(resp.getStatus()); + try { + RpcUtils.verifySuccess(resp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } return resp.getTimeZone(); } public void setTimeZone(String zoneId) throws TException, IoTDBSQLException { TSSetTimeZoneReq req = new TSSetTimeZoneReq(zoneId); - TSSetTimeZoneResp resp = client.setTimeZone(req); - Utils.verifySuccess(resp.getStatus()); + TSRPCResp resp = client.setTimeZone(req); + try { + RpcUtils.verifySuccess(resp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } this.zoneId = ZoneId.of(zoneId); } @@ -520,32 +504,4 @@ public class IoTDBConnection implements Connection { this.protocol = protocol; } - private static class SynchronizedHandler implements InvocationHandler { - - private final TSIService.Iface client; - - SynchronizedHandler(TSIService.Iface client) { - this.client = client; - } - - @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - try { - synchronized (client) { - return method.invoke(client, args); - } - } catch (InvocationTargetException e) { - // all IFace APIs throw TException - if (e.getTargetException() instanceof TException) { - throw e.getTargetException(); - } else { - // should not happen - throw new TException("Error in calling method " + method.getName(), - e.getTargetException()); - } - } catch (Exception e) { - throw new TException("Error in calling method " + method.getName(), e); - } - } - } } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java index 21e1405aa8ed21157e3917d48da87c0da2ab9116..36d01c2c830a2c28afe8ec7e7c4d67a490561819 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java @@ -25,6 +25,8 @@ import java.sql.RowIdLifetime; import java.sql.SQLException; import java.util.List; import java.util.Set; +import org.apache.iotdb.rpc.IoTDBRPCException; +import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; import org.apache.iotdb.service.rpc.thrift.TSIService; @@ -82,7 +84,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { req.setColumnPath(schemaPattern); try { TSFetchMetadataResp resp = client.fetchMetadata(req); - Utils.verifySuccess(resp.getStatus()); + try { + RpcUtils.verifySuccess(resp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } return new IoTDBMetadataResultSet(resp.getColumnsList(), null, null); } catch (TException e) { throw new TException("Conncetion error when fetching column metadata", e); @@ -92,7 +98,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { req.setColumnPath(schemaPattern); try { TSFetchMetadataResp resp = client.fetchMetadata(req); - Utils.verifySuccess(resp.getStatus()); + try { + RpcUtils.verifySuccess(resp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } return new IoTDBMetadataResultSet(resp.getColumnsList(), null, null); } catch (TException e) { throw new TException("Conncetion error when fetching delta object metadata", e); @@ -101,7 +111,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { req = new TSFetchMetadataReq(Constant.GLOBAL_SHOW_STORAGE_GROUP_REQ); try { TSFetchMetadataResp resp = client.fetchMetadata(req); - Utils.verifySuccess(resp.getStatus()); + try { + RpcUtils.verifySuccess(resp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } Set showStorageGroup = resp.getShowStorageGroups(); return new IoTDBMetadataResultSet(null, showStorageGroup, null); } catch (TException e) { @@ -112,7 +126,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { req.setColumnPath(schemaPattern); try { TSFetchMetadataResp resp = client.fetchMetadata(req); - Utils.verifySuccess(resp.getStatus()); + try { + RpcUtils.verifySuccess(resp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } List> showTimeseriesList = resp.getShowTimeseriesList(); return new IoTDBMetadataResultSet(null, null, showTimeseriesList); } catch (TException e) { @@ -1247,7 +1265,11 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { TSFetchMetadataReq req = new TSFetchMetadataReq("METADATA_IN_JSON"); TSFetchMetadataResp resp; resp = client.fetchMetadata(req); - Utils.verifySuccess(resp.getStatus()); + try { + RpcUtils.verifySuccess(resp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } return resp.getMetadataInJson(); } } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java index fbdb3ab5ad25a32179598b9bdf2bbbd55d52dc5e..2fe037073d9bcbac404ec8d01a5d95e460c3f90e 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java @@ -22,11 +22,12 @@ package org.apache.iotdb.jdbc; import java.sql.SQLException; import java.time.ZoneId; import java.util.List; + +import org.apache.iotdb.rpc.TSStatusType; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSIService.Iface; import org.apache.iotdb.service.rpc.thrift.TSInsertionReq; import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.apache.thrift.TException; public class IoTDBPreparedInsertionStatement extends IoTDBPreparedStatement { @@ -44,12 +45,12 @@ public class IoTDBPreparedInsertionStatement extends IoTDBPreparedStatement { public boolean execute() throws SQLException { try { - TSExecuteStatementResp resp = client.executeInsertion(req); + TSExecuteStatementResp resp = client.insert(req); req.unsetDeviceId(); req.unsetMeasurements(); req.unsetTimestamp(); req.unsetValues(); - return resp.getStatus().getStatusCode() == TS_StatusCode.SUCCESS_STATUS; + return resp.getStatus().getStatusType().getCode() == TSStatusType.SUCCESS_STATUS.getStatusCode(); } catch (TException e) { throw new SQLException(e); } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java index 3052dfc444703ea36638ce022e9b213a7603f2f5..52e05817c187580a1ed371e82f8735ac9cab68df 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java @@ -48,6 +48,7 @@ import java.util.List; import java.util.Map; import org.apache.iotdb.service.rpc.thrift.TSIService.Iface; import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; +import org.apache.thrift.TException; public class IoTDBPreparedStatement extends IoTDBStatement implements PreparedStatement { diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java index 632e1b11071ae34d8ba3275925e371676f85c2ef..012c7666f7f3e994732b5f1765cea7dccfa7b4bc 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java @@ -46,13 +46,9 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; +import org.apache.iotdb.rpc.IoTDBRPCException; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.tsfile.read.common.Field; import org.apache.iotdb.tsfile.read.common.RowRecord; import org.apache.thrift.TException; @@ -213,10 +209,10 @@ public class IoTDBQueryResultSet implements ResultSet { try { if (operationHandle != null) { TSCloseOperationReq closeReq = new TSCloseOperationReq(operationHandle, queryId); - TSCloseOperationResp closeResp = client.closeOperation(closeReq); - Utils.verifySuccess(closeResp.getStatus()); + TSRPCResp closeResp = client.closeOperation(closeReq); + RpcUtils.verifySuccess(closeResp.getStatus()); } - } catch (SQLException e) { + } catch (IoTDBRPCException e) { throw new SQLException("Error occurs for close opeation in server side becasuse " + e); } catch (TException e) { throw new SQLException( @@ -708,7 +704,11 @@ public class IoTDBQueryResultSet implements ResultSet { try { TSFetchResultsResp resp = client.fetchResults(req); - Utils.verifySuccess(resp.getStatus()); + try { + RpcUtils.verifySuccess(resp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } if (!resp.hasResultSet) { emptyResultSet = true; } else { diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBSQLException.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBSQLException.java index c265fe0d39510367e5985cdfbc727f3f2f89d963..8a1a916d2b4db3073881c4fef3da0a8df30fc357 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBSQLException.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBSQLException.java @@ -29,4 +29,8 @@ public class IoTDBSQLException extends SQLException { super(reason); } + public IoTDBSQLException(Throwable cause) { + super(cause); + } + } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java index 704bc5ff9f9fc6d329a72f9c2335fde232f66806..b728831f454eee65ab6ccd463691f4b6c345a82d 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java @@ -30,18 +30,11 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; -import org.apache.iotdb.service.rpc.thrift.TSCancelOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCancelOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.IoTDBRPCException; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.thrift.TException; public class IoTDBStatement implements Statement { @@ -49,6 +42,7 @@ public class IoTDBStatement implements Statement { private static final String SHOW_TIMESERIES_COMMAND_LOWERCASE = "show timeseries"; private static final String SHOW_STORAGE_GROUP_COMMAND_LOWERCASE = "show storage group"; private static final String METHOD_NOT_SUPPORTED_STRING = "Method not supported"; + ZoneId zoneId; private ResultSet resultSet = null; private IoTDBConnection connection; @@ -130,8 +124,8 @@ public class IoTDBStatement implements Statement { try { if (operationHandle != null) { TSCancelOperationReq closeReq = new TSCancelOperationReq(operationHandle); - TSCancelOperationResp closeResp = client.cancelOperation(closeReq); - Utils.verifySuccess(closeResp.getStatus()); + TSRPCResp closeResp = client.cancelOperation(closeReq); + RpcUtils.verifySuccess(closeResp.getStatus()); } } catch (Exception e) { throw new SQLException("Error occurs when canceling statement.", e); @@ -157,8 +151,8 @@ public class IoTDBStatement implements Statement { if (operationHandle != null) { TSCloseOperationReq closeReq = new TSCloseOperationReq(operationHandle, -1); closeReq.setStmtId(stmtId); - TSCloseOperationResp closeResp = client.closeOperation(closeReq); - Utils.verifySuccess(closeResp.getStatus()); + TSRPCResp closeResp = client.closeOperation(closeReq); + RpcUtils.verifySuccess(closeResp.getStatus()); } } catch (Exception e) { throw new SQLException("Error occurs when closing statement.", e); @@ -246,7 +240,11 @@ public class IoTDBStatement implements Statement { TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionHandle, sql); TSExecuteStatementResp execResp = client.executeStatement(execReq); operationHandle = execResp.getOperationHandle(); - Utils.verifySuccess(execResp.getStatus()); + try { + RpcUtils.verifySuccess(execResp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } if (execResp.getOperationHandle().hasResultSet) { IoTDBQueryResultSet resSet = new IoTDBQueryResultSet(this, execResp.getColumns(), client, @@ -288,7 +286,7 @@ public class IoTDBStatement implements Statement { TSExecuteBatchStatementReq execReq = new TSExecuteBatchStatementReq(sessionHandle, batchSQLList); TSExecuteBatchStatementResp execResp = client.executeBatchStatement(execReq); - if (execResp.getStatus().statusCode == TS_StatusCode.SUCCESS_STATUS) { + if (execResp.getStatus().getStatusType().getCode() == TSStatusType.SUCCESS_STATUS.getStatusCode()) { if (execResp.getResult() == null) { return new int[0]; } else { @@ -303,7 +301,7 @@ public class IoTDBStatement implements Statement { } else { BatchUpdateException exception; if (execResp.getResult() == null) { - exception = new BatchUpdateException(execResp.getStatus().errorMessage, new int[0]); + exception = new BatchUpdateException(execResp.getStatus().getStatusType().getMessage(), new int[0]); } else { List result = execResp.getResult(); int len = result.size(); @@ -311,7 +309,7 @@ public class IoTDBStatement implements Statement { for (int i = 0; i < len; i++) { updateArray[i] = result.get(i); } - exception = new BatchUpdateException(execResp.getStatus().errorMessage, updateArray); + exception = new BatchUpdateException(execResp.getStatus().getStatusType().getMessage(), updateArray); } throw exception; } @@ -346,7 +344,11 @@ public class IoTDBStatement implements Statement { TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionHandle, sql); TSExecuteStatementResp execResp = client.executeQueryStatement(execReq); operationHandle = execResp.getOperationHandle(); - Utils.verifySuccess(execResp.getStatus()); + try { + RpcUtils.verifySuccess(execResp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } IoTDBQueryResultSet resSet = new IoTDBQueryResultSet(this, execResp.getColumns(), client, operationHandle, sql, execResp.getOperationType(), execResp.getDataTypeList(), queryId.getAndIncrement()); @@ -399,7 +401,11 @@ public class IoTDBStatement implements Statement { TSExecuteStatementReq execReq = new TSExecuteStatementReq(sessionHandle, sql); TSExecuteStatementResp execResp = client.executeUpdateStatement(execReq); operationHandle = execResp.getOperationHandle(); - Utils.verifySuccess(execResp.getStatus()); + try { + RpcUtils.verifySuccess(execResp.getStatus()); + } catch (IoTDBRPCException e) { + throw new IoTDBSQLException(e.getMessage()); + } return 0; } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java index f00402c71ce6496ba14edb2ad1262bba03d8b927..9ca73b268c05ea1050c64d46869b2f9a4f31ed94 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java @@ -26,8 +26,6 @@ import java.util.regex.Pattern; import org.apache.iotdb.service.rpc.thrift.TSDataValue; import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; import org.apache.iotdb.service.rpc.thrift.TSRowRecord; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Field; @@ -39,18 +37,11 @@ import org.apache.iotdb.tsfile.utils.Binary; */ public class Utils { - /** - * Private constructor of Utils Class. - */ - private Utils(){ - throw new IllegalAccessError("Utility class"); - } - /** * Parse JDBC connection URL The only supported format of the URL is: * jdbc:iotdb://localhost:6667/. */ - public static IoTDBConnectionParams parseUrl(String url, Properties info) + static IoTDBConnectionParams parseUrl(String url, Properties info) throws IoTDBURLException { IoTDBConnectionParams params = new IoTDBConnectionParams(url); if (url.trim().equalsIgnoreCase(Config.IOTDB_URL_PREFIX)) { @@ -79,24 +70,13 @@ public class Utils { return params; } - /** - * verify success. - * - * @param status -status - */ - public static void verifySuccess(TS_Status status) throws IoTDBSQLException { - if (status.getStatusCode() != TS_StatusCode.SUCCESS_STATUS) { - throw new IoTDBSQLException(status.errorMessage); - } - } - /** * convert row records. * * @param tsQueryDataSet -query data set * @return -list of row record */ - public static List convertRowRecords(TSQueryDataSet tsQueryDataSet) { + static List convertRowRecords(TSQueryDataSet tsQueryDataSet) { List records = new ArrayList<>(); for (TSRowRecord ts : tsQueryDataSet.getRecords()) { RowRecord r = new RowRecord(ts.getTimestamp()); @@ -151,4 +131,5 @@ public class Utils { dataType)); } } + } diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java index 912019d3c07d601c579457a990b54f2df52c8d39..09505f44d1348ddbe3046b4fc17b3fc5de2717fb 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java @@ -29,12 +29,9 @@ import java.sql.Statement; import java.time.ZoneId; import java.util.ArrayList; import java.util.List; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.thrift.TException; import org.junit.After; import org.junit.Before; @@ -50,8 +47,10 @@ public class BatchTest { private TSIService.Iface client; @Mock private TS_SessionHandle sessHandle; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); - private TS_Status Status_ERROR = new TS_Status(TS_StatusCode.ERROR_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_StatusType errorStatus = new TS_StatusType(TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); + private TS_Status Status_ERROR = new TS_Status(errorStatus); private TSExecuteBatchStatementResp resp; private ZoneId zoneID = ZoneId.systemDefault(); diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java index e374064ef2126c22c771bd002419ccbd83d6f0c1..5012e5ed05088777116d5cc612f4e68085d81042 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java @@ -24,13 +24,9 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.List; -import org.apache.iotdb.service.rpc.thrift.ServerProperties; -import org.apache.iotdb.service.rpc.thrift.TSGetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.thrift.TException; import org.junit.After; import org.junit.Before; @@ -44,7 +40,8 @@ public class IoTDBConnectionTest { private TSIService.Iface client; private IoTDBConnection connection = new IoTDBConnection(); - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); @Before public void setUp() throws Exception { @@ -59,7 +56,7 @@ public class IoTDBConnectionTest { public void testSetTimeZone() throws TException, IoTDBSQLException { String timeZone = "Asia/Shanghai"; when(client.setTimeZone(any(TSSetTimeZoneReq.class))) - .thenReturn(new TSSetTimeZoneResp(Status_SUCCESS)); + .thenReturn(new TSRPCResp(Status_SUCCESS)); connection.client = client; connection.setTimeZone(timeZone); assertEquals(connection.getTimeZone(), timeZone); diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java index 1f5ac2597bf190bfa2b15dcef189d0bbeb87d031..2fdb40667ae688d2f61ba73e418edfcd3eb3ffb7 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java @@ -30,11 +30,9 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -66,7 +64,8 @@ public class IoTDBDatabaseMetadataTest { @Mock private TSFetchMetadataResp fetchMetadataResp; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); private DatabaseMetaData databaseMetaData; @@ -96,8 +95,7 @@ public class IoTDBDatabaseMetadataTest { String standard = "Column,\n" + "root.vehicle.d0.s0,\n" + "root.vehicle.d0.s1,\n" + "root.vehicle.d0.s2,\n"; - try { - ResultSet resultSet = databaseMetaData.getColumns(Constant.CATALOG_COLUMN, "root", null, null); + try (ResultSet resultSet = databaseMetaData.getColumns(Constant.CATALOG_COLUMN, "root", null, null)) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); int colCount = resultSetMetaData.getColumnCount(); StringBuilder resultStr = new StringBuilder(); @@ -129,9 +127,8 @@ public class IoTDBDatabaseMetadataTest { when(fetchMetadataResp.getColumnsList()).thenReturn(columnList); String standard = "Column,\n" + "root.vehicle.d0,\n"; - try { - ResultSet resultSet = databaseMetaData - .getColumns(Constant.CATALOG_DEVICE, "vehicle", null, null); + try (ResultSet resultSet = databaseMetaData + .getColumns(Constant.CATALOG_DEVICE, "vehicle", null, null)) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); int colCount = resultSetMetaData.getColumnCount(); StringBuilder resultStr = new StringBuilder(); @@ -189,9 +186,8 @@ public class IoTDBDatabaseMetadataTest { + "root.vehicle.d0.s0,root.vehicle,INT32,RLE,\n" + "root.vehicle.d0.s1,root.vehicle,INT64,RLE,\n" + "root.vehicle.d0.s2,root.vehicle,FLOAT,RLE,\n"; - try { - ResultSet resultSet = databaseMetaData - .getColumns(Constant.CATALOG_TIMESERIES, "root", null, null); + try (ResultSet resultSet = databaseMetaData + .getColumns(Constant.CATALOG_TIMESERIES, "root", null, null);) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); int colCount = resultSetMetaData.getColumnCount(); StringBuilder resultStr = new StringBuilder(); @@ -230,10 +226,9 @@ public class IoTDBDatabaseMetadataTest { when(fetchMetadataResp.getShowTimeseriesList()).thenReturn(tslist); String standard = "DataType,\n" + "INT32,\n"; - try { - ResultSet resultSet = databaseMetaData - .getColumns(Constant.CATALOG_TIMESERIES, "root.vehicle.d0.s0", null, - null); + try (ResultSet resultSet = databaseMetaData + .getColumns(Constant.CATALOG_TIMESERIES, "root.vehicle.d0.s0", null, + null)) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); StringBuilder resultStr = new StringBuilder(); resultStr.append(resultSetMetaData.getColumnName(3)).append(",\n"); @@ -259,9 +254,8 @@ public class IoTDBDatabaseMetadataTest { when(fetchMetadataResp.getShowStorageGroups()).thenReturn(sgSet); String standard = "Storage Group,\n" + "root.vehicle,\n"; - try { - ResultSet resultSet = databaseMetaData - .getColumns(Constant.CATALOG_STORAGE_GROUP, null, null, null); + try (ResultSet resultSet = databaseMetaData + .getColumns(Constant.CATALOG_STORAGE_GROUP, null, null, null)) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); int colCount = resultSetMetaData.getColumnCount(); StringBuilder resultStr = new StringBuilder(); diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java index ca9928efc738d45f2d04dd7b2c38df4ac375737e..f8c2a946ab3cf0481912dbf218c06a1632eb4297 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java @@ -26,14 +26,10 @@ import static org.mockito.Mockito.when; import java.sql.SQLException; import java.sql.Timestamp; import java.time.ZoneId; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSGetOperationStatusResp; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.service.rpc.thrift.TSIService.Iface; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -45,7 +41,7 @@ public class IoTDBPreparedStatementTest { @Mock TSExecuteStatementResp execStatementResp; @Mock - TSGetOperationStatusResp getOperationStatusResp; + TSRPCResp getOperationStatusResp; private ZoneId zoneId = ZoneId.systemDefault(); @Mock private IoTDBConnection connection; @@ -53,7 +49,8 @@ public class IoTDBPreparedStatementTest { private Iface client; @Mock private TS_SessionHandle sessHandle; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); @Mock private TSOperationHandle tOperationHandle; diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java index 353cea5bcbe7a636f2c0dd433c09ff74172352a4..f0c13f692046280e199da2cb54df9a3064ed547e 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java @@ -26,25 +26,16 @@ import static org.mockito.Mockito.when; import java.sql.ResultSet; import java.sql.ResultSetMetaData; +import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; import java.time.ZoneId; import java.util.ArrayList; import java.util.List; -import org.apache.iotdb.service.rpc.thrift.TSDataValue; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; -import org.apache.iotdb.service.rpc.thrift.TSRowRecord; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; + import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.junit.Assert; import org.junit.Before; @@ -117,7 +108,8 @@ public class IoTDBQueryResultSetTest { @Mock private TSFetchResultsResp fetchResultsResp; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); private ZoneId zoneID = ZoneId.systemDefault(); @Before @@ -137,6 +129,10 @@ public class IoTDBQueryResultSetTest { when(client.fetchResults(any(TSFetchResultsReq.class))).thenReturn(fetchResultsResp); when(fetchResultsResp.getStatus()).thenReturn(Status_SUCCESS); + + TSRPCResp closeResp = new TSRPCResp(); + closeResp.setStatus(Status_SUCCESS); + when(client.closeOperation(any(TSCloseOperationReq.class))).thenReturn(closeResp); } @SuppressWarnings("resource") @@ -181,50 +177,51 @@ public class IoTDBQueryResultSetTest { when(fetchResultsResp.getQueryDataSet()).thenReturn(tsQueryDataSet); if (hasResultSet) { - ResultSet resultSet = statement.getResultSet(); - // check columnInfoMap - Assert.assertEquals(resultSet.findColumn("Time"), 1); - Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s2"), 2); - Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s1"), 3); - Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s0"), 4); + try (ResultSet resultSet = statement.getResultSet()) { + // check columnInfoMap + Assert.assertEquals(resultSet.findColumn("Time"), 1); + Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s2"), 2); + Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s1"), 3); + Assert.assertEquals(resultSet.findColumn("root.vehicle.d0.s0"), 4); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - // check columnInfoList - Assert.assertEquals(resultSetMetaData.getColumnName(1), "Time"); - Assert.assertEquals(resultSetMetaData.getColumnName(2), "root.vehicle.d0.s2"); - Assert.assertEquals(resultSetMetaData.getColumnName(3), "root.vehicle.d0.s1"); - Assert.assertEquals(resultSetMetaData.getColumnName(4), "root.vehicle.d0.s0"); - Assert.assertEquals(resultSetMetaData.getColumnName(5), "root.vehicle.d0.s2"); - // check columnTypeList - Assert.assertEquals(resultSetMetaData.getColumnType(1), Types.TIMESTAMP); - Assert.assertEquals(resultSetMetaData.getColumnType(2), Types.FLOAT); - Assert.assertEquals(resultSetMetaData.getColumnType(3), Types.BIGINT); - Assert.assertEquals(resultSetMetaData.getColumnType(4), Types.INTEGER); - Assert.assertEquals(resultSetMetaData.getColumnType(5), Types.FLOAT); - // check fetched result - int colCount = resultSetMetaData.getColumnCount(); - StringBuilder resultStr = new StringBuilder(); - for (int i = 1; i < colCount + 1; i++) { // meta title - resultStr.append(resultSetMetaData.getColumnName(i)).append(","); - } - resultStr.append("\n"); - while (resultSet.next()) { // data - for (int i = 1; i <= colCount; i++) { - resultStr.append(resultSet.getString(i)).append(","); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + // check columnInfoList + Assert.assertEquals(resultSetMetaData.getColumnName(1), "Time"); + Assert.assertEquals(resultSetMetaData.getColumnName(2), "root.vehicle.d0.s2"); + Assert.assertEquals(resultSetMetaData.getColumnName(3), "root.vehicle.d0.s1"); + Assert.assertEquals(resultSetMetaData.getColumnName(4), "root.vehicle.d0.s0"); + Assert.assertEquals(resultSetMetaData.getColumnName(5), "root.vehicle.d0.s2"); + // check columnTypeList + Assert.assertEquals(resultSetMetaData.getColumnType(1), Types.TIMESTAMP); + Assert.assertEquals(resultSetMetaData.getColumnType(2), Types.FLOAT); + Assert.assertEquals(resultSetMetaData.getColumnType(3), Types.BIGINT); + Assert.assertEquals(resultSetMetaData.getColumnType(4), Types.INTEGER); + Assert.assertEquals(resultSetMetaData.getColumnType(5), Types.FLOAT); + // check fetched result + int colCount = resultSetMetaData.getColumnCount(); + StringBuilder resultStr = new StringBuilder(); + for (int i = 1; i < colCount + 1; i++) { // meta title + resultStr.append(resultSetMetaData.getColumnName(i)).append(","); } resultStr.append("\n"); + while (resultSet.next()) { // data + for (int i = 1; i <= colCount; i++) { + resultStr.append(resultSet.getString(i)).append(","); + } + resultStr.append("\n"); - fetchResultsResp.hasResultSet = false; // at the second time to fetch + fetchResultsResp.hasResultSet = false; // at the second time to fetch + } + String standard = + "Time,root.vehicle.d0.s2,root.vehicle.d0.s1,root.vehicle.d0.s0,root.vehicle.d0.s2,\n" + + "2,2.22,40000,null,2.22,\n" + "3,3.33,null,null,3.33,\n" + + "4,4.44,null,null,4.44,\n" + + "50,null,50000,null,null,\n" + "100,null,199,null,null,\n" + + "101,null,199,null,null,\n" + + "103,null,199,null,null,\n" + "105,11.11,199,33333,11.11,\n" + + "1000,1000.11,55555,22222,1000.11,\n"; + Assert.assertEquals(resultStr.toString(), standard); } - String standard = - "Time,root.vehicle.d0.s2,root.vehicle.d0.s1,root.vehicle.d0.s0,root.vehicle.d0.s2,\n" - + "2,2.22,40000,null,2.22,\n" + "3,3.33,null,null,3.33,\n" - + "4,4.44,null,null,4.44,\n" - + "50,null,50000,null,null,\n" + "100,null,199,null,null,\n" - + "101,null,199,null,null,\n" - + "103,null,199,null,null,\n" + "105,11.11,199,33333,11.11,\n" - + "1000,1000.11,55555,22222,1000.11,\n"; - Assert.assertEquals(resultStr.toString(), standard); } } @@ -276,14 +273,12 @@ public class IoTDBQueryResultSetTest { } else { if (i == 0) { value.setFloat_val((float) item[3 * i + 3]); - value.setType(item[3 * i + 2].toString()); } else if (i == 1) { value.setLong_val((long) item[3 * i + 3]); - value.setType(item[3 * i + 2].toString()); } else { value.setInt_val((int) item[3 * i + 3]); - value.setType(item[3 * i + 2].toString()); } + value.setType(item[3 * i + 2].toString()); } values.add(value); } diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java index d8632fc92590b6100029360d45af16847151851c..44fd2f4d224680c05068fd394b9114e9f14b01e4 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.jdbc; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.when; @@ -30,12 +31,10 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.service.rpc.thrift.TSIService.Iface; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.apache.thrift.TException; import org.junit.After; import org.junit.Assert; @@ -58,7 +57,8 @@ public class IoTDBStatementTest { @Mock private TSFetchMetadataResp fetchMetadataResp; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); private ZoneId zoneID = ZoneId.systemDefault(); @Before @@ -116,9 +116,8 @@ public class IoTDBStatementTest { + "root.vehicle.d0.s2,root.vehicle,FLOAT,RLE,\n"; when(fetchMetadataResp.getShowTimeseriesList()).thenReturn(tslist); boolean res = stmt.execute("show timeseries root"); - assertEquals(res, true); - try { - ResultSet resultSet = stmt.getResultSet(); + assertTrue(res); + try (ResultSet resultSet = stmt.getResultSet()) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); int colCount = resultSetMetaData.getColumnCount(); StringBuilder resultStr = new StringBuilder(); @@ -147,9 +146,8 @@ public class IoTDBStatementTest { when(fetchMetadataResp.getShowStorageGroups()).thenReturn(sgSet); String standard = "Storage Group,\nroot.vehicle,\n"; boolean res = stmt.execute("show storage group"); - assertEquals(res, true); - try { - ResultSet resultSet = stmt.getResultSet(); + assertTrue(res); + try (ResultSet resultSet = stmt.getResultSet()) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); int colCount = resultSetMetaData.getColumnCount(); StringBuilder resultStr = new StringBuilder(); diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java index ff6650dce7d328cd97376da20291c374f253f5eb..bc3a766823363e5554b4e70c4175e17ac044baaa 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java @@ -26,11 +26,15 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Properties; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusType; import org.apache.iotdb.service.rpc.thrift.TSDataValue; import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; import org.apache.iotdb.service.rpc.thrift.TSRowRecord; import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Field; import org.apache.iotdb.tsfile.read.common.RowRecord; @@ -69,13 +73,15 @@ public class UtilsTest { @Test public void testVerifySuccess() { try { - Utils.verifySuccess(new TS_Status(TS_StatusCode.SUCCESS_STATUS)); + TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + RpcUtils.verifySuccess(new TS_Status(successStatus)); } catch (Exception e) { fail(); } try { - Utils.verifySuccess(new TS_Status(TS_StatusCode.ERROR_STATUS)); + TS_StatusType errorStatus = new TS_StatusType(TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(), ""); + RpcUtils.verifySuccess(new TS_Status(errorStatus)); } catch (Exception e) { return; } @@ -114,23 +120,18 @@ public class UtilsTest { } else { if (i == 0) { value.setBool_val((boolean) item[3 * i + 3]); - value.setType(((TSDataType) item[3 * i + 2]).toString()); } else if (i == 1) { value.setInt_val((int) item[3 * i + 3]); - value.setType(((TSDataType) item[3 * i + 2]).toString()); } else if (i == 2) { value.setLong_val((long) item[3 * i + 3]); - value.setType(((TSDataType) item[3 * i + 2]).toString()); } else if (i == 3) { value.setFloat_val((float) item[3 * i + 3]); - value.setType(((TSDataType) item[3 * i + 2]).toString()); } else if (i == 4) { value.setDouble_val((double) item[3 * i + 3]); - value.setType(((TSDataType) item[3 * i + 2]).toString()); } else { value.setBinary_val(ByteBuffer.wrap(((String) item[3 * i + 3]).getBytes())); - value.setType(((TSDataType) item[3 * i + 2]).toString()); } + value.setType(item[3 * i + 2].toString()); } values.add(value); } diff --git a/pom.xml b/pom.xml index 0073930ee1e04c56d0cdcde3a5b75a836146d7a8..77a2219bdc13a27f075ab62b7d94d5a76b407790 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,19 @@ https://github.com/apache/incubator-iotdb rel/0.8 + + tsfile + service-rpc + jdbc + session + client + server + example + grafana + spark-tsfile + + distribution + 1.8 @@ -377,18 +390,6 @@ test - - tsfile - service-rpc - jdbc - client - server - example - grafana - spark-tsfile - - distribution - diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index e8705739e77d2febb5fffd90f0b8a4539582fc95..dddf1a2dc47d859991fc31d3d75a2e5b8886afc1 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -27,6 +27,8 @@ rpc_port=6667 rpc_thrift_compression_enable=false +rpc_max_concurrent_client_num=65535 + #################### ### Dynamic Parameter Adapter Configuration #################### @@ -157,7 +159,52 @@ concurrent_flush_thread=0 # whether take over the memory management by IoTDB rather than JVM when serializing memtable as bytes in memory # (i.e., whether use ChunkBufferPool), value true, false -chunk_buffer_pool_enable = false +chunk_buffer_pool_enable=false + +#################### +### Merge Configurations +#################### + +# How many thread will be set up to perform merge main tasks, 1 by default. +# Set to 1 when less than or equal to 0. +merge_thread_num=1 + +# How many thread will be set up to perform merge chunk sub-tasks, 4 by default. +# Set to 1 when less than or equal to 0. +merge_chunk_subthread_num=4 + +# If one merge file selection runs for more than this time, it will be ended and its current +# selection will be used as final selection. Unit: millis. +# When < 0, it means time is unbounded. +merge_fileSelection_time_budget=30000 + +# How much memory may be used in ONE merge task (in byte), 20% of maximum JVM memory by default. +# This is only a rough estimation, starting from a relatively small value to avoid OOM. +# Each new merge thread may take such memory, so merge_thread_num * merge_memory_budget is the +# total memory estimation of merge. +# merge_memory_budget=2147483648 + +# When set to true, if some crashed merges are detected during system rebooting, such merges will +# be continued, otherwise, the unfinished parts of such merges will not be continued while the +# finished parts still remains as they are. +# If you are feeling the rebooting is too slow, set this to false, false by default +continue_merge_after_reboot=false + +# A global merge will be performed each such interval, that is, each storage group will be merged +# (if proper merge candidates can be found). Unit: second, default: 1hours. +# When less than or equal to 0, timed merge is disabled. +merge_interval_sec=3600 + +# When set to true, all merges becomes full merge (the whole SeqFiles are re-written despite how +# much they are overflowed). This may increase merge overhead depending on how much the SeqFiles +# are overflowed. +force_full_merge=false + +# During a merge, if a chunk with less number of points than this parameter, the chunk will be +# merged with its succeeding chunks even if it is not overflowed, until the merged chunks reach +# this threshold and the new chunk will be flushed. +# When less than 0, this mechanism is disabled. +chunk_merge_point_threshold=20480 #################### ### Metadata Cache Configuration diff --git a/server/src/assembly/resources/tools/print-tsfile-resource-files.bat b/server/src/assembly/resources/tools/print-tsfile-resource-files.bat new file mode 100644 index 0000000000000000000000000000000000000000..bbc1acaf13b5c48378e4c1e99d8e02ad08e27599 --- /dev/null +++ b/server/src/assembly/resources/tools/print-tsfile-resource-files.bat @@ -0,0 +1,102 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + +@echo off +echo ```````````````````````` +echo Starting Printing the TsFileResources +echo ```````````````````````` + +PATH %PATH%;%JAVA_HOME%\bin\ +set "FULL_VERSION=" +set "MAJOR_VERSION=" +set "MINOR_VERSION=" + + +for /f tokens^=2-5^ delims^=.-_+^" %%j in ('java -fullversion 2^>^&1') do ( + set "FULL_VERSION=%%j-%%k-%%l-%%m" + IF "%%j" == "1" ( + set "MAJOR_VERSION=%%k" + set "MINOR_VERSION=%%l" + ) else ( + set "MAJOR_VERSION=%%j" + set "MINOR_VERSION=%%k" + ) +) + +set JAVA_VERSION=%MAJOR_VERSION% + +IF NOT %JAVA_VERSION% == 8 ( + IF NOT %JAVA_VERSION% == 11 ( + echo IoTDB only supports jdk8 or jdk11, please check your java version. + goto finally + ) +) + +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0.. +if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%cd% +popd + +set IOTDB_CONF=%IOTDB_HOME%\conf +set IOTDB_LOGS=%IOTDB_HOME%\logs + +if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.TsFileResourcePrinter +if NOT DEFINED JAVA_HOME goto :err + +@REM ----------------------------------------------------------------------------- +@REM JVM Opts we'll use in legacy run or installation +set JAVA_OPTS=-ea^ + -Dlogback.configurationFile="%IOTDB_CONF%\logback.xml"^ + -DIOTDB_HOME="%IOTDB_HOME%"^ + -DTSFILE_HOME="%IOTDB_HOME%"^ + -DIOTDB_CONF="%IOTDB_CONF%" + +@REM ***** CLASSPATH library setting ***** +@REM Ensure that any user defined CLASSPATH variables are not used on startup +set CLASSPATH="%IOTDB_HOME%\lib" + +REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable. +for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i" +set CLASSPATH=%CLASSPATH% +goto okClasspath + +:append +set CLASSPATH=%CLASSPATH%;%1 +goto :eof + +REM ----------------------------------------------------------------------------- +:okClasspath + +rem echo CLASSPATH: %CLASSPATH% +set IOTDB_DATA=%IOTDB_HOME%\data + +goto finally + +:err +echo JAVA_HOME environment variable must be set! +pause + + +@REM ----------------------------------------------------------------------------- +:finally + +pause + +ENDLOCAL diff --git a/server/src/assembly/resources/tools/print-tsfile-resource-files.sh b/server/src/assembly/resources/tools/print-tsfile-resource-files.sh new file mode 100644 index 0000000000000000000000000000000000000000..7de56338866144af96dbec10601ada9e26c92b3d --- /dev/null +++ b/server/src/assembly/resources/tools/print-tsfile-resource-files.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +echo --------------------- +echo Starting Printing the TsFileResources +echo --------------------- + +if [ -z "${IOTDB_HOME}" ]; then + export IOTDB_HOME="`dirname "$0"`/.." +fi + +IOTDB_CONF=${IOTDB_HOME}/conf + + +if [ -n "$JAVA_HOME" ]; then + for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do + if [ -x "$java" ]; then + JAVA="$java" + break + fi + done +else + JAVA=java +fi + +if [ -z $JAVA ] ; then + echo Unable to find java executable. Check JAVA_HOME and PATH environment variables. > /dev/stderr + exit 1; +fi + +CLASSPATH="" +for f in ${IOTDB_HOME}/lib/*.jar; do + CLASSPATH=${CLASSPATH}":"$f +done +classname=org.apache.iotdb.db.tools.TsFileResourcePrinter + + +launch_service() +{ + class="$1" + iotdb_parms="-Dlogback.configurationFile=${IOTDB_CONF}/logback.xml" + iotdb_parms="$iotdb_parms -DIOTDB_HOME=${IOTDB_HOME}" + iotdb_parms="$iotdb_parms -DTSFILE_HOME=${IOTDB_HOME}" + iotdb_parms="$iotdb_parms -DIOTDB_CONF=${IOTDB_CONF}" +# iotdb_parms="$iotdb_parms -Dname=iotdb\.IoTDB" + exec "$JAVA" $iotdb_parms $IOTDB_JMX_OPTS $iotdb_parms -cp "$CLASSPATH" "$class" "$WALPATH" + return $? +} + +# Start up the service +#launch_service "$classname" + +if [ ! -d ${IOTDB_HOME}/data/wal ]; then + echo "Can't find wal directory." + exit 1; +else + WALPATH=${IOTDB_HOME}/data/wal + launch_service "$classname" +fi + +exit $? diff --git a/server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g b/server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g index e53c4fc41abe637bdd12a5aa75a4901fd55a0862..aa5bc28d0737750fe8e546f3c05c61bbfd5018c3 100644 --- a/server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g +++ b/server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g @@ -119,7 +119,7 @@ ArrayList errors = new ArrayList(); private static HashMap xlateMap; static { - //this is used to support auto completion in CLI + //this is used to support auto completion in Client xlateMap = new HashMap(); // Keywords diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 202b0c9a73e2474a48706ee231fac7a261702170..0c92dee11fb8c95590f45022c6ca67301325cb21 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -23,6 +23,7 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.iotdb.db.engine.merge.selector.MergeFileStrategy; import org.apache.iotdb.db.metadata.MManager; import org.apache.iotdb.db.service.TSServiceImpl; import org.slf4j.Logger; @@ -48,6 +49,11 @@ public class IoTDBConfig { */ private int rpcPort = 6667; + /** + * Max concurrent client number + */ + private int maxConcurrentClientNum = 65535; + /** * Memory allocated for the read process */ @@ -133,12 +139,6 @@ public class IoTDBConfig { */ private int maxMemtableNumber = 20; - /** - * The maximum concurrent thread number for merging. When the value <=0 or > CPU core number, use - * the CPU core number. - */ - private int mergeConcurrentThreads = Runtime.getRuntime().availableProcessors(); - /** * The amount of data that is read every time when IoTDB merges data. */ @@ -248,6 +248,57 @@ public class IoTDBConfig { */ private boolean chunkBufferPoolEnable = false; + /** + * How much memory (in byte) can be used by a single merge task. + */ + private long mergeMemoryBudget = (long) (Runtime.getRuntime().maxMemory() * 0.2); + + /** + * How many threads will be set up to perform main merge tasks. + */ + private int mergeThreadNum = 1; + + /** + * How many threads will be set up to perform merge chunk sub-tasks. + */ + private int mergeChunkSubThreadNum = 4; + + /** + * If one merge file selection runs for more than this time, it will be ended and its current + * selection will be used as final selection. Unit: millis. + * When < 0, it means time is unbounded. + */ + private long mergeFileSelectionTimeBudget = 30 * 1000; + + /** + * When set to true, if some crashed merges are detected during system rebooting, such merges will + * be continued, otherwise, the unfinished parts of such merges will not be continued while the + * finished parts still remain as they are. + */ + private boolean continueMergeAfterReboot = true; + + /** + * A global merge will be performed each such interval, that is, each storage group will be merged + * (if proper merge candidates can be found). Unit: second. + */ + private long mergeIntervalSec = 2 * 3600L; + + /** + * When set to true, all merges becomes full merge (the whole SeqFiles are re-written despite how + * much they are overflowed). This may increase merge overhead depending on how much the SeqFiles + * are overflowed. + */ + private boolean forceFullMerge = false; + + /** + * During a merge, if a chunk with less number of chunks than this parameter, the chunk will be + * merged with its succeeding chunks even if it is not overflowed, until the merged chunks reach + * this threshold and the new chunk will be flushed. + */ + private int chunkMergePointThreshold = 20480; + + private MergeFileStrategy mergeFileStrategy = MergeFileStrategy.MAX_SERIES_NUM; + public IoTDBConfig() { // empty constructor } @@ -410,14 +461,6 @@ public class IoTDBConfig { this.indexFileDir = indexFileDir; } - public int getMergeConcurrentThreads() { - return mergeConcurrentThreads; - } - - void setMergeConcurrentThreads(int mergeConcurrentThreads) { - this.mergeConcurrentThreads = mergeConcurrentThreads; - } - public int getFetchSize() { return fetchSize; } @@ -470,6 +513,14 @@ public class IoTDBConfig { this.enableStatMonitor = enableStatMonitor; } + public int getMaxConcurrentClientNum() { + return maxConcurrentClientNum; + } + + public void setMaxConcurrentClientNum(int maxConcurrentClientNum) { + this.maxConcurrentClientNum = maxConcurrentClientNum; + } + public int getStatMonitorDetectFreqSec() { return statMonitorDetectFreqSec; } @@ -582,6 +633,38 @@ public class IoTDBConfig { this.chunkBufferPoolEnable = chunkBufferPoolEnable; } + public long getMergeMemoryBudget() { + return mergeMemoryBudget; + } + + public void setMergeMemoryBudget(long mergeMemoryBudget) { + this.mergeMemoryBudget = mergeMemoryBudget; + } + + public int getMergeThreadNum() { + return mergeThreadNum; + } + + public void setMergeThreadNum(int mergeThreadNum) { + this.mergeThreadNum = mergeThreadNum; + } + + public boolean isContinueMergeAfterReboot() { + return continueMergeAfterReboot; + } + + public void setContinueMergeAfterReboot(boolean continueMergeAfterReboot) { + this.continueMergeAfterReboot = continueMergeAfterReboot; + } + + public long getMergeIntervalSec() { + return mergeIntervalSec; + } + + public void setMergeIntervalSec(long mergeIntervalSec) { + this.mergeIntervalSec = mergeIntervalSec; + } + public boolean isEnableParameterAdapter() { return enableParameterAdapter; } @@ -630,6 +713,22 @@ public class IoTDBConfig { this.performanceStatMemoryInKB = performanceStatMemoryInKB; } + public boolean isForceFullMerge() { + return forceFullMerge; + } + + public void setForceFullMerge(boolean forceFullMerge) { + this.forceFullMerge = forceFullMerge; + } + + public int getChunkMergePointThreshold() { + return chunkMergePointThreshold; + } + + public void setChunkMergePointThreshold(int chunkMergePointThreshold) { + this.chunkMergePointThreshold = chunkMergePointThreshold; + } + public long getMemtableSizeThreshold() { return memtableSizeThreshold; } @@ -637,6 +736,31 @@ public class IoTDBConfig { public void setMemtableSizeThreshold(long memtableSizeThreshold) { this.memtableSizeThreshold = memtableSizeThreshold; } + + public MergeFileStrategy getMergeFileStrategy() { + return mergeFileStrategy; + } + + public void setMergeFileStrategy( + MergeFileStrategy mergeFileStrategy) { + this.mergeFileStrategy = mergeFileStrategy; + } + + public int getMergeChunkSubThreadNum() { + return mergeChunkSubThreadNum; + } + + public void setMergeChunkSubThreadNum(int mergeChunkSubThreadNum) { + this.mergeChunkSubThreadNum = mergeChunkSubThreadNum; + } + + public long getMergeFileSelectionTimeBudget() { + return mergeFileSelectionTimeBudget; + } + + public void setMergeFileSelectionTimeBudget(long mergeFileSelectionTimeBudget) { + this.mergeFileSelectionTimeBudget = mergeFileSelectionTimeBudget; + } public boolean isRpcThriftCompressionEnable() { return rpcThriftCompressionEnable; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java index 3ea16211d94998a67a78e31771466087b8aec2e6..02fe51cd4ba7c830ffacc9843de4d51e32a4890c 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java @@ -26,7 +26,7 @@ public class IoTDBConstant { public static final String ENV_FILE_NAME = "iotdb-env"; public static final String IOTDB_CONF = "IOTDB_CONF"; public static final String GLOBAL_DB_NAME = "IoTDB"; - public static final String VERSION = "0.7.0"; + public static final String VERSION = "0.9.0-SNAPSHOT"; public static final String REMOTE_JMX_PORT_NAME = "com.sun.management.jmxremote.port"; public static final String IOTDB_LOCAL_JMX_PORT_NAME = "iotdb.jmx.local.port"; public static final String IOTDB_REMOTE_JMX_PORT_NAME = "iotdb.jmx.remote.port"; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c0b98498320f5febbd3505e0e76a087a8c5f8e08..9a8397f18311154269e63a53f6214e1452af8a4b 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -170,14 +170,6 @@ public class IoTDBDescriptor { conf.setMultiDirStrategyClassName(properties.getProperty("multi_dir_strategy", conf.getMultiDirStrategyClassName())); - conf.setMergeConcurrentThreads(Integer - .parseInt(properties.getProperty("merge_concurrent_threads", - Integer.toString(conf.getMergeConcurrentThreads())))); - if (conf.getMergeConcurrentThreads() <= 0 - || conf.getMergeConcurrentThreads() > Runtime.getRuntime().availableProcessors()) { - conf.setMergeConcurrentThreads(Runtime.getRuntime().availableProcessors()); - } - conf.setFetchSize(Integer.parseInt(properties.getProperty("fetch_size", Integer.toString(conf.getFetchSize())))); @@ -231,6 +223,25 @@ public class IoTDBDescriptor { conf.setZoneID(ZoneId.of(tmpTimeZone.trim())); logger.info("Time zone has been set to {}", conf.getZoneID()); + conf.setMergeMemoryBudget(Long.parseLong(properties.getProperty("merge_memory_budget", + Long.toString(conf.getMergeMemoryBudget())))); + conf.setMergeThreadNum(Integer.parseInt(properties.getProperty("merge_thread_num", + Integer.toString(conf.getMergeThreadNum())))); + conf.setMergeChunkSubThreadNum(Integer.parseInt(properties.getProperty + ("merge_chunk_subthread_num", + Integer.toString(conf.getMergeChunkSubThreadNum())))); + conf.setContinueMergeAfterReboot(Boolean.parseBoolean(properties.getProperty( + "continue_merge_after_reboot", Boolean.toString(conf.isContinueMergeAfterReboot())))); + conf.setMergeFileSelectionTimeBudget(Long.parseLong(properties.getProperty + ("merge_fileSelection_time_budget", + Long.toString(conf.getMergeFileSelectionTimeBudget())))); + conf.setMergeIntervalSec(Long.parseLong(properties.getProperty("merge_interval_sec", + Long.toString(conf.getMergeIntervalSec())))); + conf.setForceFullMerge(Boolean.parseBoolean(properties.getProperty("force_full_merge", + Boolean.toString(conf.isForceFullMerge())))); + conf.setChunkMergePointThreshold(Integer.parseInt(properties.getProperty( + "chunk_merge_point_threshold", Integer.toString(conf.getChunkMergePointThreshold())))); + conf.setEnablePerformanceStat(Boolean .parseBoolean(properties.getProperty("enable_performance_stat", Boolean.toString(conf.isEnablePerformanceStat())).trim())); @@ -241,6 +252,15 @@ public class IoTDBDescriptor { conf.setPerformanceStatMemoryInKB(Integer .parseInt(properties.getProperty("performance_stat_memory_in_kb", Integer.toString(conf.getPerformanceStatMemoryInKB())).trim())); + + int maxConcurrentClientNum = Integer.parseInt(properties. + getProperty("max_concurrent_client_num", + Integer.toString(conf.getMaxConcurrentClientNum()).trim())); + if (maxConcurrentClientNum <= 0) { + maxConcurrentClientNum = 65535; + } + conf.setMaxConcurrentClientNum(maxConcurrentClientNum); + } catch (IOException e) { logger.warn("Cannot load config file because, use default configuration", e); } catch (Exception e) { diff --git a/server/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java b/server/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java index 1c7aded57638b465e1db274088a9d5a66449cbca..377023b2bbc0caa8d149c91943f61d8f474ed5bb 100644 --- a/server/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java +++ b/server/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java @@ -253,7 +253,7 @@ public class Measurement implements MeasurementMBean, IService { service = IoTDBThreadPoolFactory.newScheduledThreadPool( 2, ThreadName.TIME_COST_STATSTIC.getName()); } - //we have to check again because someone may channge the value. + //we have to check again because someone may change the value. isEnableStat = IoTDBDescriptor.getInstance().getConfig().isEnablePerformanceStat(); if (isEnableStat) { consumeFuture = service.schedule(new QueueConsumerThread(), 0, TimeUnit.MILLISECONDS); diff --git a/server/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java b/server/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java index 6f862ba4e441b96cf9909a327354fd86253e19ac..3ff43e31334e94059885d675b0b1c58235ec0bed 100644 --- a/server/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java +++ b/server/src/main/java/org/apache/iotdb/db/cost/statistic/Operation.java @@ -19,8 +19,9 @@ package org.apache.iotdb.db.cost.statistic; public enum Operation { - EXECUTE_BATCH("EXECUTE_BATCH"), + EXECUTE_JDBC_BATCH("EXECUTE_JDBC_BATCH"), EXECUTE_ONE_SQL_IN_BATCH("EXECUTE_ONE_SQL_IN_BATCH"), + EXECUTE_RPC_BATCH_INSERT("EXECUTE_RPC_BATCH_INSERT"), EXECUTE_QUERY("EXECUTE_QUERY"); public String getName() { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java index 8b5f811c0ed6db4a991142e183ca43251f4ee713..3e36516f301641cc4334f34e09b8ffbdbc7ba71b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java @@ -37,8 +37,10 @@ import org.apache.iotdb.db.exception.ProcessorException; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.StorageEngineFailureException; import org.apache.iotdb.db.metadata.MManager; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.control.JobFileManager; import org.apache.iotdb.db.service.IService; import org.apache.iotdb.db.service.ServiceType; import org.apache.iotdb.db.utils.FilePathUtils; @@ -148,7 +150,7 @@ public class StorageEngine implements IService { /** - * execute an InsertPlan on a storage group. + * insert an InsertPlan to a storage group. * * @param insertPlan physical plan of insertion * @return true if and only if this insertion succeeds @@ -169,6 +171,25 @@ public class StorageEngine implements IService { return storageGroupProcessor.insert(insertPlan); } + /** + * insert a BatchInsertPlan to a storage group + * @return result of each row + */ + public Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws StorageEngineException { + StorageGroupProcessor storageGroupProcessor; + try { + storageGroupProcessor = getProcessor(batchInsertPlan.getDeviceId()); + } catch (Exception e) { + logger.warn("get StorageGroupProcessor of device {} failed, because {}", + batchInsertPlan.getDeviceId(), + e.getMessage(), e); + throw new StorageEngineException(e); + } + + // TODO monitor: update statistics + return storageGroupProcessor.insertBatch(batchInsertPlan); + } + /** * only for unit test */ @@ -209,37 +230,17 @@ public class StorageEngine implements IService { } } - - /** - * begin a query on a given deviceId. Any TsFile contains such device should not be deleted at - * once after merge. - * - * @param deviceId queried deviceId - * @return a token for the query. - */ - public int beginQuery(String deviceId) throws StorageEngineException { - // TODO implement it when developing the merge function - return -1; - } - - /** - * end query on a given deviceId. If some TsFile has been merged and this query is the last query - * using it, the TsFile can be deleted safely. - */ - public void endQuery(String deviceId, int token) throws StorageEngineException { - // TODO implement it when developing the merge function - } - /** * query data. */ - public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context) + public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context, + JobFileManager filePathsManager) throws StorageEngineException { //TODO use context. String deviceId = seriesExpression.getSeriesPath().getDevice(); String measurementId = seriesExpression.getSeriesPath().getMeasurement(); StorageGroupProcessor storageGroupProcessor = getProcessor(deviceId); - return storageGroupProcessor.query(deviceId, measurementId, context); + return storageGroupProcessor.query(deviceId, measurementId, context, filePathsManager); } /** @@ -284,8 +285,13 @@ public class StorageEngine implements IService { * * @throws StorageEngineException StorageEngineException */ - public void mergeAll() throws StorageEngineException { - // TODO + public void mergeAll(boolean fullMerge) throws StorageEngineException { + if (IoTDBDescriptor.getInstance().getConfig().isReadOnly()) { + throw new StorageEngineException("Current system mode is read only, does not support merge"); + } + for (StorageGroupProcessor storageGroupProcessor : processorMap.values()) { + storageGroupProcessor.merge(fullMerge); + } } /** diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java index 5d2342f98df8d5f6bcb63ce6e1b4f5afbc5f6897..fd0eb14a8324e44b8933d5daf49a2dd83af97d65 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java @@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.StorageEngine; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; @@ -85,16 +86,17 @@ public class DeviceMetaDataCache { /** * get {@link ChunkMetaData}. THREAD SAFE. */ - public List get(String filePath, Path seriesPath) + public List get(TsFileResource resource, Path seriesPath) throws IOException { if (!cacheEnable) { - TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath); + TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(resource); TsDeviceMetadata deviceMetaData = TsFileMetadataUtils - .getTsDeviceMetaData(filePath, seriesPath, fileMetaData); + .getTsDeviceMetaData(resource, seriesPath, fileMetaData); return TsFileMetadataUtils.getChunkMetaDataList(seriesPath.getMeasurement(), deviceMetaData); } - StringBuilder builder = new StringBuilder(filePath).append(".").append(seriesPath.getDevice()); + StringBuilder builder = new StringBuilder(resource.getFile().getPath()).append(".").append(seriesPath + .getDevice()); String pathDeviceStr = builder.toString(); String key = builder.append(".").append(seriesPath.getMeasurement()).toString(); Object devicePathObject = pathDeviceStr.intern(); @@ -123,9 +125,9 @@ public class DeviceMetaDataCache { logger.debug("Cache didn't hit: the number of requests for cache is {}", cacheRequestNum.get()); } - TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(filePath); + TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(resource); TsDeviceMetadata deviceMetaData = TsFileMetadataUtils - .getTsDeviceMetaData(filePath, seriesPath, fileMetaData); + .getTsDeviceMetaData(resource, seriesPath, fileMetaData); // If measurement isn't included in the tsfile, empty list is returned. if (deviceMetaData == null) { return new ArrayList<>(); @@ -193,4 +195,10 @@ public class DeviceMetaDataCache { private static final DeviceMetaDataCache INSTANCE = new DeviceMetaDataCache(MEMORY_THRESHOLD_IN_B); } + + public void remove(TsFileResource resource) { + synchronized (lruCache) { + lruCache.entrySet().removeIf(e -> e.getKey().startsWith(resource.getFile().getPath())); + } + } } \ No newline at end of file diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java index 6b7e448b324f475e3fef91efde6fbcbccfd3dd09..d85a18d8d705491c763ff65218192e02e74e3e9d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java @@ -22,7 +22,9 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; +import org.apache.iotdb.tsfile.read.common.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +41,7 @@ public class TsFileMetaDataCache { /** * key: Tsfile path. value: TsFileMetaData */ - private LRULinkedHashMap cache; + private LRULinkedHashMap cache; private AtomicLong cacheHitNum = new AtomicLong(); private AtomicLong cacheRequestNum = new AtomicLong(); @@ -60,9 +62,9 @@ public class TsFileMetaDataCache { if (!cacheEnable) { return; } - cache = new LRULinkedHashMap(MEMORY_THRESHOLD_IN_B, true) { + cache = new LRULinkedHashMap(MEMORY_THRESHOLD_IN_B, true) { @Override - protected long calEntrySize(String key, TsFileMetaData value) { + protected long calEntrySize(TsFileResource key, TsFileMetaData value) { if (deviceIndexMapEntrySize == 0 && value.getDeviceMap().size() > 0) { deviceIndexMapEntrySize = RamUsageEstimator .sizeOf(value.getDeviceMap().entrySet().iterator().next()); @@ -74,7 +76,7 @@ public class TsFileMetaDataCache { long valueSize = value.getDeviceMap().size() * deviceIndexMapEntrySize + measurementSchemaEntrySize * value.getMeasurementSchema().size() + versionAndCreatebySize; - return key.length() * 2 + valueSize; + return key.getFile().getPath().length() * 2 + valueSize; } }; } @@ -84,15 +86,16 @@ public class TsFileMetaDataCache { } /** - * get the TsFileMetaData for given path. + * get the TsFileMetaData for given TsFile. * - * @param path -given path + * @param tsFileResource -given TsFile */ - public TsFileMetaData get(String path) throws IOException { + public TsFileMetaData get(TsFileResource tsFileResource) throws IOException { if (!cacheEnable) { - return TsFileMetadataUtils.getTsFileMetaData(path); + return TsFileMetadataUtils.getTsFileMetaData(tsFileResource); } + String path = tsFileResource.getFile().getPath(); Object internPath = path.intern(); cacheRequestNum.incrementAndGet(); synchronized (cache) { @@ -118,18 +121,18 @@ public class TsFileMetaDataCache { logger.debug("Cache didn't hit: the number of requests for cache is {}", cacheRequestNum.get()); } - TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(path); + TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(tsFileResource); synchronized (cache) { - cache.put(path, fileMetaData); + cache.put(tsFileResource, fileMetaData); return fileMetaData; } } } - public void remove(String path) { + public void remove(TsFileResource resource) { synchronized (cache) { if (cache != null) { - cache.remove(path); + cache.remove(resource); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java index 751939a0c5668855da96b764a1189b20b04fe66e..4bce328cf08c471fa27d2b94c8a7db4f5cfd23f3 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java @@ -20,10 +20,12 @@ package org.apache.iotdb.db.engine.cache; import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; @@ -46,30 +48,30 @@ public class TsFileMetadataUtils { /** * get tsfile meta data. * - * @param filePath -given path + * @param resource -given TsFile * @return -meta data */ - public static TsFileMetaData getTsFileMetaData(String filePath) throws IOException { - TsFileSequenceReader reader = FileReaderManager.getInstance().get(filePath, true); + public static TsFileMetaData getTsFileMetaData(TsFileResource resource) throws IOException { + TsFileSequenceReader reader = FileReaderManager.getInstance().get(resource, true); return reader.readFileMetadata(); } /** * get row group block meta data. * - * @param filePath -file path + * @param resource -TsFile * @param seriesPath -series path * @param fileMetaData -tsfile meta data * @return -device meta data */ - public static TsDeviceMetadata getTsDeviceMetaData(String filePath, Path seriesPath, + public static TsDeviceMetadata getTsDeviceMetaData(TsFileResource resource, Path seriesPath, TsFileMetaData fileMetaData) throws IOException { if (!fileMetaData.getMeasurementSchema().containsKey(seriesPath.getMeasurement())) { return null; } else { // get the index information of TsDeviceMetadata TsDeviceMetadataIndex index = fileMetaData.getDeviceMetadataIndex(seriesPath.getDevice()); - TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true); + TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(resource, true); // read TsDeviceMetadata from file return tsFileReader.readTsDeviceMetaData(index); } @@ -95,6 +97,9 @@ public class TsFileMetadataUtils { } } } + for (List chunkMetaDataList : pathToChunkMetaDataList.values()) { + chunkMetaDataList.sort(Comparator.comparingLong(ChunkMetaData::getStartTime)); + } return pathToChunkMetaDataList; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/FlushManager.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/FlushManager.java index 2ed5305e9748a8e9955848c280c8817ebe0cbcd7..f2de30598de3e828fed63a35702ab247a49205e6 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/FlushManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/FlushManager.java @@ -96,7 +96,7 @@ public class FlushManager implements FlushManagerMBean, IService { } /** - * Add BufferWriteProcessor to asyncTryToFlush manager + * Add TsFileProcessor to asyncTryToFlush manager */ @SuppressWarnings("squid:S2445") public void registerTsFileProcessor(TsFileProcessor tsFileProcessor) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java index b6cdb97cefdcb2962d81357a7da348a8d469225e..412f3b729f80f88abaacd2fce2cb751d9190ad5d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java @@ -30,7 +30,7 @@ import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer; import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; import org.slf4j.Logger; @@ -50,14 +50,14 @@ public class MemTableFlushTask { private String storageGroup; private IMemTable memTable; - private FileSchema fileSchema; + private Schema schema; private volatile boolean noMoreEncodingTask = false; private volatile boolean noMoreIOTask = false; - public MemTableFlushTask(IMemTable memTable, FileSchema fileSchema, RestorableTsFileIOWriter writer, String storageGroup) { + public MemTableFlushTask(IMemTable memTable, Schema schema, RestorableTsFileIOWriter writer, String storageGroup) { this.memTable = memTable; - this.fileSchema = fileSchema; + this.schema = schema; this.writer = writer; this.storageGroup = storageGroup; subTaskPoolManager.submit(encodingTask); @@ -78,7 +78,7 @@ public class MemTableFlushTask { for (String measurementId : memTable.getMemTableMap().get(deviceId).keySet()) { long startTime = System.currentTimeMillis(); IWritableMemChunk series = memTable.getMemTableMap().get(deviceId).get(measurementId); - MeasurementSchema desc = fileSchema.getMeasurementSchema(measurementId); + MeasurementSchema desc = schema.getMeasurementSchema(measurementId); TVList tvList = series.getSortedTVList(); sortTime += System.currentTimeMillis() - startTime; encodingTaskQueue.add(new Pair<>(tvList, desc)); @@ -168,8 +168,7 @@ public class MemTableFlushTask { Pair encodingMessage = (Pair) task; ChunkBuffer chunkBuffer = ChunkBufferPool.getInstance() .getEmptyChunkBuffer(this, encodingMessage.right); - IChunkWriter seriesWriter = new ChunkWriterImpl(encodingMessage.right, chunkBuffer, - PAGE_SIZE_THRESHOLD); + IChunkWriter seriesWriter = new ChunkWriterImpl(chunkBuffer, PAGE_SIZE_THRESHOLD); writeOneSeries(encodingMessage.left, seriesWriter, encodingMessage.right.getType()); ioTaskQueue.add(seriesWriter); memSerializeTime += System.currentTimeMillis() - starTime; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java index a4fd2a91a62092a9c17b35d5309eb226fdae98cc..2aa882e728dcf7d9dd7c25f9135395ec31a49bab 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java @@ -26,6 +26,7 @@ import java.util.Map.Entry; import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.rescon.TVListAllocator; import org.apache.iotdb.db.utils.MemUtils; @@ -88,6 +89,14 @@ public abstract class AbstractMemTable implements IMemTable { memSize += recordSizeInByte; } + @Override + public void insertBatch(BatchInsertPlan batchInsertPlan, List indexes) { + write(batchInsertPlan, indexes); + long recordSizeInByte = MemUtils.getRecordSize(batchInsertPlan); + memSize += recordSizeInByte; + } + + @Override public void write(String deviceId, String measurement, TSDataType dataType, long insertTime, String insertValue) { @@ -96,13 +105,15 @@ public abstract class AbstractMemTable implements IMemTable { } @Override - public void write(String deviceId, String measurement, TSDataType dataType, long insertTime, - Object value) { - IWritableMemChunk memSeries = createIfNotExistAndGet(deviceId, measurement, dataType); - memSeries.write(insertTime, value); - // update memory size of current memtable + public void write(BatchInsertPlan batchInsertPlan, List indexes) { + for (int i = 0; i < batchInsertPlan.getMeasurements().length; i++) { + IWritableMemChunk memSeries = createIfNotExistAndGet(batchInsertPlan.getDeviceId(), + batchInsertPlan.getMeasurements()[i], batchInsertPlan.getDataTypes()[i]); + memSeries.write(batchInsertPlan.getTimes(), batchInsertPlan.getColumns()[i], batchInsertPlan.getDataTypes()[i], indexes); + } } + @Override public long size() { long sum = 0; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java index accbc5218563553d81ae214045e6b1fb90529621..5501e334837a3121862d59e95167de43f5656dde 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java @@ -18,9 +18,11 @@ */ package org.apache.iotdb.db.engine.memtable; +import java.util.List; import java.util.Map; import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -38,8 +40,7 @@ public interface IMemTable { void write(String deviceId, String measurement, TSDataType dataType, long insertTime, String insertValue); - void write(String deviceId, String measurement, TSDataType dataType, - long insertTime, Object value); + void write(BatchInsertPlan batchInsertPlan, List indexes); /** * @return the number of points @@ -53,6 +54,8 @@ public interface IMemTable { void insert(InsertPlan insertPlan); + void insertBatch(BatchInsertPlan batchInsertPlan, List indexes); + ReadOnlyMemChunk query(String deviceId, String measurement, TSDataType dataType, Map props); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java index a7b4ed1211f296980c1d294b45ac68bec667e2d5..e5d683ce9e30667d9f6d62796908da4fcf35cae5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.engine.memtable; +import java.util.List; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; @@ -36,9 +37,21 @@ public interface IWritableMemChunk extends TimeValuePairSorter { void putBoolean(long t, boolean v); + void putLongs(long[] t, long[] v); + + void putInts(long[] t, int[] v); + + void putFloats(long[] t, float[] v); + + void putDoubles(long[] t, double[] v); + + void putBinaries(long[] t, Binary[] v); + + void putBooleans(long[] t, boolean[] v); + void write(long insertTime, String insertValue); - void write(long insertTime, Object insertValue); + void write(long[] times, Object valueList, TSDataType dataType, List indexes); long count(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java index 0d34a57183c36b25a5211d0a550dcbcdbb0f096f..b37a2237fb16a74e8818ccc59434609430e66250 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java @@ -74,25 +74,67 @@ public class WritableMemChunk implements IWritableMemChunk { } @Override - public void write(long insertTime, Object value) { + public void write(long[] times, Object valueList, TSDataType dataType, List indexes) { switch (dataType) { case BOOLEAN: - putBoolean(insertTime, (Boolean)value); + boolean[] boolValues = (boolean[]) valueList; + if (times.length == indexes.size()) { + putBooleans(times, boolValues); + break; + } + for (Integer index : indexes) { + putBoolean(times[index], boolValues[index]); + } break; case INT32: - putInt(insertTime, (Integer)value); + int[] intValues = (int[]) valueList; + if (times.length == indexes.size()) { + putInts(times, intValues); + break; + } + for (Integer index : indexes) { + putInt(times[index], intValues[index]); + } break; case INT64: - putLong(insertTime, (Long)value); + long[] longValues = (long[]) valueList; + if (times.length == indexes.size()) { + putLongs(times, longValues); + break; + } + for (Integer index : indexes) { + putLong(times[index], longValues[index]); + } break; case FLOAT: - putFloat(insertTime, (Float)value); + float[] floatValues = (float[]) valueList; + if (times.length == indexes.size()) { + putFloats(times, floatValues); + break; + } + for (Integer index : indexes) { + putFloat(times[index], floatValues[index]); + } break; case DOUBLE: - putDouble(insertTime, (Double)value); + double[] doubleValues = (double[]) valueList; + if (times.length == indexes.size()) { + putDoubles(times, doubleValues); + break; + } + for (Integer index : indexes) { + putDouble(times[index], doubleValues[index]); + } break; case TEXT: - putBinary(insertTime, (Binary)value); + Binary[] binaryValues = (Binary[]) valueList; + if (times.length == indexes.size()) { + putBinaries(times, binaryValues); + break; + } + for (Integer index : indexes) { + putBinary(times[index], binaryValues[index]); + } break; default: throw new UnSupportedDataTypeException("Unsupported data type:" + dataType); @@ -131,6 +173,36 @@ public class WritableMemChunk implements IWritableMemChunk { list.putBoolean(t, v); } + @Override + public void putLongs(long[] t, long[] v) { + list.putLongs(t, v); + } + + @Override + public void putInts(long[] t, int[] v) { + list.putInts(t, v); + } + + @Override + public void putFloats(long[] t, float[] v) { + list.putFloats(t, v); + } + + @Override + public void putDoubles(long[] t, double[] v) { + list.putDoubles(t, v); + } + + @Override + public void putBinaries(long[] t, Binary[] v) { + list.putBinaries(t, v); + } + + @Override + public void putBooleans(long[] t, boolean[] v) { + list.putBooleans(t, v); + } + @Override public synchronized TVList getSortedTVList() { list.sort(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeContext.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeContext.java new file mode 100644 index 0000000000000000000000000000000000000000..c6dd26c81ab8f41d01889e84c172563d15f2bd6a --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeContext.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.manage; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.tsfile.read.common.Path; + +/** + * MergeContext records the shared information between merge sub-tasks. + */ +public class MergeContext { + + private Map mergedChunkCnt = new HashMap<>(); + private Map unmergedChunkCnt = new HashMap<>(); + private Map>> unmergedChunkStartTimes = new HashMap<>(); + + private AtomicInteger totalChunkWritten = new AtomicInteger(); + private AtomicLong totalPointWritten = new AtomicLong(); + + public void clear() { + mergedChunkCnt.clear(); + unmergedChunkCnt.clear(); + unmergedChunkStartTimes.clear(); + } + + public Map getMergedChunkCnt() { + return mergedChunkCnt; + } + + public void setMergedChunkCnt( + Map mergedChunkCnt) { + this.mergedChunkCnt = mergedChunkCnt; + } + + public Map getUnmergedChunkCnt() { + return unmergedChunkCnt; + } + + public void setUnmergedChunkCnt( + Map unmergedChunkCnt) { + this.unmergedChunkCnt = unmergedChunkCnt; + } + + public Map>> getUnmergedChunkStartTimes() { + return unmergedChunkStartTimes; + } + + public void setUnmergedChunkStartTimes( + Map>> unmergedChunkStartTimes) { + this.unmergedChunkStartTimes = unmergedChunkStartTimes; + } + + public int getTotalChunkWritten() { + return totalChunkWritten.get(); + } + + public void incTotalChunkWritten() { + this.totalChunkWritten.incrementAndGet(); + } + + public void incTotalPointWritten(long increment) { + totalPointWritten.addAndGet(increment); + } + + public long getTotalPointWritten() { + return totalPointWritten.get(); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java new file mode 100644 index 0000000000000000000000000000000000000000..c23768ad85851327b066dabd4c9041f097bf6869 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.manage; + +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.StorageEngine; +import org.apache.iotdb.db.engine.merge.task.MergeTask; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.service.IService; +import org.apache.iotdb.db.service.ServiceType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MergeManager provides a ThreadPool to queue and run all merge tasks to restrain the total + * resources occupied by merge and manages a Timer to periodically issue a global merge. + */ +public class MergeManager implements IService { + + private static final Logger logger = LoggerFactory.getLogger(MergeManager.class); + private static final MergeManager INSTANCE = new MergeManager(); + + private AtomicInteger threadCnt = new AtomicInteger(); + private ThreadPoolExecutor mergeTaskPool; + private ThreadPoolExecutor mergeChunkSubTaskPool; + private ScheduledExecutorService timedMergeThreadPool; + + private MergeManager() { + } + + public static MergeManager getINSTANCE() { + return INSTANCE; + } + + public void submitMainTask(MergeTask mergeTask) { + mergeTaskPool.submit(mergeTask); + } + + public Future submitChunkSubTask(Callable callable) { + return mergeChunkSubTaskPool.submit(callable); + } + + @Override + public void start() { + if (mergeTaskPool == null) { + int threadNum = IoTDBDescriptor.getInstance().getConfig().getMergeThreadNum(); + if (threadNum <= 0) { + threadNum = 1; + } + + int chunkSubThreadNum = IoTDBDescriptor.getInstance().getConfig().getMergeChunkSubThreadNum(); + if (chunkSubThreadNum <= 0) { + chunkSubThreadNum = 1; + } + + mergeTaskPool = + (ThreadPoolExecutor) Executors.newFixedThreadPool(threadNum, + r -> new Thread(r, "MergeThread-" + threadCnt.getAndIncrement())); + mergeChunkSubTaskPool = + (ThreadPoolExecutor) Executors.newFixedThreadPool(threadNum * chunkSubThreadNum, + r -> new Thread(r, "MergeChunkSubThread-" + threadCnt.getAndIncrement())); + long mergeInterval = IoTDBDescriptor.getInstance().getConfig().getMergeIntervalSec(); + if (mergeInterval > 0) { + timedMergeThreadPool = Executors.newSingleThreadScheduledExecutor( r -> new Thread(r, + "TimedMergeThread")); + timedMergeThreadPool.scheduleAtFixedRate(this::mergeAll, mergeInterval, + mergeInterval, TimeUnit.SECONDS); + } + logger.info("MergeManager started"); + } + } + + @Override + public void stop() { + if (mergeTaskPool != null) { + if (timedMergeThreadPool != null) { + timedMergeThreadPool.shutdownNow(); + timedMergeThreadPool = null; + } + mergeTaskPool.shutdownNow(); + mergeChunkSubTaskPool.shutdownNow(); + logger.info("Waiting for task pool to shut down"); + while (!mergeTaskPool.isTerminated() || !mergeChunkSubTaskPool.isTerminated() ) { + // wait + } + mergeTaskPool = null; + logger.info("MergeManager stopped"); + } + } + + @Override + public ServiceType getID() { + return ServiceType.MERGE_SERVICE; + } + + private void mergeAll() { + try { + StorageEngine.getInstance().mergeAll(IoTDBDescriptor.getInstance().getConfig().isForceFullMerge()); + } catch (StorageEngineException e) { + logger.error("Cannot perform a global merge because", e); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java new file mode 100644 index 0000000000000000000000000000000000000000..042c499ad38a5a140ce807bdd0490d00b6f2c7d6 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.manage; + +import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.apache.iotdb.db.engine.modification.Modification; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.query.reader.IPointReader; +import org.apache.iotdb.db.query.reader.resourceRelated.CachedUnseqResourceMergeReader; +import org.apache.iotdb.db.utils.MergeUtils; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer; +import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MergeResource manages files and caches of readers, writers, MeasurementSchemas and + * modifications to avoid unnecessary object creations and file openings. + */ +public class MergeResource { + + private static final Logger logger = LoggerFactory.getLogger(MergeResource.class); + + private List seqFiles; + private List unseqFiles; + + private Map fileReaderCache = new HashMap<>(); + private Map fileWriterCache = new HashMap<>(); + private Map> modificationCache = new HashMap<>(); + private Map measurementSchemaMap = new HashMap<>(); + private Map chunkWriterCache = new ConcurrentHashMap<>(); + + private boolean cacheDeviceMeta = false; + + public MergeResource(List seqFiles, List unseqFiles) { + this.seqFiles = seqFiles.stream().filter(TsFileResource::isClosed).collect(Collectors.toList()); + this.unseqFiles = + unseqFiles.stream().filter(TsFileResource::isClosed).collect(Collectors.toList()); + } + + public void clear() throws IOException { + for (TsFileSequenceReader sequenceReader : fileReaderCache.values()) { + sequenceReader.close(); + } + + fileReaderCache.clear(); + fileWriterCache.clear(); + modificationCache.clear(); + measurementSchemaMap.clear(); + chunkWriterCache.clear(); + } + + public MeasurementSchema getSchema(String measurementId) { + return measurementSchemaMap.get(measurementId); + } + + /** + * Construct a new or get an existing RestorableTsFileIOWriter of a merge temp file for a + * SeqFile. The path of the merge temp file will be the seqFile's + ".merge". + * @param resource + * @return A RestorableTsFileIOWriter of a merge temp file for a SeqFile. + * @throws IOException + */ + public RestorableTsFileIOWriter getMergeFileWriter(TsFileResource resource) throws IOException { + RestorableTsFileIOWriter writer = fileWriterCache.get(resource); + if (writer == null) { + writer = new RestorableTsFileIOWriter(new File(resource.getFile().getPath() + MERGE_SUFFIX)); + fileWriterCache.put(resource, writer); + } + return writer; + } + + /** + * Query ChunkMetadata of a timeseries from the given TsFile (seq or unseq). The ChunkMetadata + * is not cached since it is usually huge. + * @param path name of the time series + * @param seqFile + * @return + * @throws IOException + */ + public List queryChunkMetadata(Path path, TsFileResource seqFile) + throws IOException { + TsFileSequenceReader sequenceReader = getFileReader(seqFile); + return sequenceReader.getChunkMetadataList(path); + } + + /** + * Construct the a new or get an existing TsFileSequenceReader of a TsFile. + * @param tsFileResource + * @return a TsFileSequenceReader + */ + public TsFileSequenceReader getFileReader(TsFileResource tsFileResource) throws IOException { + TsFileSequenceReader reader = fileReaderCache.get(tsFileResource); + if (reader == null) { + reader = new TsFileSequenceReader(tsFileResource.getFile().getPath(), true, cacheDeviceMeta); + fileReaderCache.put(tsFileResource, reader); + } + return reader; + } + + /** + * Construct UnseqResourceMergeReaders of for each timeseries over all seqFiles. The readers are + * not cached since the method is only called once for each timeseries. + * @param paths names of the timeseries + * @return an array of UnseqResourceMergeReaders each corresponding to a timeseries in paths + * @throws IOException + */ + public IPointReader[] getUnseqReaders(List paths) throws IOException { + List[] pathChunks = MergeUtils.collectUnseqChunks(paths, unseqFiles, this); + IPointReader[] ret = new IPointReader[paths.size()]; + for (int i = 0; i < paths.size(); i++) { + TSDataType dataType = getSchema(paths.get(i).getMeasurement()).getType(); + ret[i] = new CachedUnseqResourceMergeReader(pathChunks[i], dataType); + } + return ret; + } + + /** + * Construct the a new or get an existing ChunkWriter of a measurement. Different timeseries of + * the same measurement shares the same instance. + * @param measurementSchema + * @return + */ + public IChunkWriter getChunkWriter(MeasurementSchema measurementSchema) { + return chunkWriterCache.computeIfAbsent(measurementSchema, + schema -> new ChunkWriterImpl(new ChunkBuffer(schema), TSFileConfig.pageCheckSizeThreshold)); + } + + /** + * Get the modifications of a timeseries in the ModificationFile of a TsFile. Once the + * modifications of the timeseries are found out, they will be removed from the list to boost + * the next query, so two calls of the same file and timeseries are forbidden. + * @param tsFileResource + * @param path name of the time series + * @return + */ + public List getModifications(TsFileResource tsFileResource, Path path) { + // copy from TsFileResource so queries are not affected + List modifications = modificationCache.computeIfAbsent(tsFileResource, + resource -> new LinkedList<>(resource.getModFile().getModifications())); + List pathModifications = new ArrayList<>(); + Iterator modificationIterator = modifications.iterator(); + // each path is visited only once in a merge, so the modifications can be removed after visiting + while (modificationIterator.hasNext()) { + Modification modification = modificationIterator.next(); + if (modification.getPath().equals(path)) { + pathModifications.add(modification); + modificationIterator.remove(); + } + } + return pathModifications; + } + + /** + * Remove and close the writer of the merge temp file of a SeqFile. The merge temp file is also + * deleted. + * @param tsFileResource the SeqFile + * @throws IOException + */ + public void removeFileAndWriter(TsFileResource tsFileResource) throws IOException { + RestorableTsFileIOWriter newFileWriter = fileWriterCache.remove(tsFileResource); + if (newFileWriter != null) { + newFileWriter.close(); + newFileWriter.getFile().delete(); + } + } + + /** + * Remove and close the reader of the TsFile. The TsFile is NOT deleted. + * @param resource the SeqFile + * @throws IOException + */ + public void removeFileReader(TsFileResource resource) throws IOException { + TsFileSequenceReader sequenceReader = fileReaderCache.remove(resource); + if (sequenceReader != null) { + sequenceReader.close(); + } + } + + public List getSeqFiles() { + return seqFiles; + } + + public void setSeqFiles(List seqFiles) { + this.seqFiles = seqFiles; + } + + public List getUnseqFiles() { + return unseqFiles; + } + + public void setUnseqFiles( + List unseqFiles) { + this.unseqFiles = unseqFiles; + } + + public Map getMeasurementSchemaMap() { + return measurementSchemaMap; + } + + public void removeOutdatedSeqReaders() throws IOException { + Iterator> entryIterator = + fileReaderCache.entrySet().iterator(); + while (entryIterator.hasNext()) { + Entry entry = entryIterator.next(); + TsFileResource tsFile = entry.getKey(); + if (!seqFiles.contains(tsFile)) { + TsFileSequenceReader reader = entry.getValue(); + reader.close(); + entryIterator.remove(); + } + } + } + + public void setCacheDeviceMeta(boolean cacheDeviceMeta) { + this.cacheDeviceMeta = cacheDeviceMeta; + } + + public void addMeasurements(List measurementSchemas) { + for (MeasurementSchema measurementSchema : measurementSchemas) { + measurementSchemaMap.put(measurementSchema.getMeasurementId(), measurementSchema); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/package-info.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..9ed9a76f15849700a8b1af353ddeee28c21b5744 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/package-info.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Package merge contains classes supporting merge functionality (also compaction in other + * systems) that compacts several data files into a new one, making it more ordered and removing + * duplicated data. + */ +package org.apache.iotdb.db.engine.merge; \ No newline at end of file diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java new file mode 100644 index 0000000000000000000000000000000000000000..b7d98fc81ec44854e0e38a8ed3be88dd7ade1254 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/LogAnalyzer.java @@ -0,0 +1,300 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.recover; + +import static org.apache.iotdb.db.engine.merge.recover.MergeLogger.*; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.*; +import java.util.Map.Entry; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.task.MergeTask; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MetadataErrorException; +import org.apache.iotdb.db.metadata.MManager; +import org.apache.iotdb.tsfile.read.common.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * LogAnalyzer scans the "merge.log" file and recovers information such as files of last merge, + * the last available positions of each file and how many timeseries and files have been merged. + * An example of merging 1 seqFile and 1 unseqFile containing 3 series is: + * seqFiles + * server/0seq.tsfile + * unseqFiles + * server/0unseq.tsfile + * merge start + * start root.mergeTest.device0.sensor0 + * server/0seq.tsfile.merge 338 + * end + * start root.mergeTest.device0.sensor1 + * server/0seq.tsfile.merge 664 + * end + * all ts end + * server/0seq.tsfile 145462 + * end + * merge end + */ +public class LogAnalyzer { + + private static final Logger logger = LoggerFactory.getLogger(LogAnalyzer.class); + + private MergeResource resource; + private String taskName; + private File logFile; + private String storageGroupName; + + private Map fileLastPositions = new HashMap<>(); + private Map tempFileLastPositions = new HashMap<>(); + + private List mergedPaths = new ArrayList<>(); + private List unmergedPaths; + private List unmergedFiles; + private String currLine; + + private Status status; + + public LogAnalyzer(MergeResource resource, String taskName, File logFile, String storageGroupName) { + this.resource = resource; + this.taskName = taskName; + this.logFile = logFile; + this.storageGroupName = storageGroupName; + } + + /** + * Scan through the logs to find out where the last merge has stopped and store the information + * about the progress in the fields. + * @return a Status indicating the completed stage of the last merge. + * @throws IOException + */ + public Status analyze() throws IOException, MetadataErrorException { + status = Status.NONE; + try (BufferedReader bufferedReader = new BufferedReader(new FileReader(logFile))) { + currLine = bufferedReader.readLine(); + if (currLine != null) { + analyzeSeqFiles(bufferedReader); + + analyzeUnseqFiles(bufferedReader); + + List storageGroupPaths = MManager.getInstance().getPaths(storageGroupName + ".*"); + unmergedPaths = new ArrayList<>(); + for (String path : storageGroupPaths) { + unmergedPaths.add(new Path(path)); + } + + analyzeMergedSeries(bufferedReader, unmergedPaths); + + analyzeMergedFiles(bufferedReader); + } + } + return status; + } + + private void analyzeSeqFiles(BufferedReader bufferedReader) throws IOException { + if (!STR_SEQ_FILES.equals(currLine)) { + return; + } + long startTime = System.currentTimeMillis(); + List mergeSeqFiles = new ArrayList<>(); + while ((currLine = bufferedReader.readLine()) != null) { + if (STR_UNSEQ_FILES.equals(currLine)) { + break; + } + Iterator iterator = resource.getSeqFiles().iterator(); + while (iterator.hasNext()) { + TsFileResource seqFile = iterator.next(); + if (seqFile.getFile().getAbsolutePath().equals(currLine)) { + mergeSeqFiles.add(seqFile); + // remove to speed-up next iteration + iterator.remove(); + break; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("{} found {} seq files after {}ms", taskName, mergeSeqFiles.size(), + (System.currentTimeMillis() - startTime)); + } + resource.setSeqFiles(mergeSeqFiles); + } + + private void analyzeUnseqFiles(BufferedReader bufferedReader) throws IOException { + if (!STR_UNSEQ_FILES.equals(currLine)) { + return; + } + long startTime = System.currentTimeMillis(); + List mergeUnseqFiles = new ArrayList<>(); + while ((currLine = bufferedReader.readLine()) != null) { + if (currLine.equals(STR_TIMESERIES)) { + break; + } + Iterator iterator = resource.getUnseqFiles().iterator(); + while (iterator.hasNext()) { + TsFileResource unseqFile = iterator.next(); + if (unseqFile.getFile().getAbsolutePath().equals(currLine)) { + mergeUnseqFiles.add(unseqFile); + // remove to speed-up next iteration + iterator.remove(); + break; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("{} found {} unseq files after {}ms", taskName, mergeUnseqFiles.size(), + (System.currentTimeMillis() - startTime)); + } + resource.setUnseqFiles(mergeUnseqFiles); + } + + private void analyzeMergedSeries(BufferedReader bufferedReader, List unmergedPaths) throws IOException { + if (!STR_MERGE_START.equals(currLine)) { + return; + } + + status = Status.MERGE_START; + for (TsFileResource seqFile : resource.getSeqFiles()) { + File mergeFile = new File(seqFile.getFile().getPath() + MergeTask.MERGE_SUFFIX); + fileLastPositions.put(mergeFile, 0L); + } + + List currTSList = new ArrayList<>(); + long startTime = System.currentTimeMillis(); + while ((currLine = bufferedReader.readLine()) != null) { + if (STR_ALL_TS_END.equals(currLine)) { + break; + } + if (currLine.contains(STR_START)) { + // a TS starts to merge + String[] splits = currLine.split(" "); + for (int i = 1; i < splits.length; i ++) { + currTSList.add(new Path(splits[i])); + } + tempFileLastPositions.clear(); + } else if (!currLine.contains(STR_END)) { + // file position + String[] splits = currLine.split(" "); + File file = new File(splits[0]); + Long position = Long.parseLong(splits[1]); + tempFileLastPositions.put(file, position); + } else { + // a TS ends merging + unmergedPaths.removeAll(currTSList); + for (Entry entry : tempFileLastPositions.entrySet()) { + fileLastPositions.put(entry.getKey(), entry.getValue()); + } + mergedPaths.addAll(currTSList); + } + } + tempFileLastPositions = null; + if (logger.isDebugEnabled()) { + logger.debug("{} found {} series have already been merged after {}ms", taskName, + mergedPaths.size(), (System.currentTimeMillis() - startTime)); + } + } + + private void analyzeMergedFiles(BufferedReader bufferedReader) throws IOException { + if (!STR_ALL_TS_END.equals(currLine)) { + return; + } + + status = Status.ALL_TS_MERGED; + unmergedFiles = resource.getSeqFiles(); + + File currFile = null; + long startTime = System.currentTimeMillis(); + int mergedCnt = 0; + while ((currLine = bufferedReader.readLine()) != null) { + if (STR_MERGE_END.equals(currLine)) { + status = Status.MERGE_END; + break; + } + if (!currLine.contains(STR_END)) { + String[] splits = currLine.split(" "); + currFile = new File(splits[0]); + Long lastPost = Long.parseLong(splits[1]); + fileLastPositions.put(currFile, lastPost); + } else { + fileLastPositions.remove(currFile); + String seqFilePath = currFile.getAbsolutePath().replace(MergeTask.MERGE_SUFFIX, ""); + Iterator unmergedFileIter = unmergedFiles.iterator(); + while (unmergedFileIter.hasNext()) { + TsFileResource seqFile = unmergedFileIter.next(); + if (seqFile.getFile().getAbsolutePath().equals(seqFilePath)) { + mergedCnt ++; + unmergedFileIter.remove(); + break; + } + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("{} found {} files have already been merged after {}ms", taskName, + mergedCnt, (System.currentTimeMillis() - startTime)); + } + } + + public List getUnmergedPaths() { + return unmergedPaths; + } + + public void setUnmergedPaths(List unmergedPaths) { + this.unmergedPaths = unmergedPaths; + } + + public List getUnmergedFiles() { + return unmergedFiles; + } + + public void setUnmergedFiles( + List unmergedFiles) { + this.unmergedFiles = unmergedFiles; + } + + public List getMergedPaths() { + return mergedPaths; + } + + public void setMergedPaths(List mergedPaths) { + this.mergedPaths = mergedPaths; + } + + public Map getFileLastPositions() { + return fileLastPositions; + } + + public void setFileLastPositions(Map fileLastPositions) { + this.fileLastPositions = fileLastPositions; + } + + public enum Status { + // almost nothing has been done + NONE, + // at least the files and timeseries to be merged are known + MERGE_START, + // all the timeseries have been merged(merged chunks are generated) + ALL_TS_MERGED, + // all the merge files are merged with the origin files and the task is almost done + MERGE_END + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java new file mode 100644 index 0000000000000000000000000000000000000000..326647ab7735f2bc68af630c82c56a9c9faa0097 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.recover; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.tsfile.read.common.Path; + +/** + * MergeLogger records the progress of a merge in file "merge.log" as text lines. + */ +public class MergeLogger { + + public static final String MERGE_LOG_NAME = "merge.log"; + + static final String STR_SEQ_FILES = "seqFiles"; + static final String STR_UNSEQ_FILES = "unseqFiles"; + static final String STR_TIMESERIES = "timeseries"; + static final String STR_START = "start"; + static final String STR_END = "end"; + static final String STR_ALL_TS_END = "all ts end"; + static final String STR_MERGE_START = "merge start"; + static final String STR_MERGE_END = "merge end"; + + private BufferedWriter logStream; + + public MergeLogger(String storageGroupDir) throws IOException { + logStream = new BufferedWriter(new FileWriter(new File(storageGroupDir, MERGE_LOG_NAME), true)); + } + + public void close() throws IOException { + logStream.close(); + } + + public void logTSStart(List paths) throws IOException { + logStream.write(STR_START); + for (Path path : paths) { + logStream.write(" " + path.getFullPath()); + } + logStream.newLine(); + logStream.flush(); + } + + public void logFilePosition(File file) throws IOException { + logStream.write(String.format("%s %d", file.getAbsolutePath(), file.length())); + logStream.newLine(); + logStream.flush(); + } + + public void logTSEnd() throws IOException { + logStream.write(STR_END); + logStream.newLine(); + logStream.flush(); + } + + public void logAllTsEnd() throws IOException { + logStream.write(STR_ALL_TS_END); + logStream.newLine(); + logStream.flush(); + } + + public void logFileMergeStart(File file, long position) throws IOException { + logStream.write(String.format("%s %d", file.getAbsolutePath(), position)); + logStream.newLine(); + logStream.flush(); + } + + public void logFileMergeEnd() throws IOException { + logStream.write(STR_END); + logStream.newLine(); + logStream.flush(); + } + + public void logMergeEnd() throws IOException { + logStream.write(STR_MERGE_END); + logStream.newLine(); + logStream.flush(); + } + + public void logFiles(MergeResource resource) throws IOException { + logSeqFiles(resource.getSeqFiles()); + logUnseqFiles(resource.getUnseqFiles()); + } + + private void logSeqFiles(List seqFiles) throws IOException { + logStream.write(STR_SEQ_FILES); + logStream.newLine(); + for (TsFileResource tsFileResource : seqFiles) { + logStream.write(tsFileResource.getFile().getAbsolutePath()); + logStream.newLine(); + } + logStream.flush(); + } + + private void logUnseqFiles(List unseqFiles) throws IOException { + logStream.write(STR_UNSEQ_FILES); + logStream.newLine(); + for (TsFileResource tsFileResource : unseqFiles) { + logStream.write(tsFileResource.getFile().getAbsolutePath()); + logStream.newLine(); + } + logStream.flush(); + } + + public void logMergeStart() throws IOException { + logStream.write(STR_MERGE_START); + logStream.newLine(); + logStream.flush(); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IFileQueryMemMeasurement.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IFileQueryMemMeasurement.java new file mode 100644 index 0000000000000000000000000000000000000000..fde9be8d21b1db71bbc476d6089fb8cfb600c730 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IFileQueryMemMeasurement.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.selector; + +import java.io.IOException; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; + + +@FunctionalInterface +/** + * Estimate how much memory a file may occupy when being queried during merge. + * @param resource + * @return + * @throws IOException + */ +interface IFileQueryMemMeasurement { + long measure(TsFileResource resource) throws IOException; +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergeFileSelector.java new file mode 100644 index 0000000000000000000000000000000000000000..a1a11cf5f5b894aaa8ea39f080effb39685eb00a --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergeFileSelector.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.selector; + +import java.util.List; +import org.apache.iotdb.db.exception.MergeException; + +/** + * IMergeFileSelector selects a set of files from given seqFiles and unseqFiles which can be + * merged without exceeding given memory budget. + */ +public interface IMergeFileSelector { + + List[] select() throws MergeException; + + int getConcurrentMergeNum(); +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergePathSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergePathSelector.java new file mode 100644 index 0000000000000000000000000000000000000000..ed6979f186868fa8bd09ba2eadb26b82192b4045 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/IMergePathSelector.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.selector; + +import java.util.Iterator; +import java.util.List; +import org.apache.iotdb.tsfile.read.common.Path; + +/** + * IMergePathSelector select paths to be merged at a time if all paths of a device cannot be + * merged at the same time. + */ +public interface IMergePathSelector extends Iterator> { + +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java new file mode 100644 index 0000000000000000000000000000000000000000..bfa9f2f0b20ce1ee4f2b930ece3d3bc8d66758ac --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java @@ -0,0 +1,303 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.selector; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MergeException; +import org.apache.iotdb.db.utils.MergeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MaxFileMergeFileSelector selects the most files from given seqFiles and unseqFiles which can be + * merged without exceeding given memory budget. It always assume the number of timeseries being + * queried at the same time is 1 to maximize the number of file merged. + */ +public class MaxFileMergeFileSelector implements IMergeFileSelector { + + private static final Logger logger = LoggerFactory.getLogger(MaxFileMergeFileSelector.class); + private static final String LOG_FILE_COST = "Memory cost of file {} is {}"; + + MergeResource resource; + + long totalCost; + private long memoryBudget; + private long maxSeqFileCost; + + // the number of timeseries being queried at the same time + int concurrentMergeNum = 1; + + /** + * Total metadata size of each file. + */ + private Map fileMetaSizeMap = new HashMap<>(); + /** + * Maximum memory cost of querying a timeseries in each file. + */ + private Map maxSeriesQueryCostMap = new HashMap<>(); + + List selectedUnseqFiles; + List selectedSeqFiles; + + private Collection tmpSelectedSeqFiles; + private long tempMaxSeqFileCost; + + private boolean[] seqSelected; + private int seqSelectedNum; + + public MaxFileMergeFileSelector(MergeResource resource, long memoryBudget) { + this.resource = resource; + this.memoryBudget = memoryBudget; + } + + /** + * Select merge candidates from seqFiles and unseqFiles under the given memoryBudget. + * This process iteratively adds the next unseqFile from unseqFiles and its overlapping seqFiles + * as newly-added candidates and computes their estimated memory cost. If the current cost + * pluses the new cost is still under the budget, accept the unseqFile and the seqFiles as + * candidates, otherwise go to the next iteration. + * The memory cost of a file is calculated in two ways: + * The rough estimation: for a seqFile, the size of its metadata is used for estimation. + * Since in the worst case, the file only contains one timeseries and all its metadata will + * be loaded into memory with at most one actual data chunk (which is negligible) and writing + * the timeseries into a new file generate metadata of the similar size, so the size of all + * seqFiles' metadata (generated when writing new chunks) pluses the largest one (loaded + * when reading a timeseries from the seqFiles) is the total estimation of all seqFiles; for + * an unseqFile, since the merge reader may read all chunks of a series to perform a merge + * read, the whole file may be loaded into memory, so we use the file's length as the + * maximum estimation. + * The tight estimation: based on the rough estimation, we scan the file's metadata to + * count the number of chunks for each series, find the series which have the most + * chunks in the file and use its chunk proportion to refine the rough estimation. + * The rough estimation is performed first, if no candidates can be found using rough + * estimation, we run the selection again with tight estimation. + * @return two lists of TsFileResource, the former is selected seqFiles and the latter is + * selected unseqFiles or an empty array if there are no proper candidates by the budget. + * @throws MergeException + */ + @Override + public List[] select() throws MergeException { + long startTime = System.currentTimeMillis(); + try { + logger.info("Selecting merge candidates from {} seqFile, {} unseqFiles", + resource.getSeqFiles().size(), resource.getUnseqFiles().size()); + select(false); + if (selectedUnseqFiles.isEmpty()) { + select(true); + } + resource.setSeqFiles(selectedSeqFiles); + resource.setUnseqFiles(selectedUnseqFiles); + resource.removeOutdatedSeqReaders(); + if (selectedUnseqFiles.isEmpty()) { + logger.info("No merge candidates are found"); + return new List[0]; + } + } catch (IOException e) { + throw new MergeException(e); + } + if (logger.isInfoEnabled()) { + logger.info("Selected merge candidates, {} seqFiles, {} unseqFiles, total memory cost {}, " + + "time consumption {}ms", + selectedSeqFiles.size(), selectedUnseqFiles.size(), totalCost, + System.currentTimeMillis() - startTime); + } + return new List[]{selectedSeqFiles, selectedUnseqFiles}; + } + + void select(boolean useTightBound) throws IOException { + tmpSelectedSeqFiles = new HashSet<>(); + seqSelected = new boolean[resource.getSeqFiles().size()]; + seqSelectedNum = 0; + selectedSeqFiles = new ArrayList<>(); + selectedUnseqFiles = new ArrayList<>(); + maxSeqFileCost = 0; + tempMaxSeqFileCost = 0; + + totalCost = 0; + + int unseqIndex = 0; + long startTime = System.currentTimeMillis(); + long timeConsumption = 0; + long timeLimit = IoTDBDescriptor.getInstance().getConfig().getMergeFileSelectionTimeBudget(); + if (timeLimit < 0) { + timeLimit = Long.MAX_VALUE; + } + while (unseqIndex < resource.getUnseqFiles().size() && timeConsumption < timeLimit) { + // select next unseq files + TsFileResource unseqFile = resource.getUnseqFiles().get(unseqIndex); + + selectOverlappedSeqFiles(unseqFile); + + tempMaxSeqFileCost = maxSeqFileCost; + long newCost = useTightBound ? calculateTightMemoryCost(unseqFile, tmpSelectedSeqFiles, + startTime, timeLimit) : + calculateLooseMemoryCost(unseqFile, tmpSelectedSeqFiles, startTime, timeLimit); + + if (totalCost + newCost < memoryBudget) { + selectedUnseqFiles.add(unseqFile); + maxSeqFileCost = tempMaxSeqFileCost; + + for (Integer seqIdx : tmpSelectedSeqFiles) { + seqSelected[seqIdx] = true; + seqSelectedNum++; + selectedSeqFiles.add(resource.getSeqFiles().get(seqIdx)); + } + totalCost += newCost; + logger.debug("Adding a new unseqFile {} and seqFiles {} as candidates, new cost {}, total" + + " cost {}", + unseqFile, tmpSelectedSeqFiles, newCost, totalCost); + } + tmpSelectedSeqFiles.clear(); + unseqIndex++; + timeConsumption = System.currentTimeMillis() - startTime; + } + } + + private void selectOverlappedSeqFiles(TsFileResource unseqFile) { + if (seqSelectedNum == resource.getSeqFiles().size()) { + return; + } + int tmpSelectedNum = 0; + for (Entry deviceStartTimeEntry : unseqFile.getStartTimeMap().entrySet()) { + String deviceId = deviceStartTimeEntry.getKey(); + Long unseqStartTime = deviceStartTimeEntry.getValue(); + Long unseqEndTime = unseqFile.getEndTimeMap().get(deviceId); + + boolean noMoreOverlap = false; + for (int i = 0; i < resource.getSeqFiles().size() && !noMoreOverlap; i++) { + TsFileResource seqFile = resource.getSeqFiles().get(i); + if (seqSelected[i] || !seqFile.getEndTimeMap().containsKey(deviceId)) { + continue; + } + Long seqEndTime = seqFile.getEndTimeMap().get(deviceId); + if (unseqEndTime <= seqEndTime) { + // the unseqFile overlaps current seqFile + tmpSelectedSeqFiles.add(i); + tmpSelectedNum ++; + // the device of the unseqFile can not merge with later seqFiles + noMoreOverlap = true; + } else if (unseqStartTime <= seqEndTime) { + // the device of the unseqFile may merge with later seqFiles + // and the unseqFile overlaps current seqFile + tmpSelectedSeqFiles.add(i); + tmpSelectedNum++; + } + } + if (tmpSelectedNum + seqSelectedNum == resource.getSeqFiles().size()) { + break; + } + } + } + + private long calculateMemoryCost(TsFileResource tmpSelectedUnseqFile, + Collection tmpSelectedSeqFiles, IFileQueryMemMeasurement unseqMeasurement, + IFileQueryMemMeasurement seqMeasurement, long startTime, long timeLimit) throws IOException { + long cost = 0; + Long fileCost = unseqMeasurement.measure(tmpSelectedUnseqFile); + cost += fileCost; + + for (Integer seqFileIdx : tmpSelectedSeqFiles) { + TsFileResource seqFile = resource.getSeqFiles().get(seqFileIdx); + fileCost = seqMeasurement.measure(seqFile); + if (fileCost > tempMaxSeqFileCost) { + // only one file will be read at the same time, so only the largest one is recorded here + cost -= tempMaxSeqFileCost; + cost += fileCost; + tempMaxSeqFileCost = fileCost; + } + // but writing data into a new file may generate the same amount of metadata in memory + cost += calculateMetadataSize(seqFile); + long timeConsumption = System.currentTimeMillis() - startTime; + if (timeConsumption > timeLimit) { + return Long.MAX_VALUE; + } + } + return cost; + } + + private long calculateLooseMemoryCost(TsFileResource tmpSelectedUnseqFile, + Collection tmpSelectedSeqFiles, long startTime, long timeLimit) throws IOException { + return calculateMemoryCost(tmpSelectedUnseqFile, tmpSelectedSeqFiles, + TsFileResource::getFileSize, this::calculateMetadataSize, startTime, timeLimit); + } + + private long calculateTightMemoryCost(TsFileResource tmpSelectedUnseqFile, + Collection tmpSelectedSeqFiles, long startTime, long timeLimit) throws IOException { + return calculateMemoryCost(tmpSelectedUnseqFile, tmpSelectedSeqFiles, + this::calculateTightUnseqMemoryCost, this::calculateTightSeqMemoryCost, startTime, timeLimit); + } + + private long calculateMetadataSize(TsFileResource seqFile) throws IOException { + Long cost = fileMetaSizeMap.get(seqFile); + if (cost == null) { + cost = MergeUtils.getFileMetaSize(seqFile, resource.getFileReader(seqFile)); + fileMetaSizeMap.put(seqFile, cost); + logger.debug(LOG_FILE_COST, seqFile, cost); + } + return cost; + } + + private long calculateTightFileMemoryCost(TsFileResource seqFile, IFileQueryMemMeasurement measurement) + throws IOException { + Long cost = maxSeriesQueryCostMap.get(seqFile); + if (cost == null) { + long[] chunkNums = MergeUtils.findTotalAndLargestSeriesChunkNum(seqFile, resource.getFileReader(seqFile)); + long totalChunkNum = chunkNums[0]; + long maxChunkNum = chunkNums[1]; + cost = measurement.measure(seqFile) * maxChunkNum / totalChunkNum; + maxSeriesQueryCostMap.put(seqFile, cost); + logger.debug(LOG_FILE_COST, seqFile, cost); + } + return cost; + } + + // this method traverses all ChunkMetadata to find out which series has the most chunks and uses + // its proportion to all series to get a maximum estimation + private long calculateTightSeqMemoryCost(TsFileResource seqFile) throws IOException { + long singleSeriesCost = calculateTightFileMemoryCost(seqFile, this::calculateMetadataSize); + long multiSeriesCost = concurrentMergeNum * singleSeriesCost; + long maxCost = calculateMetadataSize(seqFile); + return multiSeriesCost > maxCost ? maxCost : multiSeriesCost; + } + + // this method traverses all ChunkMetadata to find out which series has the most chunks and uses + // its proportion among all series to get a maximum estimation + private long calculateTightUnseqMemoryCost(TsFileResource unseqFile) throws IOException { + long singleSeriesCost = calculateTightFileMemoryCost(unseqFile, TsFileResource::getFileSize); + long multiSeriesCost = concurrentMergeNum * singleSeriesCost; + long maxCost = unseqFile.getFileSize(); + return multiSeriesCost > maxCost ? maxCost : multiSeriesCost; + } + + @Override + public int getConcurrentMergeNum() { + return concurrentMergeNum; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxSeriesMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxSeriesMergeFileSelector.java new file mode 100644 index 0000000000000000000000000000000000000000..cd0186f4a4e0e1c87e314221678b6661724e1d0b --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxSeriesMergeFileSelector.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.selector; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MergeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MaxSeriesMergeFileSelector is an extension of IMergeFileSelector which tries to maximize the + * number of timeseries that can be merged at the same time. + */ +public class MaxSeriesMergeFileSelector extends MaxFileMergeFileSelector { + + public static final int MAX_SERIES_NUM = 1024; + private static final Logger logger = LoggerFactory.getLogger(MaxSeriesMergeFileSelector.class); + + private List lastSelectedSeqFiles = Collections.emptyList(); + private List lastSelectedUnseqFiles = Collections.emptyList(); + private long lastTotalMemoryCost; + + public MaxSeriesMergeFileSelector( + MergeResource mergeResource, + long memoryBudget) { + super(mergeResource, memoryBudget); + } + + @Override + public List[] select() throws MergeException { + long startTime = System.currentTimeMillis(); + try { + logger.info("Selecting merge candidates from {} seqFile, {} unseqFiles", resource.getSeqFiles().size(), + resource.getUnseqFiles().size()); + + searchMaxSeriesNum(); + resource.setSeqFiles(selectedSeqFiles); + resource.setUnseqFiles(selectedUnseqFiles); + resource.removeOutdatedSeqReaders(); + if (selectedUnseqFiles.isEmpty()) { + logger.info("No merge candidates are found"); + return new List[0]; + } + } catch (IOException e) { + throw new MergeException(e); + } + if (logger.isInfoEnabled()) { + logger.info("Selected merge candidates, {} seqFiles, {} unseqFiles, total memory cost {}, " + + "concurrent merge num {}" + "time consumption {}ms", + selectedSeqFiles.size(), selectedUnseqFiles.size(), totalCost, concurrentMergeNum, + System.currentTimeMillis() - startTime); + } + return new List[]{selectedSeqFiles, selectedUnseqFiles}; + } + + private void searchMaxSeriesNum() throws IOException { + binSearch(); + } + + private void binSearch() throws IOException { + int lb = 0; + int ub = MAX_SERIES_NUM + 1; + while (true) { + int mid = (lb + ub) / 2; + if (mid == lb) { + break; + } + concurrentMergeNum = mid; + select(false); + if (selectedUnseqFiles.isEmpty()) { + select(true); + } + if (selectedUnseqFiles.isEmpty()) { + ub = mid; + } else { + lastSelectedSeqFiles = selectedSeqFiles; + lastSelectedUnseqFiles = selectedUnseqFiles; + lastTotalMemoryCost = totalCost; + lb = mid; + } + } + selectedUnseqFiles = lastSelectedUnseqFiles; + selectedSeqFiles = lastSelectedSeqFiles; + concurrentMergeNum = lb; + totalCost = lastTotalMemoryCost; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..98026fb8446be342da7a13a80c404e9afd8324e9 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileStrategy.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.selector; + +public enum MergeFileStrategy { + MAX_SERIES_NUM, + MAX_FILE_NUM, + // TODO: HOW? + TRADE_OFF, +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/NaivePathSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/NaivePathSelector.java new file mode 100644 index 0000000000000000000000000000000000000000..6a0aaf7909da88997f3b1ae430049cd38657e86c --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/NaivePathSelector.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.selector; + +import java.util.List; +import java.util.NoSuchElementException; +import org.apache.iotdb.tsfile.read.common.Path; + +public class NaivePathSelector implements IMergePathSelector { + + private List paths; + private int idx; + private int maxSeriesNum; + + public NaivePathSelector(List paths, int maxSeriesNum) { + this.paths = paths; + this.maxSeriesNum = maxSeriesNum; + } + + @Override + public boolean hasNext() { + return idx < paths.size(); + } + + @Override + public List next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + List ret = idx + maxSeriesNum <= paths.size() ? paths.subList(idx, idx + maxSeriesNum) : + paths.subList(idx, paths.size()); + idx += maxSeriesNum; + return ret; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java new file mode 100644 index 0000000000000000000000000000000000000000..c66edeeada96e2f6b02177979f3d40641c107114 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.task; + +import java.io.File; +import java.util.List; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; + +@FunctionalInterface +public interface MergeCallback { + + /** + * On calling this method, the callee should: + * 1. replace the modification files of seqFiles with merging modifications since the old + * modifications have been merged into the new files. + * 2. remove the unseqFiles since they have been merged into new files. + * 3. remove the merge log file + * 4. exit merging status + * @param seqFiles + * @param unseqFiles + */ + void call(List seqFiles, List unseqFiles, File logFile); +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java new file mode 100644 index 0000000000000000000000000000000000000000..cf1f62d23ae580c1b5090046b2dc16b50519495a --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java @@ -0,0 +1,238 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.task; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.commons.io.FileUtils; +import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache; +import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache; +import org.apache.iotdb.db.engine.merge.manage.MergeContext; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.recover.MergeLogger; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException; +import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.write.schema.Schema; +import org.apache.iotdb.tsfile.write.writer.ForceAppendTsFileWriter; +import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; +import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MergeFileTask merges the merge temporary files with the seqFiles, either move the merged + * chunks in the temp files into the seqFiles or move the unmerged chunks into the merge temp + * files, depending on which one is the majority. + */ +class MergeFileTask { + + private static final Logger logger = LoggerFactory.getLogger(MergeFileTask.class); + + private String taskName; + private MergeContext context; + private MergeLogger mergeLogger; + private MergeResource resource; + private List unmergedFiles; + + MergeFileTask(String taskName, MergeContext context, MergeLogger mergeLogger, + MergeResource resource, List unmergedSeqFiles) { + this.taskName = taskName; + this.context = context; + this.mergeLogger = mergeLogger; + this.resource = resource; + this.unmergedFiles = unmergedSeqFiles; + } + + void mergeFiles() throws IOException { + // decide whether to write the unmerged chunks to the merge files or to move the merged chunks + // back to the origin seqFile's + if (logger.isInfoEnabled()) { + logger.info("{} starts to merge {} files", taskName, unmergedFiles.size()); + } + long startTime = System.currentTimeMillis(); + int cnt = 0; + for (TsFileResource seqFile : unmergedFiles) { + int mergedChunkNum = context.getMergedChunkCnt().getOrDefault(seqFile, 0); + int unmergedChunkNum = context.getUnmergedChunkCnt().getOrDefault(seqFile, 0); + if (mergedChunkNum >= unmergedChunkNum) { + // move the unmerged data to the new file + if (logger.isInfoEnabled()) { + logger.info("{} moving unmerged data of {} to the merged file, {} merged chunks, {} " + + "unmerged chunks", taskName, seqFile.getFile().getName(), mergedChunkNum, unmergedChunkNum); + } + moveUnmergedToNew(seqFile); + } else { + // move the merged data to the old file + if (logger.isInfoEnabled()) { + logger.info("{} moving merged data of {} to the old file {} merged chunks, {} " + + "unmerged chunks", taskName, seqFile.getFile().getName(), mergedChunkNum, unmergedChunkNum); + } + moveMergedToOld(seqFile); + } + cnt ++; + if (logger.isInfoEnabled()) { + logger.debug("{} has merged {}/{} files", taskName, cnt, unmergedFiles.size()); + } + } + if (logger.isInfoEnabled()) { + logger.info("{} has merged all files after {}ms", taskName, System.currentTimeMillis() - startTime); + } + mergeLogger.logMergeEnd(); + } + + private void moveMergedToOld(TsFileResource seqFile) throws IOException { + int mergedChunkNum = context.getMergedChunkCnt().getOrDefault(seqFile, 0); + if (mergedChunkNum == 0) { + resource.removeFileAndWriter(seqFile); + return; + } + + seqFile.getMergeQueryLock().writeLock().lock(); + try { + TsFileMetaDataCache.getInstance().remove(seqFile); + DeviceMetaDataCache.getInstance().remove(seqFile); + resource.removeFileReader(seqFile); + TsFileIOWriter oldFileWriter; + try { + oldFileWriter = new ForceAppendTsFileWriter(seqFile.getFile()); + mergeLogger.logFileMergeStart(seqFile.getFile(), ((ForceAppendTsFileWriter) oldFileWriter).getTruncatePosition()); + logger.debug("{} moving merged chunks of {} to the old file", taskName, seqFile); + ((ForceAppendTsFileWriter) oldFileWriter).doTruncate(); + } catch (TsFileNotCompleteException e) { + // this file may already be truncated if this merge is a system reboot merge + oldFileWriter = new RestorableTsFileIOWriter(seqFile.getFile()); + } + // filter the chunks that have been merged + oldFileWriter.filterChunks(context.getUnmergedChunkStartTimes().get(seqFile)); + + RestorableTsFileIOWriter newFileWriter = resource.getMergeFileWriter(seqFile); + newFileWriter.close(); + try (TsFileSequenceReader newFileReader = + new TsFileSequenceReader(newFileWriter.getFile().getPath())) { + List chunkGroupMetadataList = newFileWriter.getChunkGroupMetaDatas(); + if (logger.isDebugEnabled()) { + logger.debug("{} find {} merged chunk groups", taskName, chunkGroupMetadataList.size()); + } + for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetadataList) { + writeMergedChunkGroup(chunkGroupMetaData, newFileReader, oldFileWriter); + } + } + oldFileWriter.endFile(new Schema(newFileWriter.getKnownSchema())); + + seqFile.serialize(); + mergeLogger.logFileMergeEnd(); + logger.debug("{} moved merged chunks of {} to the old file", taskName, seqFile); + + newFileWriter.getFile().delete(); + } finally { + seqFile.getMergeQueryLock().writeLock().unlock(); + } + } + + private void writeMergedChunkGroup(ChunkGroupMetaData chunkGroupMetaData, + TsFileSequenceReader reader, TsFileIOWriter fileWriter) + throws IOException { + fileWriter.startChunkGroup(chunkGroupMetaData.getDeviceID()); + long version = chunkGroupMetaData.getVersion(); + for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) { + Chunk chunk = reader.readMemChunk(chunkMetaData); + fileWriter.writeChunk(chunk, chunkMetaData); + context.incTotalPointWritten(chunkMetaData.getNumOfPoints()); + } + fileWriter.endChunkGroup(version + 1); + } + + private void moveUnmergedToNew(TsFileResource seqFile) throws IOException { + Map> fileUnmergedChunkStartTimes = + context.getUnmergedChunkStartTimes().get(seqFile); + RestorableTsFileIOWriter fileWriter = resource.getMergeFileWriter(seqFile); + + mergeLogger.logFileMergeStart(fileWriter.getFile(), fileWriter.getFile().length()); + logger.debug("{} moving unmerged chunks of {} to the new file", taskName, seqFile); + + int unmergedChunkNum = context.getUnmergedChunkCnt().getOrDefault(seqFile, 0); + + if (unmergedChunkNum > 0) { + for (Entry> entry : fileUnmergedChunkStartTimes.entrySet()) { + Path path = entry.getKey(); + List chunkStartTimes = entry.getValue(); + if (chunkStartTimes.isEmpty()) { + continue; + } + + List chunkMetaDataList = resource.queryChunkMetadata(path, seqFile); + + if (logger.isDebugEnabled()) { + logger.debug("{} find {} unmerged chunks", taskName, chunkMetaDataList.size()); + } + + fileWriter.startChunkGroup(path.getDevice()); + long maxVersion = writeUnmergedChunks(chunkStartTimes, chunkMetaDataList, + resource.getFileReader(seqFile), fileWriter); + fileWriter.endChunkGroup(maxVersion + 1); + } + } + + fileWriter.endFile(new Schema(fileWriter.getKnownSchema())); + + seqFile.serialize(); + mergeLogger.logFileMergeEnd(); + logger.debug("{} moved unmerged chunks of {} to the new file", taskName, seqFile); + + seqFile.getMergeQueryLock().writeLock().lock(); + try { + resource.removeFileReader(seqFile); + TsFileMetaDataCache.getInstance().remove(seqFile); + DeviceMetaDataCache.getInstance().remove(seqFile); + seqFile.getFile().delete(); + FileUtils.moveFile(fileWriter.getFile(), seqFile.getFile()); + } finally { + seqFile.getMergeQueryLock().writeLock().unlock(); + } + } + + private long writeUnmergedChunks(List chunkStartTimes, + List chunkMetaDataList, TsFileSequenceReader reader, + RestorableTsFileIOWriter fileWriter) throws IOException { + long maxVersion = 0; + int chunkIdx = 0; + for (Long startTime : chunkStartTimes) { + for (; chunkIdx < chunkMetaDataList.size(); chunkIdx ++) { + ChunkMetaData metaData = chunkMetaDataList.get(chunkIdx); + if (metaData.getStartTime() == startTime) { + Chunk chunk = reader.readMemChunk(metaData); + fileWriter.writeChunk(chunk, metaData); + maxVersion = metaData.getVersion() > maxVersion ? metaData.getVersion() : maxVersion; + context.incTotalPointWritten(metaData.getNumOfPoints()); + break; + } + } + } + return maxVersion; + } + +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java new file mode 100644 index 0000000000000000000000000000000000000000..ce6e602f86195a231eea57b325d68c894376c86f --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeMultiChunkTask.java @@ -0,0 +1,440 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.task; + +import static org.apache.iotdb.db.utils.MergeUtils.writeBatchPoint; +import static org.apache.iotdb.db.utils.MergeUtils.writeTVPair; +import static org.apache.iotdb.db.utils.QueryUtils.modifyChunkMetaData; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.PriorityQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.merge.manage.MergeContext; +import org.apache.iotdb.db.engine.merge.manage.MergeManager; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.recover.MergeLogger; +import org.apache.iotdb.db.engine.merge.selector.IMergePathSelector; +import org.apache.iotdb.db.engine.merge.selector.NaivePathSelector; +import org.apache.iotdb.db.engine.modification.Modification; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.query.reader.IPointReader; +import org.apache.iotdb.db.utils.MergeUtils; +import org.apache.iotdb.db.utils.MergeUtils.MetaListEntry; +import org.apache.iotdb.db.utils.TimeValuePair; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; +import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter; +import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; +import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class MergeMultiChunkTask { + + private static final Logger logger = LoggerFactory.getLogger(MergeMultiChunkTask.class); + private static int minChunkPointNum = IoTDBDescriptor.getInstance().getConfig() + .getChunkMergePointThreshold(); + + private MergeLogger mergeLogger; + private List unmergedSeries; + + private String taskName; + private MergeResource resource; + private TimeValuePair[] currTimeValuePairs; + private boolean fullMerge; + + private MergeContext mergeContext; + + private AtomicInteger mergedChunkNum = new AtomicInteger(); + private AtomicInteger unmergedChunkNum = new AtomicInteger(); + private int mergedSeriesCnt; + private double progress; + + private int concurrentMergeSeriesNum; + private List currMergingPaths = new ArrayList<>(); + + MergeMultiChunkTask(MergeContext context, String taskName, MergeLogger mergeLogger, + MergeResource mergeResource, boolean fullMerge, List unmergedSeries, + int concurrentMergeSeriesNum) { + this.mergeContext = context; + this.taskName = taskName; + this.mergeLogger = mergeLogger; + this.resource = mergeResource; + this.fullMerge = fullMerge; + this.unmergedSeries = unmergedSeries; + this.concurrentMergeSeriesNum = concurrentMergeSeriesNum; + } + + void mergeSeries() throws IOException { + if (logger.isInfoEnabled()) { + logger.info("{} starts to merge {} series", taskName, unmergedSeries.size()); + } + long startTime = System.currentTimeMillis(); + for (TsFileResource seqFile : resource.getSeqFiles()) { + mergeContext.getUnmergedChunkStartTimes().put(seqFile, new HashMap<>()); + } + // merge each series and write data into each seqFile's corresponding temp merge file + List> devicePaths = MergeUtils.splitPathsByDevice(unmergedSeries); + for (List pathList : devicePaths) { + // TODO: use statistics of queries to better rearrange series + IMergePathSelector pathSelector = new NaivePathSelector(pathList, concurrentMergeSeriesNum); + while (pathSelector.hasNext()) { + currMergingPaths = pathSelector.next(); + mergePaths(); + mergedSeriesCnt += currMergingPaths.size(); + logMergeProgress(); + } + } + if (logger.isInfoEnabled()) { + logger.info("{} all series are merged after {}ms", taskName, + System.currentTimeMillis() - startTime); + } + mergeLogger.logAllTsEnd(); + } + + private void logMergeProgress() { + if (logger.isInfoEnabled()) { + double newProgress = 100 * mergedSeriesCnt / (double) (unmergedSeries.size()); + if (newProgress - progress >= 1.0) { + progress = newProgress; + logger.info("{} has merged {}% series", taskName, progress); + } + } + } + + private void mergePaths() throws IOException { + mergeLogger.logTSStart(currMergingPaths); + IPointReader[] unseqReaders; + unseqReaders = resource.getUnseqReaders(currMergingPaths); + currTimeValuePairs = new TimeValuePair[currMergingPaths.size()]; + for (int i = 0; i < currMergingPaths.size(); i++) { + if (unseqReaders[i].hasNext()) { + currTimeValuePairs[i] = unseqReaders[i].current(); + } + } + + for (int i = 0; i < resource.getSeqFiles().size(); i++) { + pathsMergeOneFile(i, unseqReaders); + } + mergeLogger.logTSEnd(); + } + + private void pathsMergeOneFile(int seqFileIdx, IPointReader[] unseqReaders) + throws IOException { + TsFileResource currTsFile = resource.getSeqFiles().get(seqFileIdx); + String deviceId = currMergingPaths.get(0).getDevice(); + Long currDeviceMinTime = currTsFile.getStartTimeMap().get(deviceId); + if (currDeviceMinTime == null) { + return; + } + + for (Path path : currMergingPaths) { + mergeContext.getUnmergedChunkStartTimes().get(currTsFile).put(path, new ArrayList<>()); + } + + // if this TsFile receives data later than fileLimitTime, it will overlap the next TsFile, + // which is forbidden + for (TimeValuePair timeValuePair : currTimeValuePairs) { + if (timeValuePair != null && timeValuePair.getTimestamp() < currDeviceMinTime) { + currDeviceMinTime = timeValuePair.getTimestamp(); + } + } + boolean isLastFile = seqFileIdx + 1 == resource.getSeqFiles().size(); + + TsFileSequenceReader fileSequenceReader = resource.getFileReader(currTsFile); + List[] modifications = new List[currMergingPaths.size()]; + List[] seqChunkMeta = new List[currMergingPaths.size()]; + for (int i = 0; i < currMergingPaths.size(); i++) { + modifications[i] = resource.getModifications(currTsFile, currMergingPaths.get(i)); + seqChunkMeta[i] = resource.queryChunkMetadata(currMergingPaths.get(i), currTsFile); + modifyChunkMetaData(seqChunkMeta[i], modifications[i]); + } + + List unskippedPathIndices = filterNoDataPaths(seqChunkMeta, seqFileIdx); + if (unskippedPathIndices.isEmpty()) { + return; + } + + RestorableTsFileIOWriter mergeFileWriter = resource.getMergeFileWriter(currTsFile); + for (Path path : currMergingPaths) { + MeasurementSchema schema = resource.getSchema(path.getMeasurement()); + mergeFileWriter.addSchema(schema); + } + // merge unseq data with seq data in this file or small chunks in this file into a larger chunk + mergeFileWriter.startChunkGroup(deviceId); + boolean dataWritten = mergeChunks(seqChunkMeta, isLastFile, fileSequenceReader, unseqReaders, + mergeFileWriter, currTsFile); + if (dataWritten) { + mergeFileWriter.endChunkGroup(0); + mergeLogger.logFilePosition(mergeFileWriter.getFile()); + currTsFile.getStartTimeMap().put(deviceId, currDeviceMinTime); + } + } + + private List filterNoDataPaths(List[] seqChunkMeta, int seqFileIdx) { + // if the last seqFile does not contains this series but the unseqFiles do, data of this + // series should also be written into a new chunk + List ret = new ArrayList<>(); + for (int i = 0; i < currMergingPaths.size(); i++) { + if (seqChunkMeta[i].isEmpty() + && !(seqFileIdx + 1 == resource.getSeqFiles().size() && currTimeValuePairs[i] != null)) { + continue; + } + ret.add(i); + } + return ret; + } + + private boolean mergeChunks(List[] seqChunkMeta, boolean isLastFile, + TsFileSequenceReader reader, IPointReader[] unseqReaders, + RestorableTsFileIOWriter mergeFileWriter, TsFileResource currFile) + throws IOException { + int[] ptWrittens = new int[seqChunkMeta.length]; + int mergeChunkSubTaskNum = IoTDBDescriptor.getInstance().getConfig().getMergeChunkSubThreadNum(); + PriorityQueue[] chunkMetaHeaps = new PriorityQueue[mergeChunkSubTaskNum]; + for (int i = 0; i < mergeChunkSubTaskNum; i++) { + chunkMetaHeaps[i] = new PriorityQueue<>(); + } + int idx = 0; + for (int i = 0; i < currMergingPaths.size(); i++) { + if (seqChunkMeta[i].isEmpty()) { + continue; + } + MetaListEntry entry = new MetaListEntry(i, seqChunkMeta[i]); + entry.next(); + + chunkMetaHeaps[idx % mergeChunkSubTaskNum].add(entry); + idx++; + ptWrittens[i] = 0; + } + + mergedChunkNum.set(0); + unmergedChunkNum.set(0); + + List futures = new ArrayList<>(); + for (int i = 0; i < mergeChunkSubTaskNum; i++) { + int finalI = i; + futures.add(MergeManager.getINSTANCE().submitChunkSubTask(() -> { + mergeChunkHeap(chunkMetaHeaps[finalI], ptWrittens, reader, mergeFileWriter, unseqReaders, currFile, + isLastFile); + return null; + })); + } + for (int i = 0; i < mergeChunkSubTaskNum; i++) { + try { + futures.get(i).get(); + } catch (InterruptedException | ExecutionException e) { + throw new IOException(e); + } + } + + // add merge and unmerged chunk statistic + mergeContext.getMergedChunkCnt().compute(currFile, (tsFileResource, anInt) -> anInt == null ? + mergedChunkNum.get() : anInt + mergedChunkNum.get()); + mergeContext.getUnmergedChunkCnt().compute(currFile, (tsFileResource, anInt) -> anInt == null ? + unmergedChunkNum.get() : anInt + unmergedChunkNum.get()); + + return mergedChunkNum.get() > 0; + } + + private void mergeChunkHeap(PriorityQueue chunkMetaHeap, int[] ptWrittens, TsFileSequenceReader reader, + RestorableTsFileIOWriter mergeFileWriter, IPointReader[] unseqReaders, TsFileResource currFile, + boolean isLastFile) throws IOException { + while (!chunkMetaHeap.isEmpty()) { + MetaListEntry metaListEntry = chunkMetaHeap.poll(); + ChunkMetaData currMeta = metaListEntry.current(); + int pathIdx = metaListEntry.getPathId(); + boolean isLastChunk = !metaListEntry.hasNext(); + Path path = currMergingPaths.get(pathIdx); + MeasurementSchema measurementSchema = resource.getSchema(path.getMeasurement()); + IChunkWriter chunkWriter = resource.getChunkWriter(measurementSchema); + + boolean chunkOverflowed = MergeUtils.isChunkOverflowed(currTimeValuePairs[pathIdx], currMeta); + boolean chunkTooSmall = MergeUtils + .isChunkTooSmall(ptWrittens[pathIdx], currMeta, isLastChunk, minChunkPointNum); + + Chunk chunk; + synchronized (reader) { + chunk = reader.readMemChunk(currMeta); + } + ptWrittens[pathIdx] = mergeChunkV2(currMeta, chunkOverflowed, chunkTooSmall, chunk, + ptWrittens[pathIdx], pathIdx, mergeFileWriter, unseqReaders[pathIdx], chunkWriter, + currFile); + + if (!isLastChunk) { + metaListEntry.next(); + chunkMetaHeap.add(metaListEntry); + } else { + // this only happens when the seqFiles do not contain this series, otherwise the remaining + // data will be merged with the last chunk in the seqFiles + if (isLastFile && currTimeValuePairs[pathIdx] != null) { + ptWrittens[pathIdx] += writeRemainingUnseq(chunkWriter, unseqReaders[pathIdx], Long.MAX_VALUE, + pathIdx); + mergedChunkNum.incrementAndGet(); + } + // the last merged chunk may still be smaller than the threshold, flush it anyway + if (ptWrittens[pathIdx] > 0) { + synchronized (mergeFileWriter) { + chunkWriter.writeToFileWriter(mergeFileWriter); + } + } + } + } + } + + /** + * merge a sequence chunk SK + * + * 1. no need to write the chunk to .merge file when: + * isn't full merge & + * there isn't unclosed chunk before & + * SK is big enough & + * SK isn't overflowed & + * SK isn't modified + * + * + * 2. write SK to .merge.file without compressing when: + * is full merge & + * there isn't unclosed chunk before & + * SK is big enough & + * SK isn't overflowed & + * SK isn't modified + * + * 3. other cases: need to unCompress the chunk and write + * 3.1 SK isn't overflowed + * 3.2 SK is overflowed + * + */ + private int mergeChunkV2(ChunkMetaData currMeta, boolean chunkOverflowed, + boolean chunkTooSmall,Chunk chunk, int lastUnclosedChunkPoint, int pathIdx, + TsFileIOWriter mergeFileWriter, IPointReader unseqReader, + IChunkWriter chunkWriter, TsFileResource currFile) throws IOException { + + int unclosedChunkPoint = lastUnclosedChunkPoint; + boolean chunkModified = currMeta.getDeletedAt() > Long.MIN_VALUE; + + // no need to write the chunk to .merge file + if (!fullMerge && lastUnclosedChunkPoint == 0 && !chunkTooSmall && !chunkOverflowed && !chunkModified) { + unmergedChunkNum.incrementAndGet(); + mergeContext.getUnmergedChunkStartTimes().get(currFile).get(currMergingPaths.get(pathIdx)) + .add(currMeta.getStartTime()); + return 0; + } + + // write SK to .merge.file without compressing + if (fullMerge && lastUnclosedChunkPoint == 0 && !chunkTooSmall && !chunkOverflowed && !chunkModified) { + synchronized (mergeFileWriter) { + mergeFileWriter.writeChunk(chunk, currMeta); + } + mergeContext.incTotalPointWritten(currMeta.getNumOfPoints()); + mergeContext.incTotalChunkWritten(); + mergedChunkNum.incrementAndGet(); + return 0; + } + + // 3.1 SK isn't overflowed, just uncompress and write sequence chunk + if (!chunkOverflowed) { + unclosedChunkPoint += MergeUtils.writeChunkWithoutUnseq(chunk, chunkWriter); + mergedChunkNum.incrementAndGet(); + } else { + // 3.2 SK is overflowed, uncompress sequence chunk and merge with unseq chunk, then write + unclosedChunkPoint += writeChunkWithUnseq(chunk, chunkWriter, unseqReader, + currMeta.getEndTime(), pathIdx); + mergedChunkNum.incrementAndGet(); + } + + // update points written statistics + mergeContext.incTotalPointWritten(unclosedChunkPoint - lastUnclosedChunkPoint); + if (minChunkPointNum > 0 && unclosedChunkPoint >= minChunkPointNum + || unclosedChunkPoint > 0 && minChunkPointNum < 0) { + // the new chunk's size is large enough and it should be flushed + synchronized (mergeFileWriter) { + chunkWriter.writeToFileWriter(mergeFileWriter); + } + unclosedChunkPoint = 0; + } + return unclosedChunkPoint; + } + + private int writeRemainingUnseq(IChunkWriter chunkWriter, + IPointReader unseqReader, long timeLimit, int pathIdx) throws IOException { + int ptWritten = 0; + while (currTimeValuePairs[pathIdx] != null + && currTimeValuePairs[pathIdx].getTimestamp() < timeLimit) { + writeTVPair(currTimeValuePairs[pathIdx], chunkWriter); + ptWritten++; + unseqReader.next(); + currTimeValuePairs[pathIdx] = unseqReader.hasNext() ? unseqReader.current() : null; + } + return ptWritten; + } + + private int writeChunkWithUnseq(Chunk chunk, IChunkWriter chunkWriter, IPointReader unseqReader, + long chunkLimitTime, int pathIdx) throws IOException { + int cnt = 0; + ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk); + while (chunkReader.hasNextBatch()) { + BatchData batchData = chunkReader.nextBatch(); + cnt += mergeWriteBatch(batchData, chunkWriter, unseqReader, pathIdx); + } + cnt += writeRemainingUnseq(chunkWriter, unseqReader, chunkLimitTime, pathIdx); + return cnt; + } + + private int mergeWriteBatch(BatchData batchData, IChunkWriter chunkWriter, + IPointReader unseqReader, int pathIdx) throws IOException { + int cnt = 0; + for (int i = 0; i < batchData.length(); i++) { + long time = batchData.getTimeByIndex(i); + // merge data in batch and data in unseqReader + + boolean overwriteSeqPoint = false; + // unseq point.time <= sequence point.time, write unseq point + while (currTimeValuePairs[pathIdx] != null + && currTimeValuePairs[pathIdx].getTimestamp() <= time) { + writeTVPair(currTimeValuePairs[pathIdx], chunkWriter); + if (currTimeValuePairs[pathIdx].getTimestamp() == time) { + overwriteSeqPoint = true; + } + unseqReader.next(); + currTimeValuePairs[pathIdx] = unseqReader.hasNext() ? unseqReader.current() : null; + cnt++; + } + // unseq point.time > sequence point.time, write seq point + if (!overwriteSeqPoint) { + writeBatchPoint(batchData, i, chunkWriter); + cnt++; + } + } + return cnt; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java new file mode 100644 index 0000000000000000000000000000000000000000..9e99747e8a64792e19007f3f78830374e05b5fc0 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.task; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import org.apache.iotdb.db.engine.merge.manage.MergeContext; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.recover.MergeLogger; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MetadataErrorException; +import org.apache.iotdb.db.metadata.MManager; +import org.apache.iotdb.db.utils.MergeUtils; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MergeTask merges given seqFiles and unseqFiles into new ones, which basically consists of three + * steps: 1. rewrite overflowed, modified or small-sized chunks into temp merge files + * 2. move the merged chunks in the temp files back to the seqFiles or move the unmerged + * chunks in the seqFiles into temp files and replace the seqFiles with the temp files. + * 3. remove unseqFiles + */ +public class MergeTask implements Callable { + + public static final String MERGE_SUFFIX = ".merge"; + private static final Logger logger = LoggerFactory.getLogger(MergeTask.class); + + MergeResource resource; + String storageGroupSysDir; + String storageGroupName; + MergeLogger mergeLogger; + MergeContext mergeContext = new MergeContext(); + + private MergeCallback callback; + int concurrentMergeSeriesNum; + String taskName; + boolean fullMerge; + + MergeTask(List seqFiles, + List unseqFiles, String storageGroupSysDir, MergeCallback callback, + String taskName, boolean fullMerge, String storageGroupName) { + this.resource = new MergeResource(seqFiles, unseqFiles); + this.storageGroupSysDir = storageGroupSysDir; + this.callback = callback; + this.taskName = taskName; + this.fullMerge = fullMerge; + this.concurrentMergeSeriesNum = 1; + this.storageGroupName = storageGroupName; + } + + public MergeTask(MergeResource mergeResource, String storageGroupSysDir, MergeCallback callback, + String taskName, boolean fullMerge, int concurrentMergeSeriesNum, String storageGroupName) { + this.resource = mergeResource; + this.storageGroupSysDir = storageGroupSysDir; + this.callback = callback; + this.taskName = taskName; + this.fullMerge = fullMerge; + this.concurrentMergeSeriesNum = concurrentMergeSeriesNum; + this.storageGroupName = storageGroupName; + } + + @Override + public Void call() throws Exception { + try { + doMerge(); + } catch (Exception e) { + logger.error("Runtime exception in merge {}", taskName, e); + cleanUp(false); + // call the callback to make sure the StorageGroup exit merging status, but passing 2 + // empty file lists to avoid files being deleted. + callback.call(Collections.emptyList(), Collections.emptyList(), new File(storageGroupSysDir, MergeLogger.MERGE_LOG_NAME)); + throw e; + } + return null; + } + + private void doMerge() throws IOException, MetadataErrorException { + if (logger.isInfoEnabled()) { + logger.info("{} starts to merge {} seqFiles, {} unseqFiles", taskName, + resource.getSeqFiles().size(), resource.getUnseqFiles().size()); + } + long startTime = System.currentTimeMillis(); + long totalFileSize = MergeUtils.collectFileSizes(resource.getSeqFiles(), + resource.getUnseqFiles()); + mergeLogger = new MergeLogger(storageGroupSysDir); + + mergeLogger.logFiles(resource); + + List measurementSchemas = MManager.getInstance() + .getSchemaForStorageGroup(storageGroupName); + resource.addMeasurements(measurementSchemas); + + List storageGroupPaths = MManager.getInstance().getPaths(storageGroupName + ".*"); + List unmergedSeries = new ArrayList<>(); + for (String path : storageGroupPaths) { + unmergedSeries.add(new Path(path)); + } + + mergeLogger.logMergeStart(); + + MergeMultiChunkTask mergeChunkTask = new MergeMultiChunkTask(mergeContext, taskName, mergeLogger, resource, + fullMerge, unmergedSeries, concurrentMergeSeriesNum); + mergeChunkTask.mergeSeries(); + + MergeFileTask mergeFileTask = new MergeFileTask(taskName, mergeContext, mergeLogger, resource, + resource.getSeqFiles()); + mergeFileTask.mergeFiles(); + + cleanUp(true); + if (logger.isInfoEnabled()) { + double elapsedTime = (double) (System.currentTimeMillis() - startTime) / 1000.0; + double byteRate = totalFileSize / elapsedTime / 1024 / 1024; + double seriesRate = unmergedSeries.size() / elapsedTime; + double chunkRate = mergeContext.getTotalChunkWritten() / elapsedTime; + double fileRate = + (resource.getSeqFiles().size() + resource.getUnseqFiles().size()) / elapsedTime; + double ptRate = mergeContext.getTotalPointWritten() / elapsedTime; + logger.info("{} ends after {}s, byteRate: {}MB/s, seriesRate {}/s, chunkRate: {}/s, " + + "fileRate: {}/s, ptRate: {}/s", + taskName, elapsedTime, byteRate, seriesRate, chunkRate, fileRate, ptRate); + } + } + + void cleanUp(boolean executeCallback) throws IOException { + logger.info("{} is cleaning up", taskName); + + resource.clear(); + mergeContext.clear(); + + if (mergeLogger != null) { + mergeLogger.close(); + } + + for (TsFileResource seqFile : resource.getSeqFiles()) { + File mergeFile = new File(seqFile.getFile().getPath() + MERGE_SUFFIX); + mergeFile.delete(); + } + + File logFile = new File(storageGroupSysDir, MergeLogger.MERGE_LOG_NAME); + if (executeCallback) { + // make sure merge.log is not deleted until unseqFiles are cleared so that when system + // reboots, the undeleted files can be deleted again + callback.call(resource.getSeqFiles(), resource.getUnseqFiles(), logFile); + } else { + logFile.delete(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java new file mode 100644 index 0000000000000000000000000000000000000000..a69571ad61ec0d1518f6459a592e73797d0a1707 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge.task; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.merge.recover.LogAnalyzer; +import org.apache.iotdb.db.engine.merge.recover.LogAnalyzer.Status; +import org.apache.iotdb.db.engine.merge.recover.MergeLogger; +import org.apache.iotdb.db.engine.merge.selector.MaxSeriesMergeFileSelector; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MetadataErrorException; +import org.apache.iotdb.db.utils.MergeUtils; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * RecoverMergeTask is an extension of MergeTask, which resumes the last merge progress by + * scanning merge.log using LogAnalyzer and continue the unfinished merge. + */ +public class RecoverMergeTask extends MergeTask { + + private static final Logger logger = LoggerFactory.getLogger(RecoverMergeTask.class); + + private LogAnalyzer analyzer; + + public RecoverMergeTask(List seqFiles, + List unseqFiles, String storageGroupSysDir, + MergeCallback callback, String taskName, + boolean fullMerge, String storageGroupName) { + super(seqFiles, unseqFiles, storageGroupSysDir, callback, taskName, fullMerge, storageGroupName); + } + + public void recoverMerge(boolean continueMerge) throws IOException, MetadataErrorException { + File logFile = new File(storageGroupSysDir, MergeLogger.MERGE_LOG_NAME); + if (!logFile.exists()) { + logger.info("{} no merge.log, merge recovery ends", taskName); + return; + } + long startTime = System.currentTimeMillis(); + + analyzer = new LogAnalyzer(resource, taskName, logFile, storageGroupName); + Status status = analyzer.analyze(); + if (logger.isInfoEnabled()) { + logger.info("{} merge recovery status determined: {} after {}ms", taskName, status, + (System.currentTimeMillis() - startTime)); + } + switch (status) { + case NONE: + logFile.delete(); + break; + case MERGE_START: + resumeAfterFilesLogged(continueMerge); + break; + case ALL_TS_MERGED: + resumeAfterAllTsMerged(continueMerge); + break; + case MERGE_END: + cleanUp(continueMerge); + break; + default: + throw new UnsupportedOperationException(taskName + " found unrecognized status " + status); + } + if (logger.isInfoEnabled()) { + logger.info("{} merge recovery ends after {}ms", taskName, + (System.currentTimeMillis() - startTime)); + } + } + + private void resumeAfterFilesLogged(boolean continueMerge) throws IOException { + if (continueMerge) { + resumeMergeProgress(); + calculateConcurrentSeriesNum(); + if (concurrentMergeSeriesNum == 0) { + throw new IOException("Merge cannot be resumed under current memory budget, please " + + "increase the budget or disable continueMergeAfterReboot"); + } + + MergeMultiChunkTask mergeChunkTask = new MergeMultiChunkTask(mergeContext, taskName, mergeLogger, resource, + fullMerge, analyzer.getUnmergedPaths(), concurrentMergeSeriesNum); + analyzer.setUnmergedPaths(null); + mergeChunkTask.mergeSeries(); + + MergeFileTask mergeFileTask = new MergeFileTask(taskName, mergeContext, mergeLogger, resource, + resource.getSeqFiles()); + mergeFileTask.mergeFiles(); + } + cleanUp(continueMerge); + } + + private void resumeAfterAllTsMerged(boolean continueMerge) throws IOException { + if (continueMerge) { + resumeMergeProgress(); + MergeFileTask mergeFileTask = new MergeFileTask(taskName, mergeContext, mergeLogger, resource, + analyzer.getUnmergedFiles()); + analyzer.setUnmergedFiles(null); + mergeFileTask.mergeFiles(); + } else { + // NOTICE: although some of the seqFiles may have been truncated in last merge, we do not + // recover them here because later TsFile recovery will recover them + truncateFiles(); + } + cleanUp(continueMerge); + } + + private void resumeMergeProgress() throws IOException { + mergeLogger = new MergeLogger(storageGroupSysDir); + truncateFiles(); + recoverChunkCounts(); + } + + private void calculateConcurrentSeriesNum() throws IOException { + long singleSeriesUnseqCost = 0; + long maxUnseqCost = 0; + for (TsFileResource unseqFile : resource.getUnseqFiles()) { + long[] chunkNums = MergeUtils.findTotalAndLargestSeriesChunkNum(unseqFile, + resource.getFileReader(unseqFile)); + long totalChunkNum = chunkNums[0]; + long maxChunkNum = chunkNums[1]; + singleSeriesUnseqCost += unseqFile.getFileSize() * maxChunkNum / totalChunkNum; + maxUnseqCost += unseqFile.getFileSize(); + } + + long singleSeriesSeqReadCost = 0; + long maxSeqReadCost = 0; + long seqWriteCost = 0; + for (TsFileResource seqFile : resource.getSeqFiles()) { + long[] chunkNums = MergeUtils.findTotalAndLargestSeriesChunkNum(seqFile, + resource.getFileReader(seqFile)); + long totalChunkNum = chunkNums[0]; + long maxChunkNum = chunkNums[1]; + long fileMetaSize = MergeUtils.getFileMetaSize(seqFile, resource.getFileReader(seqFile)); + long newSingleSeriesSeqReadCost = fileMetaSize * maxChunkNum / totalChunkNum; + singleSeriesSeqReadCost = newSingleSeriesSeqReadCost > singleSeriesSeqReadCost ? + newSingleSeriesSeqReadCost : singleSeriesSeqReadCost; + maxSeqReadCost = fileMetaSize > maxSeqReadCost ? fileMetaSize : maxSeqReadCost; + seqWriteCost += fileMetaSize; + } + + long memBudget = IoTDBDescriptor.getInstance().getConfig().getMergeMemoryBudget(); + int lb = 0; + int ub = MaxSeriesMergeFileSelector.MAX_SERIES_NUM; + int mid = (lb + ub) / 2; + while (mid != lb) { + long unseqCost = singleSeriesUnseqCost * mid < maxUnseqCost ? singleSeriesUnseqCost * mid : + maxUnseqCost; + long seqReadCos = singleSeriesSeqReadCost * mid < maxSeqReadCost ? + singleSeriesSeqReadCost * mid : maxSeqReadCost; + long totalCost = unseqCost + seqReadCos + seqWriteCost; + if (totalCost <= memBudget) { + lb = mid; + } else { + ub = mid; + } + mid = (lb + ub) / 2; + } + concurrentMergeSeriesNum = lb; + } + + // scan the metadata to compute how many chunks are merged/unmerged so at last we can decide to + // move the merged chunks or the unmerged chunks + private void recoverChunkCounts() throws IOException { + logger.info("{} recovering chunk counts", taskName); + int fileCnt = 1; + for (TsFileResource tsFileResource : resource.getSeqFiles()) { + logger.info("{} recovering {} {}/{}", taskName, tsFileResource.getFile().getName(), + fileCnt, resource.getSeqFiles().size()); + RestorableTsFileIOWriter mergeFileWriter = resource.getMergeFileWriter(tsFileResource); + mergeFileWriter.makeMetadataVisible(); + mergeContext.getUnmergedChunkStartTimes().put(tsFileResource, new HashMap<>()); + List pathsToRecover = analyzer.getMergedPaths(); + int cnt = 0; + double progress = 0.0; + for(Path path : pathsToRecover) { + recoverChunkCounts(path, tsFileResource, mergeFileWriter); + if (logger.isInfoEnabled()) { + cnt += 1.0; + double newProgress = 100.0 * cnt / pathsToRecover.size(); + if (newProgress - progress >= 1.0) { + progress = newProgress; + logger.info("{} {}% series count of {} are recovered", taskName, progress, + tsFileResource.getFile().getName()); + } + } + } + fileCnt++; + } + analyzer.setMergedPaths(null); + } + + private void recoverChunkCounts(Path path, TsFileResource tsFileResource, + RestorableTsFileIOWriter mergeFileWriter) throws IOException { + mergeContext.getUnmergedChunkStartTimes().get(tsFileResource).put(path, new ArrayList<>()); + + List seqFileChunks = resource.queryChunkMetadata(path, tsFileResource); + List mergeFileChunks = + mergeFileWriter.getVisibleMetadataList(path.getDevice(), path.getMeasurement(), null); + mergeContext.getMergedChunkCnt().compute(tsFileResource, (k, v) -> v == null ? + mergeFileChunks.size() : v + mergeFileChunks.size()); + int seqChunkIndex = 0; + int mergeChunkIndex = 0; + int unmergedCnt = 0; + while (seqChunkIndex < seqFileChunks.size() && mergeChunkIndex < mergeFileChunks.size()) { + ChunkMetaData seqChunk = seqFileChunks.get(seqChunkIndex); + ChunkMetaData mergedChunk = mergeFileChunks.get(mergeChunkIndex); + if (seqChunk.getStartTime() < mergedChunk.getStartTime()) { + // this seqChunk is unmerged + unmergedCnt ++; + seqChunkIndex ++; + mergeContext.getUnmergedChunkStartTimes().get(tsFileResource).get(path).add(seqChunk.getStartTime()); + } else if (mergedChunk.getStartTime() <= seqChunk.getStartTime() && + seqChunk.getStartTime() <= mergedChunk.getEndTime()) { + // this seqChunk is merged + seqChunkIndex ++; + } else { + // seqChunk.startTime > mergeChunk.endTime, find next mergedChunk that may cover the + // seqChunk + mergeChunkIndex ++; + } + } + int finalUnmergedCnt = unmergedCnt; + mergeContext.getUnmergedChunkCnt().compute(tsFileResource, (k, v) -> v == null ? + finalUnmergedCnt : v + finalUnmergedCnt); + } + + private void truncateFiles() throws IOException { + logger.info("{} truncating {} files", taskName, analyzer.getFileLastPositions().size()); + for (Entry entry : analyzer.getFileLastPositions().entrySet()) { + File file = entry.getKey(); + Long lastPosition = entry.getValue(); + if (file.exists() && file.length() != lastPosition) { + try (FileInputStream fileInputStream = new FileInputStream(file)) { + FileChannel channel = fileInputStream.getChannel(); + channel.truncate(lastPosition); + channel.close(); + } + } + } + analyzer.setFileLastPositions(null); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java b/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java index 964d89e52b975c11cdcad036e55504c16efc551f..26438a85b30e1339e708db7333951d50402f7c15 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/ModificationFile.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.engine.modification; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -117,4 +118,10 @@ public class ModificationFile { public void setFilePath(String filePath) { this.filePath = filePath; } + + public void remove() throws IOException { + close(); + new File(filePath).delete(); + } + } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index 6661736168d923f3b556a6779dbe8e69fce7e683..1e41c57db7816d11800f751e7cc84ba30ef2cb01 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.db.engine.storagegroup; +import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX; +import static org.apache.iotdb.db.engine.storagegroup.TsFileResource.TEMP_SUFFIX; import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; import java.io.File; @@ -34,25 +36,38 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.io.FileUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.conf.directories.DirectoryManager; +import org.apache.iotdb.db.engine.merge.manage.MergeManager; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.selector.IMergeFileSelector; +import org.apache.iotdb.db.engine.merge.selector.MaxFileMergeFileSelector; +import org.apache.iotdb.db.engine.merge.selector.MaxSeriesMergeFileSelector; +import org.apache.iotdb.db.engine.merge.selector.MergeFileStrategy; +import org.apache.iotdb.db.engine.merge.task.MergeTask; +import org.apache.iotdb.db.engine.merge.task.RecoverMergeTask; import org.apache.iotdb.db.engine.modification.Deletion; +import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.version.SimpleFileVersionController; import org.apache.iotdb.db.engine.version.VersionController; import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; +import org.apache.iotdb.db.exception.MergeException; +import org.apache.iotdb.db.exception.MetadataErrorException; import org.apache.iotdb.db.exception.ProcessorException; import org.apache.iotdb.db.exception.StorageGroupProcessorException; import org.apache.iotdb.db.exception.TsFileProcessorException; import org.apache.iotdb.db.metadata.MManager; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.control.JobFileManager; +import org.apache.iotdb.rpc.TSStatusType; import org.apache.iotdb.db.utils.CopyOnReadLinkedList; import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; @@ -61,7 +76,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.Pair; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,10 +103,11 @@ import org.slf4j.LoggerFactory; */ public class StorageGroupProcessor { + private static final String MERGING_MODIFICAITON_FILE_NAME = "merge.mods"; private static final Logger logger = LoggerFactory.getLogger(StorageGroupProcessor.class); /** * a read write lock for guaranteeing concurrent safety when accessing all fields in this class - * (i.e., fileSchema, (un)sequenceFileList, work(un)SequenceTsFileProcessor, + * (i.e., schema, (un)sequenceFileList, work(un)SequenceTsFileProcessor, * closing(Un)SequenceTsFileProcessor, latestTimeForEachDevice, and * latestFlushedTimeForEachDevice) */ @@ -107,7 +123,7 @@ public class StorageGroupProcessor { /** * the schema of time series that belong this storage group */ - private FileSchema fileSchema; + private Schema schema; // includes sealed and unsealed sequence TsFiles private List sequenceFileList = new ArrayList<>(); private TsFileProcessor workSequenceTsFileProcessor = null; @@ -130,6 +146,8 @@ public class StorageGroupProcessor { */ private Map latestFlushedTimeForEachDevice = new HashMap<>(); private String storageGroupName; + private File storageGroupSysDir; + /** * versionController assigns a version for each MemTable and deletion/update such that after they * are persisted, the order of insertions, deletions and updates can be re-determined. @@ -137,12 +155,10 @@ public class StorageGroupProcessor { private VersionController versionController; /** - * mergeDeleteLock is to be used in the merge process. Concurrent deletion and merge may result in - * losing some deletion in the merged new file, so a lock is necessary. TODO reconsidering this - * when implementing the merge process. + * mergeLock is to be used in the merge process. Concurrent queries, deletions and merges may + * result in losing some deletion in the merged new file, so a lock is necessary. */ - @SuppressWarnings("unused") // to be used in merge - private ReentrantLock mergeDeleteLock = new ReentrantLock(); + private ReentrantReadWriteLock mergeLock = new ReentrantReadWriteLock(); /** * This is the modification file of the result of the current merge. Because the merged file may @@ -150,6 +166,9 @@ public class StorageGroupProcessor { */ private ModificationFile mergingModification; + private volatile boolean isMerging = false; + private long mergeStartTime; + /** * This linked list records the access order of measurements used by query. */ @@ -162,15 +181,15 @@ public class StorageGroupProcessor { this.storageGroupName = storageGroupName; // construct the file schema - this.fileSchema = constructFileSchema(storageGroupName); + this.schema = constructSchema(storageGroupName); try { - File storageGroupSysDir = new File(systemInfoDir, storageGroupName); + storageGroupSysDir = new File(systemInfoDir, storageGroupName); if (storageGroupSysDir.mkdirs()) { logger.info("Storage Group system Directory {} doesn't exist, create it", storageGroupSysDir.getPath()); } else if (!storageGroupSysDir.exists()) { - logger.error("craete Storage Group system Directory {} failed", + logger.error("create Storage Group system Directory {} failed", storageGroupSysDir.getPath()); } @@ -185,13 +204,31 @@ public class StorageGroupProcessor { private void recover() throws ProcessorException { logger.info("recover Storage Group {}", storageGroupName); - // collect TsFiles from sequential data directory - List tsFiles = getAllFiles(DirectoryManager.getInstance().getAllSequenceFileFolders()); - recoverSeqFiles(tsFiles); - - // collect TsFiles from unsequential data directory - tsFiles = getAllFiles(DirectoryManager.getInstance().getAllUnSequenceFileFolders()); - recoverUnseqFiles(tsFiles); + try { + // collect TsFiles from sequential and unsequential data directory + List seqTsFiles = getAllFiles(DirectoryManager.getInstance().getAllSequenceFileFolders()); + List unseqTsFiles = + getAllFiles(DirectoryManager.getInstance().getAllUnSequenceFileFolders()); + + recoverSeqFiles(seqTsFiles); + recoverUnseqFiles(unseqTsFiles); + + String taskName = storageGroupName + "-" + System.currentTimeMillis(); + File mergingMods = new File(storageGroupSysDir, MERGING_MODIFICAITON_FILE_NAME); + if (mergingMods.exists()) { + mergingModification = new ModificationFile(mergingMods.getPath()); + } + RecoverMergeTask recoverMergeTask = new RecoverMergeTask(seqTsFiles, unseqTsFiles, + storageGroupSysDir.getPath(), this::mergeEndAction, taskName, + IoTDBDescriptor.getInstance().getConfig().isForceFullMerge(), storageGroupName); + logger.info("{} a RecoverMergeTask {} starts...", storageGroupName, taskName); + recoverMergeTask.recoverMerge(IoTDBDescriptor.getInstance().getConfig().isContinueMergeAfterReboot()); + if (!IoTDBDescriptor.getInstance().getConfig().isContinueMergeAfterReboot()) { + mergingMods.delete(); + } + } catch (IOException | MetadataErrorException e) { + throw new ProcessorException(e); + } for (TsFileResource resource : sequenceFileList) { latestTimeForEachDevice.putAll(resource.getEndTimeMap()); @@ -199,39 +236,63 @@ public class StorageGroupProcessor { } } - private List getAllFiles(List folders) { + private List getAllFiles(List folders) throws IOException { List tsFiles = new ArrayList<>(); for (String baseDir : folders) { File fileFolder = new File(baseDir, storageGroupName); if (!fileFolder.exists()) { continue; } + // some TsFileResource may be being persisted when the system crashed, try recovering such + // resources + continueFailedRenames(fileFolder, TEMP_SUFFIX); + + // some TsFiles were going to be replaced by the merged files when the system crashed and + // the process was interrupted before the merged files could be named + continueFailedRenames(fileFolder, MERGE_SUFFIX); + Collections .addAll(tsFiles, fileFolder.listFiles(file -> file.getName().endsWith(TSFILE_SUFFIX))); } - return tsFiles; + tsFiles.sort(this::compareFileName); + List ret = new ArrayList<>(); + tsFiles.forEach(f -> ret.add(new TsFileResource(f))); + return ret; } - private void recoverSeqFiles(List tsFiles) throws ProcessorException { - tsFiles.sort(this::compareFileName); - for (File tsFile : tsFiles) { - TsFileResource tsFileResource = new TsFileResource(tsFile); + private void continueFailedRenames(File fileFolder, String suffix) { + File[] files = fileFolder.listFiles(file -> file.getName().endsWith(suffix)); + if (files != null) { + for (File tempResource : files) { + File originResource = new File(tempResource.getPath().replace(suffix, "")); + if (originResource.exists()) { + tempResource.delete(); + } else { + tempResource.renameTo(originResource); + } + } + } + } + + private void recoverSeqFiles(List tsFiles) throws ProcessorException { + + for (TsFileResource tsFileResource : tsFiles) { sequenceFileList.add(tsFileResource); TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(storageGroupName + "-" - , fileSchema, versionController, tsFileResource, false); + , schema, versionController, tsFileResource, false); recoverPerformer.recover(); + tsFileResource.setClosed(true); } } - private void recoverUnseqFiles(List tsFiles) throws ProcessorException { - tsFiles.sort(this::compareFileName); - for (File tsFile : tsFiles) { - TsFileResource tsFileResource = new TsFileResource(tsFile); + private void recoverUnseqFiles(List tsFiles) throws ProcessorException { + for (TsFileResource tsFileResource : tsFiles) { unSequenceFileList.add(tsFileResource); TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(storageGroupName + "-", - fileSchema, + schema, versionController, tsFileResource, true); recoverPerformer.recover(); + tsFileResource.setClosed(true); } } @@ -247,11 +308,11 @@ public class StorageGroupProcessor { } } - private FileSchema constructFileSchema(String storageGroupName) { + private Schema constructSchema(String storageGroupName) { List columnSchemaList; columnSchemaList = MManager.getInstance().getSchemaForStorageGroup(storageGroupName); - FileSchema schema = new FileSchema(); + Schema schema = new Schema(); for (MeasurementSchema measurementSchema : columnSchemaList) { schema.registerMeasurement(measurementSchema); } @@ -260,13 +321,13 @@ public class StorageGroupProcessor { /** - * add a measurement into the fileSchema. + * add a measurement into the schema. */ public void addMeasurement(String measurementId, TSDataType dataType, TSEncoding encoding, CompressionType compressor, Map props) { writeLock(); try { - fileSchema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding, + schema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding, compressor, props)); } finally { writeUnlock(); @@ -283,40 +344,84 @@ public class StorageGroupProcessor { // insert to sequence or unSequence file return insertToTsFileProcessor(insertPlan, insertPlan.getTime() > latestFlushedTimeForEachDevice.get(insertPlan.getDeviceId())); - } catch (IOException e) { - logger.error("insert tsRecord to unsealed data file failed, because {}", e.getMessage(), e); - return false; } finally { writeUnlock(); } } - private boolean insertToTsFileProcessor(InsertPlan insertPlan, boolean sequence) - throws IOException { - TsFileProcessor tsFileProcessor; - boolean result; - + public Integer[] insertBatch(BatchInsertPlan batchInsertPlan) { + writeLock(); try { - if (sequence) { - if (workSequenceTsFileProcessor == null) { - // create a new TsfileProcessor - workSequenceTsFileProcessor = createTsFileProcessor(true); - sequenceFileList.add(workSequenceTsFileProcessor.getTsFileResource()); + // init map + latestTimeForEachDevice.putIfAbsent(batchInsertPlan.getDeviceId(), Long.MIN_VALUE); + latestFlushedTimeForEachDevice.putIfAbsent(batchInsertPlan.getDeviceId(), Long.MIN_VALUE); + + Integer[] results = new Integer[batchInsertPlan.getRowCount()]; + List sequenceIndexes = new ArrayList<>(); + List unsequenceIndexes = new ArrayList<>(); + + for (int i = 0; i < batchInsertPlan.getRowCount(); i++) { + results[i] = TSStatusType.SUCCESS_STATUS.getStatusCode(); + if (batchInsertPlan.getTimes()[i] > latestFlushedTimeForEachDevice + .get(batchInsertPlan.getDeviceId())) { + sequenceIndexes.add(i); + } else { + unsequenceIndexes.add(i); } - tsFileProcessor = workSequenceTsFileProcessor; + } + + if (!sequenceIndexes.isEmpty()) { + insertBatchToTsFileProcessor(batchInsertPlan, sequenceIndexes, true, results); + } + + if (!unsequenceIndexes.isEmpty()) { + insertBatchToTsFileProcessor(batchInsertPlan, unsequenceIndexes, false, results); + } + return results; + } finally { + writeUnlock(); + } + } + + private void insertBatchToTsFileProcessor(BatchInsertPlan batchInsertPlan, + List indexes, boolean sequence, Integer[] results) { + + TsFileProcessor tsFileProcessor = getOrCreateTsFileProcessor(sequence); + if (tsFileProcessor == null) { + for (int index : indexes) { + results[index] = TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(); + } + return; + } + + boolean result = tsFileProcessor.insertBatch(batchInsertPlan, indexes, results); + + // try to update the latest time of the device of this tsRecord + if (result && latestTimeForEachDevice.get(batchInsertPlan.getDeviceId()) < batchInsertPlan.getMaxTime()) { + latestTimeForEachDevice.put(batchInsertPlan.getDeviceId(), batchInsertPlan.getMaxTime()); + } + + // check memtable size and may asyncTryToFlush the work memtable + if (tsFileProcessor.shouldFlush()) { + logger.info("The memtable size {} reaches the threshold, async flush it to tsfile: {}", + tsFileProcessor.getWorkMemTableMemory(), + tsFileProcessor.getTsFileResource().getFile().getAbsolutePath()); + + if (tsFileProcessor.shouldClose()) { + moveOneWorkProcessorToClosingList(sequence); } else { - if (workUnSequenceTsFileProcessor == null) { - // create a new TsfileProcessor - workUnSequenceTsFileProcessor = createTsFileProcessor(false); - unSequenceFileList.add(workUnSequenceTsFileProcessor.getTsFileResource()); - } - tsFileProcessor = workUnSequenceTsFileProcessor; + tsFileProcessor.asyncFlush(); } - } catch (DiskSpaceInsufficientException e) { - logger.error( - "disk space is insufficient when creating TsFile processor, change system mode to read-only", - e); - IoTDBDescriptor.getInstance().getConfig().setReadOnly(true); + } + } + + private boolean insertToTsFileProcessor(InsertPlan insertPlan, boolean sequence) { + TsFileProcessor tsFileProcessor; + boolean result; + + tsFileProcessor = getOrCreateTsFileProcessor(sequence); + + if (tsFileProcessor == null) { return false; } @@ -343,6 +448,37 @@ public class StorageGroupProcessor { return result; } + private TsFileProcessor getOrCreateTsFileProcessor(boolean sequence) { + TsFileProcessor tsFileProcessor = null; + try { + if (sequence) { + if (workSequenceTsFileProcessor == null) { + // create a new TsfileProcessor + workSequenceTsFileProcessor = createTsFileProcessor(true); + sequenceFileList.add(workSequenceTsFileProcessor.getTsFileResource()); + } + tsFileProcessor = workSequenceTsFileProcessor; + } else { + if (workUnSequenceTsFileProcessor == null) { + // create a new TsfileProcessor + workUnSequenceTsFileProcessor = createTsFileProcessor(false); + unSequenceFileList.add(workUnSequenceTsFileProcessor.getTsFileResource()); + } + tsFileProcessor = workUnSequenceTsFileProcessor; + } + } catch (DiskSpaceInsufficientException e) { + logger.error( + "disk space is insufficient when creating TsFile processor, change system mode to read-only", + e); + IoTDBDescriptor.getInstance().getConfig().setReadOnly(true); + } catch (IOException e) { + logger.error("meet IOException when creating TsFileProcessor, change system mode to read-only", + e); + IoTDBDescriptor.getInstance().getConfig().setReadOnly(true); + } + return tsFileProcessor; + } + private TsFileProcessor createTsFileProcessor(boolean sequence) throws IOException, DiskSpaceInsufficientException { String baseDir; @@ -359,15 +495,16 @@ public class StorageGroupProcessor { if (sequence) { return new TsFileProcessor(storageGroupName, new File(filePath), - fileSchema, versionController, this::closeUnsealedTsFileProcessor, + schema, versionController, this::closeUnsealedTsFileProcessor, this::updateLatestFlushTimeCallback, sequence); } else { return new TsFileProcessor(storageGroupName, new File(filePath), - fileSchema, versionController, this::closeUnsealedTsFileProcessor, + schema, versionController, this::closeUnsealedTsFileProcessor, () -> true, sequence); } } + /** * only called by insert(), thread-safety should be ensured by caller */ @@ -456,8 +593,10 @@ public class StorageGroupProcessor { } // TODO need a read lock, please consider the concurrency with flush manager threads. - public QueryDataSource query(String deviceId, String measurementId, QueryContext context) { + public QueryDataSource query(String deviceId, String measurementId, QueryContext context, + JobFileManager filePathsManager) { insertLock.readLock().lock(); + mergeLock.readLock().lock(); synchronized (lruForSensorUsedInQuery) { if (lruForSensorUsedInQuery.size() >= MAX_CACHE_SENSORS) { lruForSensorUsedInQuery.removeFirst(); @@ -469,9 +608,17 @@ public class StorageGroupProcessor { deviceId, measurementId, context); List unseqResources = getFileReSourceListForQuery(unSequenceFileList, deviceId, measurementId, context); - return new QueryDataSource(new Path(deviceId, measurementId), seqResources, unseqResources); + QueryDataSource dataSource = new QueryDataSource(new Path(deviceId, measurementId), seqResources, unseqResources); + // used files should be added before mergeLock is unlocked, or they may be deleted by + // running merge + // is null only in tests + if (filePathsManager != null) { + filePathsManager.addUsedFilesForGivenJob(context.getJobId(), dataSource); + } + return dataSource; } finally { insertLock.readLock().unlock(); + mergeLock.readLock().unlock(); } } @@ -512,7 +659,7 @@ public class StorageGroupProcessor { private List getFileReSourceListForQuery(List tsFileResources, String deviceId, String measurementId, QueryContext context) { - MeasurementSchema mSchema = fileSchema.getMeasurementSchema(measurementId); + MeasurementSchema mSchema = schema.getMeasurementSchema(measurementId); TSDataType dataType = mSchema.getType(); List tsfileResourcesForQuery = new ArrayList<>(); @@ -557,6 +704,7 @@ public class StorageGroupProcessor { public void delete(String deviceId, String measurementId, long timestamp) throws IOException { // TODO: how to avoid partial deletion? writeLock(); + mergeLock.writeLock().lock(); // record files which are updated so that we can roll back them in case of exception List updatedModFiles = new ArrayList<>(); @@ -584,7 +732,6 @@ public class StorageGroupProcessor { Path fullPath = new Path(deviceId, measurementId); Deletion deletion = new Deletion(fullPath, versionController.nextVersion(), timestamp); if (mergingModification != null) { - //TODO check me when implementing the merge process. mergingModification.write(deletion); updatedModFiles.add(mergingModification); } @@ -600,6 +747,7 @@ public class StorageGroupProcessor { throw new IOException(e); } finally { writeUnlock(); + mergeLock.writeLock().unlock(); } } @@ -674,6 +822,134 @@ public class StorageGroupProcessor { } } + public void merge(boolean fullMerge) { + writeLock(); + try { + if (isMerging) { + if (logger.isInfoEnabled()) { + logger.info("{} Last merge is ongoing, currently consumed time: {}ms", storageGroupName, + (System.currentTimeMillis() - mergeStartTime)); + } + return; + } + if (unSequenceFileList.isEmpty() || sequenceFileList.isEmpty()) { + logger.info("{} no files to be merged", storageGroupName); + return; + } + + long budget = IoTDBDescriptor.getInstance().getConfig().getMergeMemoryBudget(); + MergeResource mergeResource = new MergeResource(sequenceFileList, unSequenceFileList); + IMergeFileSelector fileSelector = getMergeFileSelector(budget, mergeResource); + try { + List[] mergeFiles = fileSelector.select(); + if (mergeFiles.length == 0) { + logger.info("{} cannot select merge candidates under the budget {}", storageGroupName, + budget); + return; + } + // avoid pending tasks holds the metadata and streams + mergeResource.clear(); + String taskName = storageGroupName + "-" + System.currentTimeMillis(); + // do not cache metadata until true candidates are chosen, or too much metadata will be + // cached during selection + mergeResource.setCacheDeviceMeta(true); + + MergeTask mergeTask = new MergeTask(mergeResource, storageGroupSysDir.getPath(), + this::mergeEndAction, taskName, fullMerge, fileSelector.getConcurrentMergeNum(), storageGroupName); + mergingModification = new ModificationFile(storageGroupSysDir + File.separator + MERGING_MODIFICAITON_FILE_NAME); + MergeManager.getINSTANCE().submitMainTask(mergeTask); + if (logger.isInfoEnabled()) { + logger.info("{} submits a merge task {}, merging {} seqFiles, {} unseqFiles", + storageGroupName, taskName, mergeFiles[0].size(), mergeFiles[1].size()); + } + isMerging = true; + mergeStartTime = System.currentTimeMillis(); + + } catch (MergeException | IOException e) { + logger.error("{} cannot select file for merge", storageGroupName, e); + } + } finally { + writeUnlock(); + } + } + + private IMergeFileSelector getMergeFileSelector(long budget, MergeResource resource) { + MergeFileStrategy strategy = IoTDBDescriptor.getInstance().getConfig().getMergeFileStrategy(); + switch (strategy) { + case MAX_FILE_NUM: + return new MaxFileMergeFileSelector(resource, budget); + case MAX_SERIES_NUM: + return new MaxSeriesMergeFileSelector(resource, budget); + default: + throw new UnsupportedOperationException("Unknown MergeFileStrategy " + strategy); + } + } + + protected void mergeEndAction(List seqFiles, List unseqFiles, + File mergeLog) { + logger.info("{} a merge task is ending...", storageGroupName); + + if (unseqFiles.isEmpty()) { + // merge runtime exception arose, just end this merge + isMerging = false; + logger.info("{} a merge task abnormally ends", storageGroupName); + return; + } + + mergeLock.writeLock().lock(); + try { + unSequenceFileList.removeAll(unseqFiles); + } finally { + mergeLock.writeLock().unlock(); + } + + for (TsFileResource unseqFile : unseqFiles) { + unseqFile.getMergeQueryLock().writeLock().lock(); + try { + unseqFile.remove(); + } finally { + unseqFile.getMergeQueryLock().writeLock().unlock(); + } + } + + for (int i = 0; i < seqFiles.size(); i++) { + TsFileResource seqFile = seqFiles.get(i); + seqFile.getMergeQueryLock().writeLock().lock(); + mergeLock.writeLock().lock(); + try { + logger.debug("{} is updating the {} merged file's modification file", storageGroupName, i); + try { + // remove old modifications and write modifications generated during merge + seqFile.removeModFile(); + if (mergingModification != null) { + for (Modification modification : mergingModification.getModifications()) { + seqFile.getModFile().write(modification); + } + } + } catch (IOException e) { + logger.error("{} cannot clean the ModificationFile of {} after merge", storageGroupName, + seqFile.getFile(), e); + } + if (i == seqFiles.size() - 1) { + try { + if (mergingModification != null) { + mergingModification.remove(); + mergingModification = null; + } + } catch (IOException e) { + logger.error("{} cannot remove merging modification ", storageGroupName, e); + } + isMerging = false; + } + } finally { + mergeLog.delete(); + seqFile.getMergeQueryLock().writeLock().unlock(); + mergeLock.writeLock().unlock(); + } + } + logger.info("{} a merge task ends", storageGroupName); + } + public TsFileProcessor getWorkSequenceTsFileProcessor() { return workSequenceTsFileProcessor; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 34dcbac55609a8b260ef9dcf6207bd518ec7ee14..5d237d19f7c4817b26c10527681c25d47cab64b9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -44,16 +44,18 @@ import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor.CloseTsFile import org.apache.iotdb.db.engine.version.VersionController; import org.apache.iotdb.db.exception.TsFileProcessorException; import org.apache.iotdb.db.qp.constant.DatetimeUtils; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.rescon.MemTablePool; +import org.apache.iotdb.rpc.TSStatusType; import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager; import org.apache.iotdb.db.writelog.node.WriteLogNode; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Pair; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +66,7 @@ public class TsFileProcessor { private RestorableTsFileIOWriter writer; - private FileSchema fileSchema; + private Schema schema; private final String storageGroupName; @@ -109,13 +111,13 @@ public class TsFileProcessor { private long totalMemTableSize; - TsFileProcessor(String storageGroupName, File tsfile, FileSchema fileSchema, + TsFileProcessor(String storageGroupName, File tsfile, Schema schema, VersionController versionController, CloseTsFileCallBack closeTsFileCallback, Supplier updateLatestFlushTimeCallback, boolean sequence) throws IOException { this.storageGroupName = storageGroupName; - this.fileSchema = fileSchema; + this.schema = schema; this.tsFileResource = new TsFileResource(tsfile, this); this.versionController = versionController; this.writer = new RestorableTsFileIOWriter(tsfile); @@ -134,13 +136,7 @@ public class TsFileProcessor { public boolean insert(InsertPlan insertPlan) { if (workMemTable == null) { - // TODO change the impl of getAvailableMemTable to non-blocking workMemTable = MemTablePool.getInstance().getAvailableMemTable(this); - - // no empty memtable, return failure - if (workMemTable == null) { - return false; - } } if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) { @@ -165,6 +161,37 @@ public class TsFileProcessor { return true; } + public boolean insertBatch(BatchInsertPlan batchInsertPlan, List indexes, + Integer[] results) { + if (workMemTable == null) { + workMemTable = MemTablePool.getInstance().getAvailableMemTable(this); + } + + if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) { + try { + getLogNode().write(batchInsertPlan); + } catch (IOException e) { + logger.error("write WAL failed", e); + for (int index: indexes) { + results[index] = TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(); + } + return false; + } + } + + tsFileResource.updateStartTime(batchInsertPlan.getDeviceId(), batchInsertPlan.getMinTime()); + + //for sequence tsfile, we update the endTime only when the file is prepared to be closed. + //for unsequence tsfile, we have to update the endTime for each insertion. + if (!sequence) { + tsFileResource.updateEndTime(batchInsertPlan.getDeviceId(), batchInsertPlan.getMaxTime()); + } + + // insert insertPlan to the work memtable + workMemTable.insertBatch(batchInsertPlan, indexes); + return true; + } + /** * Delete data which belongs to the timeseries `deviceId.measurementId` and the timestamp of which * <= 'timestamp' in the deletion.
@@ -372,7 +399,7 @@ public class TsFileProcessor { // signal memtable only may appear when calling asyncClose() if (!memTableToFlush.isSignalMemTable()) { - MemTableFlushTask flushTask = new MemTableFlushTask(memTableToFlush, fileSchema, writer, + MemTableFlushTask flushTask = new MemTableFlushTask(memTableToFlush, schema, writer, storageGroupName); try { writer.mark(); @@ -439,7 +466,7 @@ public class TsFileProcessor { long closeStartTime = System.currentTimeMillis(); tsFileResource.serialize(); - writer.endFile(fileSchema); + writer.endFile(schema); // remove this processor from Closing list in StorageGroupProcessor, // mark the TsFileResource closed, no need writer anymore diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index fae36f2e81522e6ff2757f0691a627c2dbe2847f..3a7af7ec5d2b7788535093c9d93ac4c976eec541 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@ -30,6 +30,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.io.FileUtils; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; @@ -40,6 +44,7 @@ public class TsFileResource { private File file; public static final String RESOURCE_SUFFIX = ".resource"; + public static final String TEMP_SUFFIX = ".temp"; /** * device -> start time @@ -63,23 +68,24 @@ public class TsFileResource { */ private List chunkMetaDatas; - /** * Mem chunk data. Only be set in a temporal TsFileResource in a query process. */ private ReadOnlyMemChunk readOnlyMemChunk; + private ReentrantReadWriteLock mergeQueryLock = new ReentrantReadWriteLock(); + public TsFileResource(File file) { this.file = file; - this.startTimeMap = new HashMap<>(); + this.startTimeMap = new ConcurrentHashMap<>(); this.endTimeMap = new HashMap<>(); this.closed = true; } public TsFileResource(File file, TsFileProcessor processor) { this.file = file; - this.startTimeMap = new HashMap<>(); - this.endTimeMap = new HashMap<>(); + this.startTimeMap = new ConcurrentHashMap<>(); + this.endTimeMap = new ConcurrentHashMap<>(); this.processor = processor; } @@ -106,7 +112,7 @@ public class TsFileResource { public void serialize() throws IOException { try (OutputStream outputStream = new BufferedOutputStream( - new FileOutputStream(file + RESOURCE_SUFFIX))) { + new FileOutputStream(file + RESOURCE_SUFFIX + TEMP_SUFFIX))) { ReadWriteIOUtils.write(this.startTimeMap.size(), outputStream); for (Entry entry : this.startTimeMap.entrySet()) { ReadWriteIOUtils.write(entry.getKey(), outputStream); @@ -118,6 +124,10 @@ public class TsFileResource { ReadWriteIOUtils.write(entry.getValue(), outputStream); } } + File src = new File(file + RESOURCE_SUFFIX + TEMP_SUFFIX); + File dest = new File(file + RESOURCE_SUFFIX); + dest.delete(); + FileUtils.moveFile(src, dest); } public void deSerialize() throws IOException { @@ -221,11 +231,44 @@ public class TsFileResource { return processor; } - public void updateTime(String deviceId, long time) { - startTimeMap.putIfAbsent(deviceId, time); - Long endTime = endTimeMap.get(deviceId); - if (endTime == null || endTime < time) { - endTimeMap.put(deviceId, time); + public ReentrantReadWriteLock getMergeQueryLock() { + return mergeQueryLock; + } + + public void removeModFile() throws IOException { + getModFile().remove(); + modFile = null; + } + + public void remove() { + file.delete(); + new File(file.getPath() + RESOURCE_SUFFIX).delete(); + new File(file.getPath() + ModificationFile.FILE_SUFFIX).delete(); + } + + @Override + public String toString() { + return file.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; } + TsFileResource that = (TsFileResource) o; + return Objects.equals(file, that.file); + } + + @Override + public int hashCode() { + return Objects.hash(file); + } + + public void setClosed(boolean closed) { + this.closed = closed; } } diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/MetadataDemo.java b/server/src/main/java/org/apache/iotdb/db/exception/MergeException.java similarity index 61% rename from jdbc/src/test/java/org/apache/iotdb/jdbc/demo/MetadataDemo.java rename to server/src/main/java/org/apache/iotdb/db/exception/MergeException.java index 41c22d3805ce8295466a9e14d1346b43dd4f1854..2980cfb9183822ada9a462582ad85bef7eaa44cd 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/demo/MetadataDemo.java +++ b/server/src/main/java/org/apache/iotdb/db/exception/MergeException.java @@ -16,21 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.jdbc.demo; -import java.sql.SQLException; +package org.apache.iotdb.db.exception; -public class MetadataDemo { +public class MergeException extends Exception{ - public static void main(String[] args) throws ClassNotFoundException, SQLException { - // Class.forName(TsfileJDBCConfig.JDBC_DRIVER_NAME); - // Connection connection = null; - // try { - // connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - // DatabaseMetaData databaseMetaData = connection.getMetaData(); - // } finally { - // connection.close(); - // } + public MergeException() { } -} \ No newline at end of file + public MergeException(String message) { + super(message); + } + + public MergeException(String message, Throwable cause) { + super(message, cause); + } + + public MergeException(Throwable cause) { + super(cause); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java b/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java index ac71b97dba32fd62d512023f72bfd009a609a663..64d10d517c13cb4c14a20d6b418ec88743290f67 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/MGraph.java @@ -376,14 +376,12 @@ public class MGraph implements Serializable { } /** - * * @return storage group name -> the series number */ Map countSeriesNumberInEachStorageGroup() throws PathErrorException { Map res = new HashMap<>(); Set storageGroups = this.getAllStorageGroup(); for (String sg : storageGroups) { - this.getNumSchemaMapForOneFileNode(sg); MNode node = mtree.getNodeByPath(sg); res.put(sg, node.getLeafCount()); } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java index a6aefd6c62cfa9f51ee6b0e0657e6e50f5e74344..d9654bc07185744965b550162799de7d23d5fde0 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.exception.MetadataErrorException; import org.apache.iotdb.db.exception.PathErrorException; import org.apache.iotdb.db.exception.ProcessorException; import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.query.context.QueryContext; @@ -114,6 +115,12 @@ public interface IQueryProcessExecutor { */ boolean insert(InsertPlan insertPlan) throws ProcessorException; + /** + * execute batch insert plan + * @return result of each row + */ + Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws ProcessorException; + boolean judgePathExists(Path fullPath); /** diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java index 6d5ee597a7afd80d015e8e039dbaafe5d3541975..a73691575d21969904918e261e02859c40254575 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.qp.logical.sys.AuthorOperator.AuthorType; import org.apache.iotdb.db.qp.logical.sys.MetadataOperator; import org.apache.iotdb.db.qp.logical.sys.PropertyOperator; import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.qp.physical.crud.UpdatePlan; @@ -200,20 +201,18 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor { @Override - public boolean insert(InsertPlan insertPlan) - throws ProcessorException { - + public boolean insert(InsertPlan insertPlan) throws ProcessorException { try { String[] measurementList = insertPlan.getMeasurements(); String deviceId = insertPlan.getDeviceId(); - MNode node = mManager.getNodeByDeviceIdFromCache(insertPlan.getDeviceId()); + MNode node = mManager.getNodeByDeviceIdFromCache(deviceId); String[] values = insertPlan.getValues(); TSDataType[] dataTypes = new TSDataType[measurementList.length]; for (int i = 0; i < measurementList.length; i++) { if (!node.hasChild(measurementList[i])) { throw new ProcessorException( - String.format("Current deviceId[%s] does not contains measurement:%s", + String.format("Current deviceId[%s] does not contain measurement:%s", deviceId, measurementList[i])); } MNode measurementNode = node.getChild(measurementList[i]); @@ -234,6 +233,32 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor { } } + @Override + public Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws ProcessorException { + try { + String[] measurementList = batchInsertPlan.getMeasurements(); + String deviceId = batchInsertPlan.getDeviceId(); + MNode node = mManager.getNodeByDeviceIdFromCache(deviceId); + + for (String s : measurementList) { + if (!node.hasChild(s)) { + throw new ProcessorException( + String.format("Current deviceId[%s] does not contain measurement:%s", + deviceId, s)); + } + MNode measurementNode = node.getChild(s); + if (!measurementNode.isLeaf()) { + throw new ProcessorException( + String.format("Current Path is not leaf node. %s.%s", deviceId, s)); + } + } + return storageEngine.insertBatch(batchInsertPlan); + + } catch (PathErrorException | StorageEngineException e) { + throw new ProcessorException(e); + } + } + @Override public List getAllPaths(String originPath) throws MetadataErrorException { return MManager.getInstance().getPaths(originPath); @@ -352,7 +377,7 @@ public class QueryProcessExecutor extends AbstractQueryProcessExecutor { storageEngine.deleteAllDataFilesInOneStorageGroup(deleteStorageGroup); } break; - case SET_FILE_LEVEL: + case SET_STORAGE_GROUP: mManager.setStorageLevelToMTree(path.getFullPath()); break; default: diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java index 498b429ef50434834efb2a6aeca585da685d0776..f18df8782f539312c64c25a3b4d38a85a27900a5 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/Operator.java @@ -67,7 +67,7 @@ public abstract class Operator { */ public enum OperatorType { SFW, JOIN, UNION, FILTER, GROUPBY, ORDERBY, LIMIT, SELECT, SEQTABLESCAN, HASHTABLESCAN, - MERGEJOIN, FILEREAD, NULL, TABLESCAN, UPDATE, INSERT, DELETE, BASIC_FUNC, QUERY, MERGEQUERY, + MERGEJOIN, FILEREAD, NULL, TABLESCAN, UPDATE, INSERT, BATCHINSERT, DELETE, BASIC_FUNC, QUERY, MERGEQUERY, AGGREGATION, AUTHOR, FROM, FUNC, LOADDATA, METADATA, PROPERTY, INDEX, INDEXQUERY, FILL, SET_STORAGE_GROUP, CREATE_TIMESERIES, DELETE_TIMESERIES, CREATE_USER, DELETE_USER, MODIFY_PASSWORD, GRANT_USER_PRIVILEGE, REVOKE_USER_PRIVILEGE, GRANT_USER_ROLE, REVOKE_USER_ROLE, CREATE_ROLE, diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/MetadataOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/MetadataOperator.java index 4f2c6bd21a6f5bf762abaa637779f42944370d3a..2a80f6b387f99483911a11672422c9922c82e057 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/MetadataOperator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/MetadataOperator.java @@ -46,7 +46,7 @@ public class MetadataOperator extends RootOperator { super(tokenIntType); namespaceType = type; switch (type) { - case SET_FILE_LEVEL: + case SET_STORAGE_GROUP: operatorType = OperatorType.SET_STORAGE_GROUP; break; case ADD_PATH: @@ -113,7 +113,7 @@ public class MetadataOperator extends RootOperator { } public enum NamespaceType { - ADD_PATH, DELETE_PATH, SET_FILE_LEVEL; + ADD_PATH, DELETE_PATH, SET_STORAGE_GROUP; /** * deserialize short number. @@ -128,7 +128,7 @@ public class MetadataOperator extends RootOperator { case 1: return DELETE_PATH; case 2: - return SET_FILE_LEVEL; + return SET_STORAGE_GROUP; default: return null; } @@ -145,7 +145,7 @@ public class MetadataOperator extends RootOperator { return 0; case DELETE_PATH: return 1; - case SET_FILE_LEVEL: + case SET_STORAGE_GROUP: return 2; default: return -1; diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java index b60599f87007c5e5cf0e3029e5a102fa6e269b27..4288539484b2f8375bed787bc65de31bc796ff98 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import org.apache.iotdb.db.qp.logical.Operator; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.tsfile.read.common.Path; @@ -118,6 +119,10 @@ public abstract class PhysicalPlan { plan = new DeletePlan(); plan.deserializeFrom(buffer); break; + case BATCHINSERT: + plan = new BatchInsertPlan(); + plan.deserializeFrom(buffer); + break; default: throw new IOException("unrecognized log type " + type); } @@ -126,7 +131,7 @@ public abstract class PhysicalPlan { } public enum PhysicalPlanType { - INSERT, DELETE + INSERT, DELETE, BATCHINSERT } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java new file mode 100644 index 0000000000000000000000000000000000000000..ce907ed941073ad2fd685baeb631519a8fed4ff7 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/BatchInsertPlan.java @@ -0,0 +1,276 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.qp.physical.crud; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import org.apache.iotdb.db.qp.logical.Operator.OperatorType; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.db.utils.QueryDataSetUtils; +import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.BytesUtils; + +public class BatchInsertPlan extends PhysicalPlan { + + private String deviceId; + private String[] measurements; + private TSDataType[] dataTypes; + + private long[] times; + private ByteBuffer timeBuffer; + + private Object[] columns; + private ByteBuffer valueBuffer; + + private int rowCount = 0; + + // cached values + private Long maxTime = null; + private Long minTime = null; + private List paths; + + public BatchInsertPlan() { + super(false, OperatorType.BATCHINSERT); + } + + public BatchInsertPlan(String deviceId, List measurements) { + super(false, OperatorType.BATCHINSERT); + this.deviceId = deviceId; + setMeasurements(measurements); + } + + public BatchInsertPlan(String deviceId, String[] measurements, List dataTypes) { + super(false, OperatorType.BATCHINSERT); + this.deviceId = deviceId; + this.measurements = measurements; + setDataTypes(dataTypes); + } + + + @Override + public List getPaths() { + if (paths != null) { + return paths; + } + List ret = new ArrayList<>(); + for (String m : measurements) { + ret.add(new Path(deviceId, m)); + } + paths = ret; + return ret; + } + + @Override + public void serializeTo(ByteBuffer buffer) { + int type = PhysicalPlanType.BATCHINSERT.ordinal(); + buffer.put((byte) type); + + putString(buffer, deviceId); + + buffer.putInt(measurements.length); + for (String m : measurements) { + putString(buffer, m); + } + + for (TSDataType dataType: dataTypes) { + buffer.putShort(dataType.serialize()); + } + + buffer.putInt(times.length); + + if (timeBuffer == null) { + for (long time: times) { + buffer.putLong(time); + } + } else { + buffer.put(timeBuffer.array()); + timeBuffer = null; + } + + if (valueBuffer == null) { + for (int i = 0; i < measurements.length; i++) { + TSDataType dataType = dataTypes[i]; + switch (dataType) { + case INT32: + int[] intValues = (int[]) columns[i]; + for (int index = 0; index < rowCount; index++) { + buffer.putInt(intValues[index]); + } + break; + case INT64: + long[] longValues = (long[]) columns[i]; + for (int index = 0; index < rowCount; index++) { + buffer.putLong(longValues[index]); + } + break; + case FLOAT: + float[] floatValues = (float[]) columns[i]; + for (int index = 0; index < rowCount; index++) { + buffer.putFloat(floatValues[index]); + } + break; + case DOUBLE: + double[] doubleValues = (double[]) columns[i]; + for (int index = 0; index < rowCount; index++) { + buffer.putDouble(doubleValues[index]); + } + break; + case BOOLEAN: + boolean[] boolValues = (boolean[]) columns[i]; + for (int index = 0; index < rowCount; index++) { + buffer.put(BytesUtils.boolToByte(boolValues[index])); + } + break; + case TEXT: + Binary[] binaryValues = (Binary[]) columns[i]; + for (int index = 0; index < rowCount; index++) { + buffer.putInt(binaryValues[index].getLength()); + buffer.put(binaryValues[index].getValues()); + } + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", dataType)); + } + } + } else { + buffer.put(valueBuffer.array()); + valueBuffer = null; + } + } + + public void setTimeBuffer(ByteBuffer timeBuffer) { + this.timeBuffer = timeBuffer; + this.timeBuffer.position(0); + } + + public void setValueBuffer(ByteBuffer valueBuffer) { + this.valueBuffer = valueBuffer; + this.timeBuffer.position(0); + } + + @Override + public void deserializeFrom(ByteBuffer buffer) { + this.deviceId = readString(buffer); + + int measurementSize = buffer.getInt(); + this.measurements = new String[measurementSize]; + for (int i = 0; i < measurementSize; i++) { + measurements[i] = readString(buffer); + } + + this.dataTypes = new TSDataType[measurementSize]; + for (int i = 0; i < measurementSize; i++) { + dataTypes[i] = TSDataType.deserialize(buffer.getShort()); + } + + int rows = buffer.getInt(); + this.times = new long[rows]; + QueryDataSetUtils.readTimesFromBuffer(buffer, rows); + + QueryDataSetUtils.readValuesFromBuffer(buffer, dataTypes, measurementSize, rows); + } + + + public String getDeviceId() { + return deviceId; + } + + public void setDeviceId(String deviceId) { + this.deviceId = deviceId; + } + + public String[] getMeasurements() { + return measurements; + } + + public void setMeasurements(List measurements) { + this.measurements = new String[measurements.size()]; + measurements.toArray(this.measurements); + } + + public void setMeasurements(String[] measurements) { + this.measurements = measurements; + } + + public TSDataType[] getDataTypes() { + return dataTypes; + } + + public void setDataTypes(List dataTypes) { + this.dataTypes = new TSDataType[dataTypes.size()]; + for (int i = 0; i < dataTypes.size(); i++) { + this.dataTypes[i] = TSDataType.values()[dataTypes.get(i)]; + } + } + + public Object[] getColumns() { + return columns; + } + + public long getMinTime() { + if (minTime != null) { + return minTime; + } + minTime = Long.MAX_VALUE; + for (Long time: times) { + if (time < minTime) { + minTime = time; + } + } + return minTime; + } + + public long getMaxTime() { + if (maxTime != null) { + return maxTime; + } + long maxTime = Long.MIN_VALUE; + for (Long time: times) { + if (time > maxTime) { + maxTime = time; + } + } + return maxTime; + } + + public long[] getTimes() { + return times; + } + + public void setTimes(long[] times) { + this.times = times; + } + + public int getRowCount() { + return rowCount; + } + + public void setRowCount(int size) { + this.rowCount = size; + } + + public void setColumns(Object[] columns) { + this.columns = columns; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/MetadataPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/MetadataPlan.java index 263ea533c87331f2e1a19e6be4020276533e7fce..cae6f63c5ae9653ca0fbd962d17aa1cbf0f67ca4 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/MetadataPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/MetadataPlan.java @@ -55,19 +55,24 @@ public class MetadataPlan extends PhysicalPlan { this.encoding = encoding; this.props = props; this.deletePathList = deletePathList; - switch (namespaceType) { - case SET_FILE_LEVEL: - setOperatorType(Operator.OperatorType.SET_STORAGE_GROUP); - break; - case ADD_PATH: - setOperatorType(Operator.OperatorType.CREATE_TIMESERIES); - break; - case DELETE_PATH: - setOperatorType(Operator.OperatorType.DELETE_TIMESERIES); - break; - default: - break; - } + setOperatorType(namespaceType); + } + + public MetadataPlan(MetadataOperator.NamespaceType namespaceType, Path path, TSDataType dataType, + TSEncoding encoding) { + super(false, Operator.OperatorType.METADATA); + this.namespaceType = namespaceType; + this.path = path; + this.dataType = dataType; + this.encoding = encoding; + setOperatorType(namespaceType); + } + + public MetadataPlan(MetadataOperator.NamespaceType namespaceType, Path path) { + super(false, Operator.OperatorType.METADATA); + this.namespaceType = namespaceType; + this.path = path; + setOperatorType(namespaceType); } public Path getPath() { @@ -171,4 +176,20 @@ public class MetadataPlan extends PhysicalPlan { .hash(getNamespaceType(), getPath(), getDataType(), getCompressor(), getEncoding(), getProps(), getDeletePathList()); } + + private void setOperatorType(MetadataOperator.NamespaceType namespaceType) { + switch (namespaceType) { + case SET_STORAGE_GROUP: + setOperatorType(Operator.OperatorType.SET_STORAGE_GROUP); + break; + case ADD_PATH: + setOperatorType(Operator.OperatorType.CREATE_TIMESERIES); + break; + case DELETE_PATH: + setOperatorType(Operator.OperatorType.DELETE_TIMESERIES); + break; + default: + break; + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java index 13be7d33794cc1384efffc83872364dda1eaef50..fb288f750364af778025363ae678077c85c1ae02 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java @@ -425,7 +425,7 @@ public class LogicalGenerator { private void analyzeMetadataSetFileLevel(AstNode astNode) { MetadataOperator metadataOperator = new MetadataOperator( SQLConstant.TOK_METADATA_SET_FILE_LEVEL, - MetadataOperator.NamespaceType.SET_FILE_LEVEL); + MetadataOperator.NamespaceType.SET_STORAGE_GROUP); Path path = parsePath(astNode.getChild(0).getChild(0)); metadataOperator.setPath(path); initializedOperator = metadataOperator; @@ -533,10 +533,6 @@ public class LogicalGenerator { if (filterOperator.getTokenIntType() == LESSTHAN) { time = time - 1; } - // time must greater than 0 now - if (time <= 0) { - throw new LogicalOperatorException("delete Time:" + time + ", time must > 0"); - } return time; } diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java index 29d1bc9d8dcd69c64dfc57b47c8018b27fdaed52..b41a1efcb162383bc18420fc5d02d5b8fbfde905 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java @@ -90,9 +90,6 @@ public class PhysicalGenerator { case DELETE: DeleteOperator delete = (DeleteOperator) operator; paths = delete.getSelectedPaths(); - if (delete.getTime() <= 0) { - throw new LogicalOperatorException("For Delete command, time must greater than 0."); - } return new DeletePlan(delete.getTime(), paths); case INSERT: InsertOperator insert = (InsertOperator) operator; @@ -101,9 +98,6 @@ public class PhysicalGenerator { throw new LogicalOperatorException( "For Insert command, cannot specified more than one seriesPath:" + paths); } - if (insert.getTime() <= 0) { - throw new LogicalOperatorException("For Insert command, time must greater than 0."); - } return new InsertPlan(paths.get(0).getFullPath(), insert.getTime(), insert.getMeasurementList(), insert.getValueList()); diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java index 8d33137e6a928b2cf75f2d28a3a5913e1eaa6d2a..76dad2f1757d9725355c5d45296509134c1dd1e3 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java +++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java @@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.service.IService; import org.apache.iotdb.db.service.ServiceType; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; @@ -50,23 +51,23 @@ public class FileReaderManager implements IService { * the key of closedFileReaderMap is the file path and the value of closedFileReaderMap * is the corresponding reader. */ - private ConcurrentHashMap closedFileReaderMap; + private ConcurrentHashMap closedFileReaderMap; /** * the key of unclosedFileReaderMap is the file path and the value of unclosedFileReaderMap * is the corresponding reader. */ - private ConcurrentHashMap unclosedFileReaderMap; + private ConcurrentHashMap unclosedFileReaderMap; /** * the key of closedFileReaderMap is the file path and the value of closedFileReaderMap * is the file's reference count. */ - private ConcurrentHashMap closedReferenceMap; + private ConcurrentHashMap closedReferenceMap; /** * the key of unclosedFileReaderMap is the file path and the value of unclosedFileReaderMap * is the file's reference count. */ - private ConcurrentHashMap unclosedReferenceMap; + private ConcurrentHashMap unclosedReferenceMap; private ScheduledExecutorService executorService; @@ -97,9 +98,9 @@ public class FileReaderManager implements IService { }, 0, examinePeriod, TimeUnit.MILLISECONDS); } - private void clearMap(Map readerMap, - Map refMap) { - for (Map.Entry entry : readerMap.entrySet()) { + private void clearMap(Map readerMap, + Map refMap) { + for (Map.Entry entry : readerMap.entrySet()) { TsFileSequenceReader reader = entry.getValue(); AtomicInteger refAtom = refMap.get(entry.getKey()); @@ -120,71 +121,73 @@ public class FileReaderManager implements IService { * exists, just get it from closedFileReaderMap or unclosedFileReaderMap depending on isClosing . * Otherwise a new reader will be created and cached. * - * @param filePath the path of the file, of which the reader is desired. + * @param tsFile the path of the file, of which the reader is desired. * @param isClosed whether the corresponding file still receives insertions or not. * @return the reader of the file specified by filePath. * @throws IOException when reader cannot be created. */ - public synchronized TsFileSequenceReader get(String filePath, boolean isClosed) + public synchronized TsFileSequenceReader get(TsFileResource tsFile, boolean isClosed) throws IOException { - Map readerMap = !isClosed ? unclosedFileReaderMap + Map readerMap = !isClosed ? unclosedFileReaderMap : closedFileReaderMap; - if (!readerMap.containsKey(filePath)) { + if (!readerMap.containsKey(tsFile)) { if (readerMap.size() >= MAX_CACHED_FILE_SIZE) { logger.warn("Query has opened {} files !", readerMap.size()); } - TsFileSequenceReader tsFileReader = !isClosed ? new UnClosedTsFileReader(filePath) - : new TsFileSequenceReader(filePath); + TsFileSequenceReader tsFileReader = !isClosed ? new UnClosedTsFileReader(tsFile.getFile().getPath()) + : new TsFileSequenceReader(tsFile.getFile().getPath()); - readerMap.put(filePath, tsFileReader); + readerMap.put(tsFile, tsFileReader); return tsFileReader; } - return readerMap.get(filePath); + return readerMap.get(tsFile); } /** * Increase the reference count of the reader specified by filePath. Only when the reference count * of a reader equals zero, the reader can be closed and removed. */ - public synchronized void increaseFileReaderReference(String filePath, boolean isClosed) { + public synchronized void increaseFileReaderReference(TsFileResource tsFile, boolean isClosed) { // TODO : this should be called in get() if (!isClosed) { - unclosedReferenceMap.computeIfAbsent(filePath, k -> new AtomicInteger()).getAndIncrement(); + unclosedReferenceMap.computeIfAbsent(tsFile, k -> new AtomicInteger()).getAndIncrement(); } else { - closedReferenceMap.computeIfAbsent(filePath, k -> new AtomicInteger()).getAndIncrement(); + closedReferenceMap.computeIfAbsent(tsFile, k -> new AtomicInteger()).getAndIncrement(); } + tsFile.getMergeQueryLock().readLock().lock(); } /** * Decrease the reference count of the reader specified by filePath. This method is latch-free. * Only when the reference count of a reader equals zero, the reader can be closed and removed. */ - public synchronized void decreaseFileReaderReference(String filePath, boolean isClosed) { - if (!isClosed && unclosedReferenceMap.containsKey(filePath)) { - unclosedReferenceMap.get(filePath).getAndDecrement(); - } else if (closedReferenceMap.containsKey(filePath)){ - closedReferenceMap.get(filePath).getAndDecrement(); + public synchronized void decreaseFileReaderReference(TsFileResource tsFile, boolean isClosed) { + if (!isClosed && unclosedReferenceMap.containsKey(tsFile)) { + unclosedReferenceMap.get(tsFile).getAndDecrement(); + } else if (closedReferenceMap.containsKey(tsFile)){ + closedReferenceMap.get(tsFile).getAndDecrement(); } + tsFile.getMergeQueryLock().readLock().unlock(); } /** * This method is used when the given file path is deleted. */ - public synchronized void closeFileAndRemoveReader(String filePath) + public synchronized void closeFileAndRemoveReader(TsFileResource tsFile) throws IOException { - if (unclosedFileReaderMap.containsKey(filePath)) { - unclosedReferenceMap.remove(filePath); - unclosedFileReaderMap.get(filePath).close(); - unclosedFileReaderMap.remove(filePath); + if (unclosedFileReaderMap.containsKey(tsFile)) { + unclosedReferenceMap.remove(tsFile); + unclosedFileReaderMap.get(tsFile).close(); + unclosedFileReaderMap.remove(tsFile); } - if (closedFileReaderMap.containsKey(filePath)) { - closedReferenceMap.remove(filePath); - closedFileReaderMap.get(filePath).close(); - closedFileReaderMap.remove(filePath); + if (closedFileReaderMap.containsKey(tsFile)) { + closedReferenceMap.remove(tsFile); + closedFileReaderMap.get(tsFile).close(); + closedFileReaderMap.remove(tsFile); } } @@ -193,12 +196,12 @@ public class FileReaderManager implements IService { * integration tests will not conflict with each other. */ public synchronized void closeAndRemoveAllOpenedReaders() throws IOException { - for (Map.Entry entry : closedFileReaderMap.entrySet()) { + for (Map.Entry entry : closedFileReaderMap.entrySet()) { entry.getValue().close(); closedReferenceMap.remove(entry.getKey()); closedFileReaderMap.remove(entry.getKey()); } - for (Map.Entry entry : unclosedFileReaderMap.entrySet()) { + for (Map.Entry entry : unclosedFileReaderMap.entrySet()) { entry.getValue().close(); unclosedReferenceMap.remove(entry.getKey()); unclosedFileReaderMap.remove(entry.getKey()); @@ -208,9 +211,9 @@ public class FileReaderManager implements IService { /** * This method is only for unit tests. */ - public synchronized boolean contains(String filePath, boolean isClosed) { - return (isClosed && closedFileReaderMap.containsKey(filePath)) - || (!isClosed && unclosedFileReaderMap.containsKey(filePath)); + public synchronized boolean contains(TsFileResource tsFile, boolean isClosed) { + return (isClosed && closedFileReaderMap.containsKey(tsFile)) + || (!isClosed && unclosedFileReaderMap.containsKey(tsFile)); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java index f102e4448df26bc22c02ac45f301c6d54ddd278b..d8901f790fe436323afd4521108302a379b4e07b 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java +++ b/server/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.query.control; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -34,8 +35,8 @@ public class JobFileManager { /** * Map> */ - private ConcurrentHashMap> sealedFilePathsMap; - private ConcurrentHashMap> unsealedFilePathsMap; + private ConcurrentHashMap> sealedFilePathsMap; + private ConcurrentHashMap> unsealedFilePathsMap; public JobFileManager() { sealedFilePathsMap = new ConcurrentHashMap<>(); @@ -59,14 +60,12 @@ public class JobFileManager { //sequence data for(TsFileResource tsFileResource : dataSource.getSeqResources()){ - String path = tsFileResource.getFile().getPath(); - addFilePathToMap(jobId, path, tsFileResource.isClosed()); + addFilePathToMap(jobId, tsFileResource, tsFileResource.isClosed()); } //unsequence data for(TsFileResource tsFileResource : dataSource.getUnseqResources()){ - String path = tsFileResource.getFile().getPath(); - addFilePathToMap(jobId, path, tsFileResource.isClosed()); + addFilePathToMap(jobId, tsFileResource, tsFileResource.isClosed()); } } @@ -75,14 +74,20 @@ public class JobFileManager { * this jdbc request must be cleared and thus the usage reference must be decreased. */ void removeUsedFilesForGivenJob(long jobId) { - for (String filePath : sealedFilePathsMap.get(jobId)) { - FileReaderManager.getInstance().decreaseFileReaderReference(filePath, true); + Set tsFiles = sealedFilePathsMap.get(jobId); + if (tsFiles != null) { + for (TsFileResource tsFile : sealedFilePathsMap.get(jobId)) { + FileReaderManager.getInstance().decreaseFileReaderReference(tsFile, true); } sealedFilePathsMap.remove(jobId); - for (String filePath : unsealedFilePathsMap.get(jobId)) { - FileReaderManager.getInstance().decreaseFileReaderReference(filePath, false); + } + tsFiles = unsealedFilePathsMap.get(jobId); + if (tsFiles != null) { + for (TsFileResource tsFile : unsealedFilePathsMap.get(jobId)) { + FileReaderManager.getInstance().decreaseFileReaderReference(tsFile, false); } unsealedFilePathsMap.remove(jobId); + } } /** @@ -91,13 +96,13 @@ public class JobFileManager { * so sealedFilePathsMap.get(jobId) or unsealedFilePathsMap.get(jobId) * must not return null. */ - void addFilePathToMap(long jobId, String filePath, boolean isSealed) { - ConcurrentHashMap> pathMap = !isSealed ? unsealedFilePathsMap : + void addFilePathToMap(long jobId, TsFileResource tsFile, boolean isClosed) { + ConcurrentHashMap> pathMap = !isClosed ? unsealedFilePathsMap : sealedFilePathsMap; //TODO this is not an atomic operation, is there concurrent problem? - if (!pathMap.get(jobId).contains(filePath)) { - pathMap.get(jobId).add(filePath); - FileReaderManager.getInstance().increaseFileReaderReference(filePath, isSealed); + if (!pathMap.get(jobId).contains(tsFile)) { + pathMap.get(jobId).add(tsFile); + FileReaderManager.getInstance().increaseFileReaderReference(tsFile, isClosed); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java index be81daf8a63dd045236b9677326f433cd72c43b4..e05943e9bf6aae62f2119635710e9fa9df25deae 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java +++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java @@ -18,12 +18,7 @@ */ package org.apache.iotdb.db.query.control; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; @@ -40,56 +35,15 @@ import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression; * QueryResourceManager manages resource (file streams) used by each query job, and assign Ids to the jobs. * During the life cycle of a query, the following methods must be called in strict order: * 1. assignJobId - get an Id for the new job. - * 2. beginQueryOfGivenQueryPaths - remind StorageEngine that some files are being used - * 3. (if using filter)beginQueryOfGivenExpression - * - remind StorageEngine that some files are being used - * 4. getQueryDataSource - open files for the job or reuse existing readers. - * 5. endQueryForGivenJob - putBack the resource used by this job. + * 2. getQueryDataSource - open files for the job or reuse existing readers. + * 3. endQueryForGivenJob - release the resource used by this job. *

*/ public class QueryResourceManager { - /** - * Map<jobId, Map<deviceId, List<token>>>. - * - *

- * Key of queryTokensMap is job id, value of queryTokensMap is a deviceId-tokenList map, key of - * the deviceId-tokenList map is device id, value of deviceId-tokenList map is a list of tokens. - *

- * - *

- * For example, during a query process Q1, given a query sql select device_1.sensor_1, - * device_1.sensor_2, device_2.sensor_1, device_2.sensor_2, we will invoke - * StorageEngine.getInstance().beginQuery(device_1) and - * StorageEngine.getInstance().beginQuery(device_2) both once. Although there - * exists four paths, but the unique devices are only `device_1` and `device_2`. When invoking - * StorageEngine.getInstance().beginQuery(device_1), it returns result token `1`. - * Similarly, - * StorageEngine.getInstance().beginQuery(device_2) returns result token `2`. - * - * In the meanwhile, another query process Q2 aroused by other client is triggered, whose sql - * statement is same to Q1. Although StorageEngine.getInstance().beginQuery(device_1) - * - * and - * StorageEngine.getInstance().beginQuery(device_2) will be invoked again, it - * returns result token `3` and `4` . - * - * StorageEngine.getInstance().endQueryForGivenJob(device_1, 1) and - * StorageEngine.getInstance().endQueryForGivenJob(device_2, 2) must be invoked no matter how - * query process Q1 exits normally or abnormally. So is Q2, - * StorageEngine.getInstance().endQueryForGivenJob(device_1, 3) and - * StorageEngine.getInstance().endQueryForGivenJob(device_2, 4) must be invoked - * - * Last but no least, to ensure the correctness of insert process and query process of IoTDB, - * StorageEngine.getInstance().beginQuery() and - * StorageEngine.getInstance().endQueryForGivenJob() must be executed rightly. - *

- */ - private ConcurrentHashMap>> queryTokensMap; private JobFileManager filePathsManager; private AtomicLong maxJobId; private QueryResourceManager() { - queryTokensMap = new ConcurrentHashMap<>(); filePathsManager = new JobFileManager(); maxJobId = new AtomicLong(0); } @@ -104,86 +58,24 @@ public class QueryResourceManager { */ public long assignJobId() { long jobId = maxJobId.incrementAndGet(); - queryTokensMap.computeIfAbsent(jobId, x -> new ConcurrentHashMap<>()); filePathsManager.addJobId(jobId); return jobId; } - /** - * Begin query and set query tokens of queryPaths. This method is used for projection - * calculation. - */ - public void beginQueryOfGivenQueryPaths(long jobId, List queryPaths) - throws StorageEngineException { - Set deviceIdSet = new HashSet<>(); - queryPaths.forEach(path -> deviceIdSet.add(path.getDevice())); - - for (String deviceId : deviceIdSet) { - putQueryTokenForCurrentRequestThread(jobId, deviceId, - StorageEngine.getInstance().beginQuery(deviceId)); - } - } - - /** - * Begin query and set query tokens of all paths in expression. This method is used in filter - * calculation. - */ - public void beginQueryOfGivenExpression(long jobId, IExpression expression) - throws StorageEngineException { - Set deviceIdSet = new HashSet<>(); - getUniquePaths(expression, deviceIdSet); - for (String deviceId : deviceIdSet) { - putQueryTokenForCurrentRequestThread(jobId, deviceId, - StorageEngine.getInstance().beginQuery(deviceId)); - } - } - - /** - * Begin query and set query tokens of all filter paths in expression. This method is used in - * filter calculation. - * @param remoteDeviceIdSet device id set which can not handle locally - * Note : the method is for cluster - */ - public void beginQueryOfGivenExpression(long jobId, IExpression expression, - Set remoteDeviceIdSet) throws StorageEngineException { - Set deviceIdSet = new HashSet<>(); - getUniquePaths(expression, deviceIdSet); - deviceIdSet.removeAll(remoteDeviceIdSet); - for (String deviceId : deviceIdSet) { - putQueryTokenForCurrentRequestThread(jobId, deviceId, - StorageEngine.getInstance().beginQuery(deviceId)); - } - } - public QueryDataSource getQueryDataSource(Path selectedPath, QueryContext context) throws StorageEngineException { SingleSeriesExpression singleSeriesExpression = new SingleSeriesExpression(selectedPath, null); - QueryDataSource queryDataSource = StorageEngine - .getInstance().query(singleSeriesExpression, context); - - // add used files to current thread request cached map - filePathsManager.addUsedFilesForGivenJob(context.getJobId(), queryDataSource); - - return queryDataSource; + return StorageEngine + .getInstance().query(singleSeriesExpression, context, filePathsManager); } /** * Whenever the jdbc request is closed normally or abnormally, this method must be invoked. All * query tokens created by this jdbc request must be cleared. */ - public void endQueryForGivenJob(long jobId) throws StorageEngineException { - if (queryTokensMap.get(jobId) == null) { - // no resource need to be released. - return; - } - for (Map.Entry> entry : queryTokensMap.get(jobId).entrySet()) { - for (int token : entry.getValue()) { - StorageEngine.getInstance().endQuery(entry.getKey(), token); - } - } - queryTokensMap.remove(jobId); + public void endQueryForGivenJob(long jobId) { // remove usage of opened file paths of current thread filePathsManager.removeUsedFilesForGivenJob(jobId); } @@ -198,10 +90,6 @@ public class QueryResourceManager { } } - private void putQueryTokenForCurrentRequestThread(long jobId, String deviceId, int queryToken) { - queryTokensMap.get(jobId).computeIfAbsent(deviceId, x -> new ArrayList<>()).add(queryToken); - } - private static class QueryTokenManagerHelper { private static final QueryResourceManager INSTANCE = new QueryResourceManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java index 2993a7da7aa1c23d65f1993541574b77e5141ea5..d5771f3650c117a1b33161b691118d9cf21a4373 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java @@ -73,9 +73,6 @@ public class GroupByWithValueFilterDataSet extends GroupByEngineDataSet { throws StorageEngineException, PathErrorException, ProcessorException, IOException { initAggreFuction(aggres); - QueryResourceManager.getInstance().beginQueryOfGivenExpression(context.getJobId(), expression); - QueryResourceManager - .getInstance().beginQueryOfGivenQueryPaths(context.getJobId(), selectedSeries); this.timestampGenerator = new EngineTimeGenerator(expression, context); this.allDataReaderList = new ArrayList<>(); for (Path path : selectedSeries) { diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java index 47ff47ad6b84a8cae33ebbef2a74bf37b23686bf..3043d09a8cf52471e29a53f5c0122dea70d4d70a 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java +++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java @@ -76,7 +76,6 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet { throws StorageEngineException, PathErrorException, ProcessorException, IOException { initAggreFuction(aggres); // init reader - QueryResourceManager.getInstance().beginQueryOfGivenQueryPaths(jobId, selectedSeries); if (expression != null) { timeFilter = ((GlobalTimeExpression) expression).getFilter(); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java index 6b31fe9682e4d99a2327d8af848f4c1d86777f4c..994ffa6467a01afc8b402d755816926af7799182 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java @@ -86,8 +86,6 @@ public class AggregateEngineExecutor { if (expression != null) { timeFilter = ((GlobalTimeExpression) expression).getFilter(); } - QueryResourceManager - .getInstance().beginQueryOfGivenQueryPaths(context.getJobId(), selectedSeries); List readersOfSequenceData = new ArrayList<>(); List readersOfUnSequenceData = new ArrayList<>(); @@ -229,14 +227,16 @@ public class AggregateEngineExecutor { } else { // cal by pageData BatchData batchData = sequenceReader.nextBatch(); - if (lastBatchTimeStamp > batchData.currentTime()) { - // the chunk is end. - isChunkEnd = true; - } else { - // current page and last page are in the same chunk. - lastBatchTimeStamp = batchData.currentTime(); + if (batchData.length() > 0) { + if (lastBatchTimeStamp > batchData.currentTime()) { + // the chunk is end. + isChunkEnd = true; + } else { + // current page and last page are in the same chunk. + lastBatchTimeStamp = batchData.currentTime(); + } + function.calculateValueFromPageData(batchData, unSequenceReader); } - function.calculateValueFromPageData(batchData, unSequenceReader); } if (isChunkEnd) { break; @@ -258,9 +258,6 @@ public class AggregateEngineExecutor { */ public QueryDataSet executeWithValueFilter(QueryContext context) throws StorageEngineException, PathErrorException, IOException, ProcessorException { - QueryResourceManager - .getInstance().beginQueryOfGivenQueryPaths(context.getJobId(), selectedSeries); - QueryResourceManager.getInstance().beginQueryOfGivenExpression(context.getJobId(), expression); EngineTimeGenerator timestampGenerator = new EngineTimeGenerator(expression, context); List readersOfSelectedSeries = new ArrayList<>(); diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java index 0279f9af57526e525e246bfae0deec82bf31cc27..c2bd99605be2457857f3b59219a698e4d380019d 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutor.java @@ -56,8 +56,6 @@ public class EngineExecutor { */ public QueryDataSet executeWithoutValueFilter(QueryContext context) throws StorageEngineException, IOException { - QueryResourceManager.getInstance() - .beginQueryOfGivenQueryPaths(context.getJobId(), queryExpression.getSelectedSeries()); Filter timeFilter = null; if (queryExpression.hasQueryFilter()) { @@ -94,11 +92,6 @@ public class EngineExecutor { */ public QueryDataSet executeWithValueFilter(QueryContext context) throws StorageEngineException, IOException { - QueryResourceManager.getInstance() - .beginQueryOfGivenQueryPaths(context.getJobId(), queryExpression.getSelectedSeries()); - QueryResourceManager.getInstance() - .beginQueryOfGivenExpression(context.getJobId(), queryExpression.getExpression()); - EngineTimeGenerator timestampGenerator = new EngineTimeGenerator( queryExpression.getExpression(), context); diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java index 20f9b006115f59c9f8fdd767e27f94f6cd37df20..3ac598bd0909b16cfae863f5422df15fb6ae463b 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java @@ -58,7 +58,6 @@ public class FillEngineExecutor { */ public QueryDataSet execute(QueryContext context) throws StorageEngineException, PathErrorException, IOException { - QueryResourceManager.getInstance().beginQueryOfGivenQueryPaths(jobId, selectedSeries); List fillList = new ArrayList<>(); List dataTypeList = new ArrayList<>(); diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/IPointReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/IPointReader.java index cca6968c2db644337ca0814fd7c0f34265a6822b..40a10e0fa424291b1f08d190eac628e2d63e72e6 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/IPointReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/IPointReader.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.utils.TimeValuePair; public interface IPointReader { + // FIXME: this means hasCurrent actually boolean hasNext() throws IOException; TimeValuePair next() throws IOException; diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/CachedDiskChunkReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/CachedDiskChunkReader.java new file mode 100644 index 0000000000000000000000000000000000000000..6f09219ad8af9d9775d25eafbf835f3d22498fa7 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/CachedDiskChunkReader.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.reader.chunkRelated; + +import java.io.IOException; +import org.apache.iotdb.db.query.reader.IPointReader; +import org.apache.iotdb.db.utils.TimeValuePair; +import org.apache.iotdb.db.utils.TimeValuePairUtils; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; + +public class CachedDiskChunkReader implements IPointReader { + + private ChunkReader chunkReader; + private BatchData data; + private TimeValuePair prev; + private TimeValuePair current; + + public CachedDiskChunkReader(ChunkReader chunkReader) { + this.chunkReader = chunkReader; + this.prev = + TimeValuePairUtils.getEmptyTimeValuePair(chunkReader.getChunkHeader().getDataType()); + } + + @Override + public boolean hasNext() throws IOException { + if (data != null && data.hasNext()) { + return true; + } + while (chunkReader.hasNextBatch()) { + data = chunkReader.nextBatch(); + if (data.hasNext()) { + return true; + } + } + return false; + } + + @Override + public TimeValuePair next() throws IOException { + TimeValuePairUtils.setCurrentTimeValuePair(data, prev); + data.next(); + if (data.hasNext()) { + TimeValuePairUtils.setCurrentTimeValuePair(data, current()); + } else { + while (chunkReader.hasNextBatch()) { + data = chunkReader.nextBatch(); + if (data.hasNext()) { + TimeValuePairUtils.setCurrentTimeValuePair(data, current()); + break; + } + } + } + return prev; + } + + @Override + public TimeValuePair current() { + if (current == null) { + this.current = + TimeValuePairUtils.getEmptyTimeValuePair(chunkReader.getChunkHeader().getDataType()); + TimeValuePairUtils.setCurrentTimeValuePair(data, current); + } + return current; + } + + @Override + public void close() { + this.chunkReader.close(); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/DiskChunkReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/DiskChunkReader.java index d2bae96d8213356f8600c6f0a4a6c50ae1bbda52..e272ffbeaeb2138d3404861bcc8737d8d4c157d0 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/DiskChunkReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunkRelated/DiskChunkReader.java @@ -66,6 +66,7 @@ public class DiskChunkReader implements IPointReader { @Override public TimeValuePair current() { + // FIXME: if data.hasNext() = false and this method is called... return TimeValuePairUtils.getCurrentTimeValuePair(data); } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileIterateReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileIterateReader.java index b0a3f0453da30f5d2785ad0bce3da2f2293769d6..2477076e2baf92d32347f02a66d730f490bdabcc 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileIterateReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileIterateReader.java @@ -28,7 +28,7 @@ import org.apache.iotdb.db.query.reader.chunkRelated.MemChunkReader; import org.apache.iotdb.db.query.reader.universal.IterateReader; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader; @@ -114,8 +114,8 @@ public class UnSealedTsFileIterateReader extends IterateReader { // prepare chunkLoader TsFileSequenceReader unClosedTsFileReader = FileReaderManager.getInstance() - .get(unSealedTsFile.getFile().getPath(), false); - ChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader); + .get(unSealedTsFile, false); + IChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader); // init fileSeriesReader FileSeriesReader fileSeriesReader; diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderByTimestamp.java b/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderByTimestamp.java index f9fdd2709b0870e977a56e49bf44d6c5bc5e2045..86d6d76da6bc247ec5bfec0205f9e41a677ef8ea 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderByTimestamp.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderByTimestamp.java @@ -24,7 +24,7 @@ import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.db.query.reader.IReaderByTimestamp; import org.apache.iotdb.db.query.reader.chunkRelated.MemChunkReaderByTimestamp; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderByTimestamp; @@ -64,8 +64,8 @@ public class UnSealedTsFileReaderByTimestamp implements IReaderByTimestamp { // create FileSeriesReaderByTimestamp for data which has been flushed to disk TsFileSequenceReader unClosedTsFileReader = FileReaderManager.getInstance() - .get(unsealedTsFile.getFile().getPath(), false); - ChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader); + .get(unsealedTsFile, false); + IChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader); unSealedTsFileDiskReaderByTs = new FileSeriesReaderByTimestamp(chunkLoader, unsealedTsFile.getChunkMetaDatas()); diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/CachedUnseqResourceMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/CachedUnseqResourceMergeReader.java new file mode 100644 index 0000000000000000000000000000000000000000..15525a2020f9f74c1c197292d1aaaad7324c9169 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/CachedUnseqResourceMergeReader.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.reader.resourceRelated; + +import java.io.IOException; +import java.util.List; +import org.apache.iotdb.db.query.reader.chunkRelated.CachedDiskChunkReader; +import org.apache.iotdb.db.query.reader.universal.CachedPriorityMergeReader; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; +import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter; + +public class CachedUnseqResourceMergeReader extends CachedPriorityMergeReader { + + public CachedUnseqResourceMergeReader(List chunks, TSDataType dataType) + throws IOException { + super(dataType); + int priorityValue = 1; + for (Chunk chunk : chunks) { + ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk); + addReaderWithPriority(new CachedDiskChunkReader(chunkReader), priorityValue++); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java index 9ba9128c05f5fbcca9c98ae85a61ce17673cfd00..b72dd63210ce968c9d49a4c2f8bcb5e6a3270b2c 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceIterateReader.java @@ -34,7 +34,7 @@ import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader; @@ -159,8 +159,7 @@ public class SeqResourceIterateReader extends IterateReader { QueryContext context) throws IOException { // prepare metaDataList List metaDataList = DeviceMetaDataCache.getInstance() - .get(sealedTsFile.getFile().getPath(), seriesPath); - + .get(sealedTsFile, seriesPath); List pathModifications = context.getPathModifications(sealedTsFile.getModFile(), seriesPath.getFullPath()); if (!pathModifications.isEmpty()) { @@ -172,8 +171,8 @@ public class SeqResourceIterateReader extends IterateReader { } // prepare chunkLoader TsFileSequenceReader tsFileReader = FileReaderManager.getInstance() - .get(sealedTsFile.getFile().getPath(), true); - ChunkLoader chunkLoader = new ChunkLoaderImpl(tsFileReader); + .get(sealedTsFile, true); + IChunkLoader chunkLoader = new ChunkLoaderImpl(tsFileReader); // init fileSeriesReader FileSeriesReader fileSeriesReader; diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderByTimestamp.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderByTimestamp.java index 07b2313db07718b3a308494c218bf07e71cf69ab..5b2ac5acb3ea4ae46ca995adb0895e67e33b2592 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderByTimestamp.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderByTimestamp.java @@ -32,7 +32,7 @@ import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderByTimestamp; @@ -172,7 +172,7 @@ public class SeqResourceReaderByTimestamp implements IReaderByTimestamp { QueryContext context) throws IOException { // prepare metaDataList List metaDataList = DeviceMetaDataCache.getInstance() - .get(sealedTsFile.getFile().getPath(), seriesPath); + .get(sealedTsFile, seriesPath); List pathModifications = context.getPathModifications(sealedTsFile.getModFile(), seriesPath.getFullPath()); @@ -181,8 +181,8 @@ public class SeqResourceReaderByTimestamp implements IReaderByTimestamp { } // prepare chunkLoader TsFileSequenceReader tsFileReader = FileReaderManager.getInstance() - .get(sealedTsFile.getFile().getPath(), true); - ChunkLoader chunkLoader = new ChunkLoaderImpl(tsFileReader); + .get(sealedTsFile, true); + IChunkLoader chunkLoader = new ChunkLoaderImpl(tsFileReader); return new FileSeriesReaderByTimestampAdapter( new FileSeriesReaderByTimestamp(chunkLoader, metaDataList)); diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java index 0d3eb663d8304e3fbdd0d47f7bb8054000b9f23b..9d8cf965bd869dac1baa2c2077c9c6c143ba0163 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java @@ -16,9 +16,11 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.query.reader.resourceRelated; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache; import org.apache.iotdb.db.engine.modification.Modification; @@ -29,8 +31,8 @@ import org.apache.iotdb.db.query.reader.chunkRelated.DiskChunkReader; import org.apache.iotdb.db.query.reader.chunkRelated.MemChunkReader; import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader; import org.apache.iotdb.db.utils.QueryUtils; -import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; @@ -68,8 +70,9 @@ public class UnseqResourceMergeReader extends PriorityMergeReader { if (isTsFileNotSatisfied(tsFileResource, filter)) { continue; } + metaDataList = DeviceMetaDataCache.getInstance() - .get(tsFileResource.getFile().getPath(), seriesPath); + .get(tsFileResource, seriesPath); List pathModifications = context .getPathModifications(tsFileResource.getModFile(), seriesPath.getFullPath()); if (!pathModifications.isEmpty()) { @@ -88,18 +91,23 @@ public class UnseqResourceMergeReader extends PriorityMergeReader { if (!metaDataList.isEmpty()) { // create and add ChunkReader with priority TsFileSequenceReader tsFileReader = FileReaderManager.getInstance() - .get(tsFileResource.getFile().getPath(), tsFileResource.isClosed()); + .get(tsFileResource, tsFileResource.isClosed()); chunkLoader = new ChunkLoaderImpl(tsFileReader); } for (ChunkMetaData chunkMetaData : metaDataList) { if (filter != null) { + ByteBuffer minValue = null; + ByteBuffer maxValue = null; + ByteBuffer[] statistics = chunkMetaData.getDigest().getStatistics(); + if (statistics != null) { + minValue = statistics[StatisticType.min_value.ordinal()]; // note still CAN be null + maxValue = statistics[StatisticType.max_value.ordinal()]; // note still CAN be null + } + DigestForFilter digest = new DigestForFilter(chunkMetaData.getStartTime(), - chunkMetaData.getEndTime(), - chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MIN_VALUE), - chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MAX_VALUE), - chunkMetaData.getTsDataType()); + chunkMetaData.getEndTime(), minValue, maxValue, chunkMetaData.getTsDataType()); if (!filter.satisfy(digest)) { continue; } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderByTimestamp.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderByTimestamp.java index 6a6123dfbce4abc6858b6826bb9606d26bd9d640..36a5e6498407ff96e2f3c51ee973f5e6370e63b8 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderByTimestamp.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderByTimestamp.java @@ -57,7 +57,7 @@ public class UnseqResourceReaderByTimestamp extends PriorityMergeReaderByTimesta List metaDataList; if (tsFileResource.isClosed()) { metaDataList = DeviceMetaDataCache.getInstance() - .get(tsFileResource.getFile().getPath(), seriesPath); + .get(tsFileResource, seriesPath); List pathModifications = context .getPathModifications(tsFileResource.getModFile(), seriesPath.getFullPath()); if (!pathModifications.isEmpty()) { @@ -71,7 +71,7 @@ public class UnseqResourceReaderByTimestamp extends PriorityMergeReaderByTimesta if (!metaDataList.isEmpty()) { // create and add ChunkReader with priority TsFileSequenceReader tsFileReader = FileReaderManager.getInstance() - .get(tsFileResource.getFile().getPath(), tsFileResource.isClosed()); + .get(tsFileResource, tsFileResource.isClosed()); chunkLoader = new ChunkLoaderImpl(tsFileReader); } for (ChunkMetaData chunkMetaData : metaDataList) { diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/universal/CachedPriorityMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/universal/CachedPriorityMergeReader.java new file mode 100644 index 0000000000000000000000000000000000000000..12e8f137da292561c1dd75fd5ac357defebdd12a --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/universal/CachedPriorityMergeReader.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.query.reader.universal; + +import java.io.IOException; +import org.apache.iotdb.db.utils.TimeValuePair; +import org.apache.iotdb.db.utils.TimeValuePairUtils; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; + +/** + * CachedPriorityMergeReader use a cache to reduce unnecessary heap updates and increase locality. + */ +public class CachedPriorityMergeReader extends PriorityMergeReader { + + private static final int CACHE_SIZE = 100; + + private TimeValuePair[] timeValuePairCache = new TimeValuePair[CACHE_SIZE]; + private int cacheLimit = 0; + private int cacheIdx = 0; + private TSDataType dataType; + + public CachedPriorityMergeReader(TSDataType dataType) { + this.dataType = dataType; + for (int i = 0; i < CACHE_SIZE; i++) { + timeValuePairCache[i] = TimeValuePairUtils.getEmptyTimeValuePair(dataType); + } + } + + @Override + public boolean hasNext() { + return cacheIdx < cacheLimit || !heap.isEmpty(); + } + + private void fetch() throws IOException { + cacheLimit = 0; + cacheIdx = 0; + while (!heap.isEmpty() && cacheLimit < CACHE_SIZE) { + Element top = heap.poll(); + if (cacheLimit == 0 || top.currTime() != timeValuePairCache[cacheLimit - 1].getTimestamp()) { + TimeValuePairUtils.setTimeValuePair(top.timeValuePair, timeValuePairCache[cacheLimit++]); + if (top.hasNext()) { + top.next(); + heap.add(top); + } else { + top.close(); + } + } else if (top.currTime() == timeValuePairCache[cacheLimit - 1].getTimestamp()) { + if (top.hasNext()) { + top.next(); + heap.add(top); + } else { + top.close(); + } + } + } + } + + @Override + public TimeValuePair next() throws IOException { + TimeValuePair ret; + if (cacheIdx < cacheLimit) { + ret = timeValuePairCache[cacheIdx++]; + } else { + fetch(); + ret = timeValuePairCache[cacheIdx++]; + } + return ret; + } + + @Override + public TimeValuePair current() { + if (0 <= cacheIdx && cacheIdx < cacheLimit) { + return timeValuePairCache[cacheIdx]; + } else { + return heap.peek().timeValuePair; + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/universal/PriorityMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/universal/PriorityMergeReader.java index 106a9061d7c07171045b8e0b16ece92a6dfa74d3..bb24010289a33f6a4a1794ae8efce0614b2d65de 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/universal/PriorityMergeReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/universal/PriorityMergeReader.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.query.reader.universal; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.PriorityQueue; import org.apache.iotdb.db.query.reader.IPointReader; import org.apache.iotdb.db.utils.TimeValuePair; @@ -30,16 +28,18 @@ import org.apache.iotdb.db.utils.TimeValuePair; */ public class PriorityMergeReader implements IPointReader { - private List readerList = new ArrayList<>(); - private List priorityList = new ArrayList<>(); - private PriorityQueue heap = new PriorityQueue<>(); + PriorityQueue heap = new PriorityQueue<>((o1, o2) -> { + int timeCompare = Long.compare(o1.timeValuePair.getTimestamp(), + o2.timeValuePair.getTimestamp()); + return timeCompare != 0 ? timeCompare : Integer.compare(o2.priority, o1.priority); + }); public void addReaderWithPriority(IPointReader reader, int priority) throws IOException { if (reader.hasNext()) { - heap.add(new Element(readerList.size(), reader.next(), priority)); + heap.add(new Element(reader, reader.next(), priority)); + } else { + reader.close(); } - readerList.add(reader); - priorityList.add(priority); } @Override @@ -49,9 +49,20 @@ public class PriorityMergeReader implements IPointReader { @Override public TimeValuePair next() throws IOException { - Element top = heap.peek(); - updateHeap(top); - return top.timeValuePair; + Element top = heap.poll(); + TimeValuePair ret = top.timeValuePair; + TimeValuePair topNext = null; + if (top.hasNext()) { + top.next(); + topNext = top.currPair(); + } + long topNextTime = topNext == null ? Long.MAX_VALUE : topNext.getTimestamp(); + updateHeap(ret.getTimestamp(), topNextTime); + if (topNext != null) { + top.timeValuePair = topNext; + heap.add(top); + } + return ret; } @Override @@ -59,66 +70,68 @@ public class PriorityMergeReader implements IPointReader { return heap.peek().timeValuePair; } - private void updateHeap(Element top) throws IOException { - while (!heap.isEmpty() && heap.peek().timeValuePair.getTimestamp() == top.timeValuePair - .getTimestamp()) { + private void updateHeap(long topTime, long topNextTime) throws IOException { + while (!heap.isEmpty() && heap.peek().currTime() == topTime) { Element e = heap.poll(); - IPointReader reader = readerList.get(e.index); - if (reader.hasNext()) { - heap.add(new Element(e.index, reader.next(), priorityList.get(e.index))); + if (!e.hasNext()) { + e.reader.close(); + continue; + } + + e.next(); + if (e.currTime() == topNextTime) { + // if the next value of the peek will be overwritten by the next of the top, skip it + if (e.hasNext()) { + e.next(); + heap.add(e); + } else { + // the chunk is end + e.close(); + } + } else { + heap.add(e); } } } @Override public void close() throws IOException { - for (IPointReader reader : readerList) { - reader.close(); + while (!heap.isEmpty()) { + Element e = heap.poll(); + e.close(); } } - protected class Element implements Comparable { + class Element { - int index; + IPointReader reader; TimeValuePair timeValuePair; - Integer priority; + int priority; - public Element(int index, TimeValuePair timeValuePair, int priority) { - this.index = index; + Element(IPointReader reader, TimeValuePair timeValuePair, int priority) { + this.reader = reader; this.timeValuePair = timeValuePair; this.priority = priority; } - @Override - public int compareTo( - Element o) { - - if (this.timeValuePair.getTimestamp() > o.timeValuePair.getTimestamp()) { - return 1; - } + long currTime() { + return timeValuePair.getTimestamp(); + } - if (this.timeValuePair.getTimestamp() < o.timeValuePair.getTimestamp()) { - return -1; - } + TimeValuePair currPair() { + return timeValuePair; + } - return o.priority.compareTo(this.priority); + boolean hasNext() throws IOException { + return reader.hasNext(); } - @Override - public boolean equals(Object o) { - if (o instanceof Element) { - Element element = (Element) o; - if (this.timeValuePair.getTimestamp() == element.timeValuePair.getTimestamp() - && this.priority.equals(element.priority)) { - return true; - } - } - return false; + void next() throws IOException { + timeValuePair = reader.next(); } - @Override - public int hashCode() { - return (int) (timeValuePair.getTimestamp() * 31 + priority.hashCode()); + void close() throws IOException { + reader.close(); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/MemTablePool.java b/server/src/main/java/org/apache/iotdb/db/rescon/MemTablePool.java index 210ee9949e69c005916d9350bc39c51bd4e0d230..49188d1b58df715449216b3687dbf3bbcc6c6812 100644 --- a/server/src/main/java/org/apache/iotdb/db/rescon/MemTablePool.java +++ b/server/src/main/java/org/apache/iotdb/db/rescon/MemTablePool.java @@ -42,6 +42,7 @@ public class MemTablePool { private MemTablePool() { } + // TODO change the impl of getAvailableMemTable to non-blocking public IMemTable getAvailableMemTable(Object applier) { synchronized (availableMemTables) { if (availableMemTables.isEmpty() && size < CONFIG.getMaxMemtableNumber()) { diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index 42f3f6d3a795d3692fe8b89c3cd7cc16aab4aa6a..18f7e8d6ca90bd6e41add11f6dd82f3959c96e9f 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.conf.adapter.ManageDynamicParameters; import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter; import org.apache.iotdb.db.cost.statistic.Measurement; import org.apache.iotdb.db.engine.StorageEngine; +import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.flush.FlushManager; import org.apache.iotdb.db.exception.StartupException; import org.apache.iotdb.db.metadata.MManager; @@ -80,10 +81,6 @@ public class IoTDB implements IoTDBMBean { Runtime.getRuntime().addShutdownHook(new IoTDBShutdownHook()); setUncaughtExceptionHandler(); - boolean enableWAL = IoTDBDescriptor.getInstance().getConfig().isEnableWal(); - IoTDBDescriptor.getInstance().getConfig().setEnableWal(false); - IoTDBDescriptor.getInstance().getConfig().setEnableWal(enableWAL); - // When registering statMonitor, we should start recovering some statistics // with latest values stored // Warn: registMonitor() method should be called after systemDataRecovery() @@ -103,6 +100,7 @@ public class IoTDB implements IoTDBMBean { registerManager.register(SyncServerManager.getInstance()); registerManager.register(TVListAllocator.getInstance()); registerManager.register(FlushManager.getInstance()); + registerManager.register(MergeManager.getINSTANCE()); JMXService.registerMBean(getInstance(), mbeanName); diff --git a/server/src/main/java/org/apache/iotdb/db/service/JDBCService.java b/server/src/main/java/org/apache/iotdb/db/service/JDBCService.java index 132ddebd79ab820af810221e50c5fbed5f6e37bf..45981f8d820bfe248a263aab01fcfee27038fa3c 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/JDBCService.java +++ b/server/src/main/java/org/apache/iotdb/db/service/JDBCService.java @@ -210,7 +210,8 @@ public class JDBCService implements JDBCServiceMBean, IService { IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); serverTransport = new TServerSocket(new InetSocketAddress(config.getRpcAddress(), config.getRpcPort())); - poolArgs = new TThreadPoolServer.Args(serverTransport); + poolArgs = new TThreadPoolServer.Args(serverTransport).maxWorkerThreads(IoTDBDescriptor. + getInstance().getConfig().getMaxConcurrentClientNum()).minWorkerThreads(1); poolArgs.executorService = IoTDBThreadPoolFactory.createThriftRpcClientThreadPool(poolArgs, ThreadName.JDBC_CLIENT.getName()); poolArgs.processor(processor); diff --git a/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java b/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java index 71047631a9d16c5cbbc5be58b04d8c3320c170c8..2212c3009e7d50ea21a03509fc6e9a0cc20ec12c 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java +++ b/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java @@ -31,6 +31,7 @@ public enum ServiceType { AUTHORIZATION_SERVICE("Authorization ServerService", ""), FILE_READER_MANAGER_SERVICE("File reader manager ServerService", ""), SYNC_SERVICE("SYNC ServerService", ""), + MERGE_SERVICE("Merge Manager", ""), PERFORMANCE_STATISTIC_SERVICE("PERFORMANCE_STATISTIC_SERVICE","PERFORMANCE_STATISTIC_SERVICE"), MANAGE_DYNAMIC_PARAMETERS_SERVICE("Manage Dynamic Parameters", "Manage Dynamic Parameters"), TVLIST_ALLOCATOR_SERVICE("TVList Allocator", ""), @@ -41,7 +42,7 @@ public enum ServiceType { private String name; private String jmxName; - private ServiceType(String name, String jmxName) { + ServiceType(String name, String jmxName) { this.name = name; this.jmxName = jmxName; } diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index 531719fa43f516a9628b56bab0351c66d10da667..9856e8af78ae7d189d489d8fd8b68b0fdbf25f0a 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -18,23 +18,6 @@ */ package org.apache.iotdb.db.service; -import static org.apache.iotdb.db.conf.IoTDBConstant.PRIVILEGE; -import static org.apache.iotdb.db.conf.IoTDBConstant.ROLE; -import static org.apache.iotdb.db.conf.IoTDBConstant.USER; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.regex.Pattern; import org.apache.iotdb.db.auth.AuthException; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.auth.authorizer.IAuthorizer; @@ -48,56 +31,28 @@ import org.apache.iotdb.db.cost.statistic.Measurement; import org.apache.iotdb.db.cost.statistic.Operation; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.flush.pool.FlushTaskPoolManager; -import org.apache.iotdb.db.exception.ArgsErrorException; -import org.apache.iotdb.db.exception.MetadataErrorException; -import org.apache.iotdb.db.exception.PathErrorException; -import org.apache.iotdb.db.exception.ProcessorException; -import org.apache.iotdb.db.exception.QueryInBatchStmtException; -import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.exception.*; import org.apache.iotdb.db.exception.qp.IllegalASTFormatException; import org.apache.iotdb.db.exception.qp.QueryProcessorException; import org.apache.iotdb.db.metadata.MManager; import org.apache.iotdb.db.metadata.Metadata; import org.apache.iotdb.db.qp.QueryProcessor; import org.apache.iotdb.db.qp.executor.QueryProcessExecutor; +import org.apache.iotdb.db.qp.logical.sys.MetadataOperator; import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.qp.physical.sys.AuthorPlan; +import org.apache.iotdb.db.qp.physical.sys.MetadataPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.QueryResourceManager; import org.apache.iotdb.db.utils.QueryDataSetUtils; -import org.apache.iotdb.service.rpc.thrift.ServerProperties; -import org.apache.iotdb.service.rpc.thrift.TSCancelOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCancelOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq; -import org.apache.iotdb.service.rpc.thrift.TSCloseSessionResp; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp; -import org.apache.iotdb.service.rpc.thrift.TSGetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TSHandleIdentifier; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSInsertionReq; -import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; -import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; -import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.thrift.TException; @@ -105,6 +60,18 @@ import org.apache.thrift.server.ServerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.ZoneId; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; + +import static org.apache.iotdb.db.conf.IoTDBConstant.*; + /** * Thrift RPC implementation at server side. */ @@ -113,7 +80,6 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { private static final Logger logger = LoggerFactory.getLogger(TSServiceImpl.class); private static final String INFO_NOT_LOGIN = "{}: Not login."; - private static final String ERROR_NOT_LOGIN = "Not login"; protected QueryProcessor processor; // Record the username for every rpc connection. Username.get() is null if @@ -154,23 +120,20 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } TS_Status tsStatus; if (status) { - tsStatus = new TS_Status(TS_StatusCode.SUCCESS_STATUS); - tsStatus.setErrorMessage("login successfully."); + tsStatus = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS, "Login successfully")); username.set(req.getUsername()); zoneIds.set(config.getZoneID()); initForOneSession(); } else { - tsStatus = new TS_Status(TS_StatusCode.ERROR_STATUS); - tsStatus.setErrorMessage("login failed. Username or password is wrong."); + tsStatus = getStatus(TSStatusType.WRONG_LOGIN_PASSWORD_ERROR); } TSOpenSessionResp resp = new TSOpenSessionResp(tsStatus, - TSProtocolVersion.TSFILE_SERVICE_PROTOCOL_V1); + TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1); resp.setSessionHandle( new TS_SessionHandle(new TSHandleIdentifier(ByteBuffer.wrap(req.getUsername().getBytes()), ByteBuffer.wrap(req.getPassword().getBytes())))); logger.info("{}: Login status: {}. User : {}", IoTDBConstant.GLOBAL_DB_NAME, - tsStatus.getErrorMessage(), - req.getUsername()); + tsStatus.getStatusType().getMessage(), req.getUsername()); return resp; } @@ -181,32 +144,31 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } @Override - public TSCloseSessionResp closeSession(TSCloseSessionReq req) { + public TSRPCResp closeSession(TSCloseSessionReq req) { logger.info("{}: receive close session", IoTDBConstant.GLOBAL_DB_NAME); TS_Status tsStatus; if (username.get() == null) { - tsStatus = new TS_Status(TS_StatusCode.ERROR_STATUS); - tsStatus.setErrorMessage("Has not logged in"); + tsStatus = getStatus(TSStatusType.NOT_LOGIN_ERROR); if (zoneIds.get() != null) { zoneIds.remove(); } } else { - tsStatus = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + tsStatus = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); username.remove(); if (zoneIds.get() != null) { zoneIds.remove(); } } - return new TSCloseSessionResp(tsStatus); + return new TSRPCResp(tsStatus); } @Override - public TSCancelOperationResp cancelOperation(TSCancelOperationReq req) { - return new TSCancelOperationResp(new TS_Status(TS_StatusCode.SUCCESS_STATUS)); + public TSRPCResp cancelOperation(TSCancelOperationReq req) { + return new TSRPCResp(getStatus(TSStatusType.SUCCESS_STATUS)); } @Override - public TSCloseOperationResp closeOperation(TSCloseOperationReq req) { + public TSRPCResp closeOperation(TSCloseOperationReq req) { logger.info("{}: receive close operation", IoTDBConstant.GLOBAL_DB_NAME); try { @@ -221,7 +183,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } catch (Exception e) { logger.error("Error in closeOperation : ", e); } - return new TSCloseOperationResp(new TS_Status(TS_StatusCode.SUCCESS_STATUS)); + return new TSRPCResp(getStatus(TSStatusType.SUCCESS_STATUS)); } private void releaseQueryResource(TSCloseOperationReq req) throws StorageEngineException { @@ -250,9 +212,29 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } } - private TS_Status getErrorStatus(String message) { - TS_Status status = new TS_Status(TS_StatusCode.ERROR_STATUS); - status.setErrorMessage(message); + /** + * convert from TSStatusType to TS_Status according to status code and status message + * + * @param statusType status type + * @return + */ + private TS_Status getStatus(TSStatusType statusType) { + TS_StatusType statusCodeAndMessage = new TS_StatusType(statusType.getStatusCode(), statusType.getStatusMessage()); + TS_Status status = new TS_Status(statusCodeAndMessage); + return status; + } + + /** + * convert from TSStatusType to TS_Status, which has message appending with existed status message + * + * @param statusType status type + * @param appendMessage appending message + * @return + */ + private TS_Status getStatus(TSStatusType statusType, String appendMessage) { + TS_StatusType statusCodeAndMessage = new TS_StatusType(statusType.getStatusCode(), + statusType.getStatusMessage() + ": " + appendMessage); + TS_Status status = new TS_Status(statusCodeAndMessage); return status; } @@ -261,7 +243,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { TS_Status status; if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - status = getErrorStatus(ERROR_NOT_LOGIN); + status = getStatus(TSStatusType.NOT_LOGIN_ERROR); return new TSFetchMetadataResp(status); } TSFetchMetadataResp resp = new TSFetchMetadataResp(); @@ -271,17 +253,17 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { String path = req.getColumnPath(); List> showTimeseriesList = getTimeSeriesForPath(path); resp.setShowTimeseriesList(showTimeseriesList); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "SHOW_STORAGE_GROUP": Set storageGroups = getAllStorageGroups(); resp.setShowStorageGroups(storageGroups); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "METADATA_IN_JSON": String metadataInJson = getMetadataInString(); resp.setMetadataInJson(metadataInJson); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "DELTA_OBEJECT": Metadata metadata = getMetadata(); @@ -292,21 +274,18 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } else { resp.setColumnsList(deviceMap.get(column)); } - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "COLUMN": resp.setDataType(getSeriesType(req.getColumnPath()).toString()); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "ALL_COLUMNS": resp.setColumnsList(getPaths(req.getColumnPath())); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; default: - status = new TS_Status(TS_StatusCode.ERROR_STATUS); - status - .setErrorMessage( - String.format("Unsupported fetch metadata operation %s", req.getType())); + status = getStatus(TSStatusType.FETCH_METADATA_ERROR, req.getType()); break; } } catch (PathErrorException | MetadataErrorException | OutOfMemoryError e) { @@ -314,8 +293,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { .error(String.format("Failed to fetch timeseries %s's metadata", req.getColumnPath()), e); Thread.currentThread().interrupt(); - status = getErrorStatus( - String.format("Failed to fetch metadata because: %s", e)); + status = getStatus(TSStatusType.FETCH_METADATA_ERROR, e.getMessage()); resp.setStatus(status); return resp; } @@ -391,7 +369,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { * @return true if the statement is ADMIN COMMAND * @throws IOException exception */ - private boolean execAdminCommand(String statement) { + private boolean execAdminCommand(String statement) throws StorageEngineException { if (!"root".equals(username.get())) { return false; } @@ -404,8 +382,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { StorageEngine.getInstance().syncCloseAllProcessor(); return true; case "merge": - // TODO change to merge!!! - throw new UnsupportedOperationException("merge not implemented"); + StorageEngine.getInstance() + .mergeAll(IoTDBDescriptor.getInstance().getConfig().isForceFullMerge()); + return true; + case "full merge": + StorageEngine.getInstance().mergeAll(true); + return true; default: return false; } @@ -418,7 +400,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { try { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN, null); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR), null); } List statements = req.getStatements(); @@ -433,18 +415,17 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } if (isAllSuccessful) { - return getTSBathExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, - "Execute batch statements successfully", result); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, + "Execute batch statements successfully"), result); } else { - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - batchErrorMessage.toString(), - result); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, + batchErrorMessage.toString()), result); } } catch (Exception e) { logger.error("{}: error occurs when executing statements", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage(), null); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage()), null); } finally { - Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_BATCH, t1); + Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_JDBC_BATCH, t1); } } @@ -458,11 +439,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { throw new QueryInBatchStmtException("Query statement not allowed in batch: " + statement); } TSExecuteStatementResp resp = executeUpdateStatement(physicalPlan); - if (resp.getStatus().getStatusCode().equals(TS_StatusCode.SUCCESS_STATUS)) { + if (resp.getStatus().getStatusType().getCode() == TSStatusType.SUCCESS_STATUS.getStatusCode()) { result.add(Statement.SUCCESS_NO_INFO); } else { result.add(Statement.EXECUTE_FAILED); - batchErrorMessage.append(resp.getStatus().getErrorMessage()).append("\n"); + batchErrorMessage.append(resp.getStatus().getStatusType().getCode()).append("\n"); return false; } } catch (Exception e) { @@ -484,12 +465,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { try { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN); + return getTSExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } String statement = req.getStatement(); if (execAdminCommand(statement)) { - return getTSExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, "ADMIN_COMMAND_SUCCESS"); + return getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, "ADMIN_COMMAND_SUCCESS")); } if (execShowFlushInfo(statement)) { @@ -498,7 +479,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { FlushTaskPoolManager.getInstance().getTotalTasks(), FlushTaskPoolManager.getInstance().getWorkingTasksNumber(), FlushTaskPoolManager.getInstance().getWaitingTasksNumber()); - return getTSExecuteStatementResp(TS_StatusCode.SUCCESS_WITH_INFO_STATUS, msg); + return getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, msg)); } if (execShowDynamicParameters(statement)) { @@ -512,12 +493,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { MManager.getInstance().getAllStorageGroup().size(), IoTDBConfigDynamicAdapter.getInstance().getTotalTimeseries(), MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups()); - return getTSExecuteStatementResp(TS_StatusCode.SUCCESS_WITH_INFO_STATUS, msg); + return getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, msg)); } if (execSetConsistencyLevel(statement)) { - return getTSExecuteStatementResp(TS_StatusCode.SUCCESS_WITH_INFO_STATUS, - "Execute set consistency level successfully"); + return getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, + "Execute set consistency level successfully")); } PhysicalPlan physicalPlan; @@ -529,11 +510,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } } catch (IllegalASTFormatException e) { logger.debug("meet error while parsing SQL to physical plan: ", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Statement format is not right:" + e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.SQL_PARSE_ERROR, + "Statement format is not right: " + e.getMessage())); } catch (Exception e) { logger.info("meet error while executing statement: {}", req.getStatement(), e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage())); } } @@ -605,7 +586,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { return resp; } catch (Exception e) { logger.error("{}: Internal server error: ", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.INTERNAL_SERVER_ERROR, e.getMessage())); } finally { Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_QUERY, t1); } @@ -615,7 +596,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { public TSExecuteStatementResp executeQueryStatement(TSExecuteStatementReq req) { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN); + return getTSExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } String statement = req.getStatement(); @@ -624,12 +605,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { physicalPlan = processor.parseSQLToPhysicalPlan(statement, zoneIds.get()); } catch (QueryProcessorException | ArgsErrorException | MetadataErrorException e) { logger.error("meet error while parsing SQL to physical plan!", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.SQL_PARSE_ERROR, e.getMessage())); } if (!physicalPlan.isQuery()) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Statement is not a query statement."); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, + "Statement is not a query statement.")); } return executeQueryStatement(statement, physicalPlan); } @@ -643,7 +624,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } private TSExecuteStatementResp executeAuthQuery(PhysicalPlan plan, List columns) { - TSExecuteStatementResp resp = getTSExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, ""); + TSExecuteStatementResp resp = getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS)); resp.setIgnoreTimeStamp(true); AuthorPlan authorPlan = (AuthorPlan) plan; switch (authorPlan.getAuthorType()) { @@ -667,8 +648,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { columns.add(PRIVILEGE); break; default: - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, String.format("%s is not an " - + "auth query", authorPlan.getAuthorType())); + return getTSExecuteStatementResp(getStatus(TSStatusType.SQL_PARSE_ERROR, + String.format("%s is not an auth query", authorPlan.getAuthorType()))); } return resp; } @@ -679,7 +660,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { // check seriesPath exists if (paths.isEmpty()) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, "Timeseries does not exist."); + return getTSExecuteStatementResp(getStatus(TSStatusType.TIMESERIES_NOT_EXIST_ERROR)); } // check file level set @@ -687,16 +668,15 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { checkFileLevelSet(paths); } catch (PathErrorException e) { logger.error("meet error while checking file level.", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.CHECK_FILE_LEVEL_ERROR, e.getMessage())); } // check permissions if (!checkAuthorization(paths, plan)) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "No permissions for this query."); + return getTSExecuteStatementResp(getStatus(TSStatusType.NO_PERMISSION_ERROR)); } - TSExecuteStatementResp resp = getTSExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, ""); + TSExecuteStatementResp resp = getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS)); // Restore column header of aggregate to func(column_name), only // support single aggregate function for now switch (plan.getOperatorType()) { @@ -732,12 +712,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { public TSFetchResultsResp fetchResults(TSFetchResultsReq req) { try { if (!checkLogin()) { - return getTSFetchResultsResp(TS_StatusCode.ERROR_STATUS, "Not login."); + return getTSFetchResultsResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } String statement = req.getStatement(); if (!queryStatus.get().containsKey(statement)) { - return getTSFetchResultsResp(TS_StatusCode.ERROR_STATUS, "Has not executed statement"); + return getTSFetchResultsResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, "Has not executed statement")); } QueryDataSet queryDataSet; @@ -756,14 +736,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { queryRet.get().remove(statement); } - TSFetchResultsResp resp = getTSFetchResultsResp(TS_StatusCode.SUCCESS_STATUS, - "FetchResult successfully. Has more result: " + hasResultSet); + TSFetchResultsResp resp = getTSFetchResultsResp(getStatus(TSStatusType.SUCCESS_STATUS, + "FetchResult successfully. Has more result: " + hasResultSet)); resp.setHasResultSet(hasResultSet); resp.setQueryDataSet(result); return resp; } catch (Exception e) { logger.error("{}: Internal server error: ", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSFetchResultsResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSFetchResultsResp(getStatus(TSStatusType.INTERNAL_SERVER_ERROR, e.getMessage())); } } @@ -798,42 +778,24 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { try { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN); + return getTSExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } String statement = req.getStatement(); return executeUpdateStatement(statement); } catch (Exception e) { logger.error("{}: server Internal Error: ", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.INTERNAL_SERVER_ERROR, e.getMessage())); } } private TSExecuteStatementResp executeUpdateStatement(PhysicalPlan plan) { - List paths = plan.getPaths(); - try { - if (!checkAuthorization(paths, plan)) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "No permissions for this operation " + plan.getOperatorType()); - } - } catch (AuthException e) { - logger.error("meet error while checking authorization.", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Uninitialized authorizer " + e.getMessage()); - } - // TODO - // In current version, we only return OK/ERROR - // Do we need to add extra information of executive condition - boolean execRet; - try { - execRet = executeNonQuery(plan); - } catch (ProcessorException e) { - logger.debug("meet error while processing non-query. ", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + TS_Status status = checkAuthority(plan); + if (status != null) { + return new TSExecuteStatementResp(status); } - TS_StatusCode statusCode = execRet ? TS_StatusCode.SUCCESS_STATUS : TS_StatusCode.ERROR_STATUS; - String msg = execRet ? "Execute successfully" : "Execute statement error."; - TSExecuteStatementResp resp = getTSExecuteStatementResp(statusCode, msg); + status = executePlan(plan); + TSExecuteStatementResp resp = getTSExecuteStatementResp(status); TSHandleIdentifier operationId = new TSHandleIdentifier( ByteBuffer.wrap(username.get().getBytes()), ByteBuffer.wrap("PASS".getBytes())); @@ -858,12 +820,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { physicalPlan = processor.parseSQLToPhysicalPlan(statement, zoneIds.get()); } catch (QueryProcessorException | ArgsErrorException | MetadataErrorException e) { logger.error("meet error while parsing SQL to physical plan!", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.SQL_PARSE_ERROR, e.getMessage())); } if (physicalPlan.isQuery()) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Statement is a query statement."); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, + "Statement is a query statement.")); } return executeUpdateStatement(physicalPlan); @@ -892,10 +854,9 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { return AuthorityChecker.check(username.get(), paths, plan.getOperatorType(), targetUser); } - private TSExecuteStatementResp getTSExecuteStatementResp(TS_StatusCode code, String msg) { + private TSExecuteStatementResp getTSExecuteStatementResp(TS_Status status) { TSExecuteStatementResp resp = new TSExecuteStatementResp(); - TS_Status tsStatus = new TS_Status(code); - tsStatus.setErrorMessage(msg); + TS_Status tsStatus = new TS_Status(status); resp.setStatus(tsStatus); TSHandleIdentifier operationId = new TSHandleIdentifier( ByteBuffer.wrap(username.get().getBytes()), @@ -905,21 +866,17 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { return resp; } - private TSExecuteBatchStatementResp getTSBathExecuteStatementResp(TS_StatusCode code, - String msg, - List result) { + private TSExecuteBatchStatementResp getTSBatchExecuteStatementResp(TS_Status status, List result) { TSExecuteBatchStatementResp resp = new TSExecuteBatchStatementResp(); - TS_Status tsStatus = new TS_Status(code); - tsStatus.setErrorMessage(msg); + TS_Status tsStatus = new TS_Status(status); resp.setStatus(tsStatus); resp.setResult(result); return resp; } - private TSFetchResultsResp getTSFetchResultsResp(TS_StatusCode code, String msg) { + private TSFetchResultsResp getTSFetchResultsResp(TS_Status status) { TSFetchResultsResp resp = new TSFetchResultsResp(); - TS_Status tsStatus = new TS_Status(code); - tsStatus.setErrorMessage(msg); + TS_Status tsStatus = new TS_Status(status); resp.setStatus(tsStatus); return resp; } @@ -934,30 +891,28 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { TS_Status tsStatus; TSGetTimeZoneResp resp; try { - tsStatus = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + tsStatus = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); resp = new TSGetTimeZoneResp(tsStatus, zoneIds.get().toString()); } catch (Exception e) { logger.error("meet error while generating time zone.", e); - tsStatus = new TS_Status(TS_StatusCode.ERROR_STATUS); - tsStatus.setErrorMessage(e.getMessage()); + tsStatus = getStatus(TSStatusType.GENERATE_TIME_ZONE_ERROR); resp = new TSGetTimeZoneResp(tsStatus, "Unknown time zone"); } return resp; } @Override - public TSSetTimeZoneResp setTimeZone(TSSetTimeZoneReq req) { + public TSRPCResp setTimeZone(TSSetTimeZoneReq req) { TS_Status tsStatus; try { String timeZoneID = req.getTimeZone(); zoneIds.set(ZoneId.of(timeZoneID)); - tsStatus = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + tsStatus = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); } catch (Exception e) { logger.error("meet error while setting time zone.", e); - tsStatus = new TS_Status(TS_StatusCode.ERROR_STATUS); - tsStatus.setErrorMessage(e.getMessage()); + tsStatus = getStatus(TSStatusType.SET_TIME_ZONE_ERROR); } - return new TSSetTimeZoneResp(tsStatus); + return new TSRPCResp(tsStatus); } @Override @@ -973,10 +928,10 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } @Override - public TSExecuteStatementResp executeInsertion(TSInsertionReq req) { + public TSExecuteStatementResp insert(TSInsertionReq req) { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN); + return getTSExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } long stmtId = req.getStmtId(); @@ -1000,13 +955,119 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { return executeUpdateStatement(plan); } catch (Exception e) { logger.info("meet error while executing an insertion into {}", req.getDeviceId(), e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage())); } } + @Override + public TSExecuteBatchStatementResp insertBatch(TSBatchInsertionReq req) { + long t1 = System.currentTimeMillis(); + try { + if (!checkLogin()) { + logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR), null); + } + + BatchInsertPlan batchInsertPlan = new BatchInsertPlan(req.deviceId, req.measurements); + batchInsertPlan.setTimes(QueryDataSetUtils.readTimesFromBuffer(req.timestamps, req.size)); + batchInsertPlan.setColumns(QueryDataSetUtils + .readValuesFromBuffer(req.values, req.types, req.measurements.size(), req.size)); + batchInsertPlan.setRowCount(req.size); + batchInsertPlan.setTimeBuffer(req.timestamps); + batchInsertPlan.setValueBuffer(req.values); + batchInsertPlan.setDataTypes(req.types); + + boolean isAllSuccessful = true; + TS_Status status = checkAuthority(batchInsertPlan); + if (status != null) { + return new TSExecuteBatchStatementResp(status); + } + Integer[] results = processor.getExecutor().insertBatch(batchInsertPlan); + + for (Integer result : results) { + if (result != TSStatusType.SUCCESS_STATUS.getStatusCode()) { + isAllSuccessful = false; + break; + } + } + + if (isAllSuccessful) { + logger.debug("Insert one RowBatch successfully"); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS), Arrays.asList(results)); + } else { + logger.debug("Insert one RowBatch failed!"); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.INTERNAL_SERVER_ERROR), Arrays.asList(results)); + } + } catch (Exception e) { + logger.error("{}: error occurs when executing statements", IoTDBConstant.GLOBAL_DB_NAME, e); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage()), null); + } finally { + Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_RPC_BATCH_INSERT, t1); + } + } + + @Override + public TSRPCResp setStorageGroup(TSSetStorageGroupReq req) throws TException { + if (!checkLogin()) { + logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); + return new TSRPCResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); + } + + MetadataPlan plan = new MetadataPlan(MetadataOperator.NamespaceType.SET_STORAGE_GROUP, new Path(req.getStorageGroupId())); + TS_Status status = checkAuthority(plan); + if (status != null) { + return new TSRPCResp(status); + } + return new TSRPCResp(executePlan(plan)); + } + + @Override + public TSRPCResp createTimeseries(TSCreateTimeseriesReq req) throws TException { + if (!checkLogin()) { + logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); + return new TSRPCResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); + } + MetadataPlan plan = new MetadataPlan(MetadataOperator.NamespaceType.ADD_PATH, new Path(req.getPath()), + TSDataType.values()[req.getDataType()], TSEncoding.values()[req.getEncoding()]); + TS_Status status = checkAuthority(plan); + if (status != null) { + return new TSRPCResp(status); + } + return new TSRPCResp(executePlan(plan)); + } + @Override public long requestStatementId() { return globalStmtId.incrementAndGet(); } + + private TS_Status checkAuthority(PhysicalPlan plan) { + List paths = plan.getPaths(); + try { + if (!checkAuthorization(paths, plan)) { + return getStatus(TSStatusType.NO_PERMISSION_ERROR, plan.getOperatorType().toString()); + } + } catch (AuthException e) { + logger.error("meet error while checking authorization.", e); + return getStatus(TSStatusType.UNINITIALIZED_AUTH_ERROR, e.getMessage()); + } + return null; + } + + private TS_Status executePlan(PhysicalPlan plan) { + // TODO + // In current version, we only return OK/ERROR + // Do we need to add extra information of executive condition + boolean execRet; + try { + execRet = executeNonQuery(plan); + } catch (ProcessorException e) { + logger.debug("meet error while processing non-query. ", e); + return getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + + return execRet ? getStatus(TSStatusType.SUCCESS_STATUS, "Execute successfully") + : getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java new file mode 100644 index 0000000000000000000000000000000000000000..ed384ce2abb8a3d29054c4ba141b448314de088d --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileResourcePrinter.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.tools; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.qp.constant.DatetimeUtils; + +/** + * this tool can analyze the tsfile.resource files from a folder. + */ +public class TsFileResourcePrinter { + + public static void main(String[] args) throws IOException { + + String folder = "test"; + if (args.length >= 1) { + folder = args[0]; + } + File folderFile = new File(folder); + File[] files = folderFile.listFiles((dir, name) -> name.endsWith(".tsfile.resource")); + Arrays.sort(files, (x, y) -> Long + .compare(Long.valueOf(x.getName().split("-")[0]), Long.valueOf(y.getName().split("-")[0]))); + + for (File file : files) { + printResource(file.getAbsolutePath()); + } + System.out.println("analyzing the resource file finished."); + } + + public static void printResource(String filename) throws IOException { + filename = filename.substring(0, filename.length() - 9); + TsFileResource resource = new TsFileResource(new File(filename)); + System.err.println(String.format("analyzing %s ...", filename)); + resource.deSerialize(); + + for (String device : resource.getStartTimeMap().keySet()) { + System.out.println(String.format("device %s, start time %d (%s), end time %d (%s)", device, + resource.getStartTimeMap().get(device), + DatetimeUtils.convertMillsecondToZonedDateTime(resource.getStartTimeMap().get(device)), + resource.getEndTimeMap().get(device), + DatetimeUtils.convertMillsecondToZonedDateTime(resource.getEndTimeMap().get(device)))); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index 381d25881f795b7c75804bd5eaa330cd44e81687..23454222f06a716b42243a70bdeba4ba4cae3657 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.utils; import org.apache.iotdb.db.conf.IoTDBConstant; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.write.record.TSRecord; @@ -59,7 +60,7 @@ public class MemUtils { case BOOLEAN: memSize += 8L + 1L; break; case TEXT: - memSize += 8L + insertPlan.getValues()[i].length() * 2; break; + memSize += 8L + insertPlan.getValues()[i].length(); break; default: memSize += 8L + 8L; } @@ -67,6 +68,33 @@ public class MemUtils { return memSize; } + public static long getRecordSize(BatchInsertPlan batchInsertPlan) { + long memSize = 0; + for (int i = 0; i < batchInsertPlan.getMeasurements().length; i++) { + switch (batchInsertPlan.getDataTypes()[i]) { + case INT32: + memSize += batchInsertPlan.getRowCount() * (8L + 4L); break; + case INT64: + memSize += batchInsertPlan.getRowCount() * (8L + 8L); break; + case FLOAT: + memSize += batchInsertPlan.getRowCount() * (8L + 4L); break; + case DOUBLE: + memSize += batchInsertPlan.getRowCount() * (8L + 8L); break; + case BOOLEAN: + memSize += batchInsertPlan.getRowCount() * (8L + 1L); break; + case TEXT: + memSize += batchInsertPlan.getRowCount() * 8L; + for (int j = 0; j < batchInsertPlan.getRowCount(); j++) { + memSize += ((Binary[]) batchInsertPlan.getColumns()[i])[j].getLength(); + } + break; + default: + memSize += batchInsertPlan.getRowCount() * (8L + 8L); + } + } + return memSize; + } + /** * Calculate how much memory will be used if the given record is written to sequence file. */ diff --git a/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..c48eceb6911bd8198516f894078eeee72b469184 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java @@ -0,0 +1,306 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.modification.Modification; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; +import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; +import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter; +import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MergeUtils { + + private static final Logger logger = LoggerFactory.getLogger(MergeUtils.class); + + private MergeUtils() { + // util class + } + + public static void writeTVPair(TimeValuePair timeValuePair, IChunkWriter chunkWriter) { + switch (chunkWriter.getDataType()) { + case TEXT: + chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary()); + break; + case DOUBLE: + chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble()); + break; + case BOOLEAN: + chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean()); + break; + case INT64: + chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong()); + break; + case INT32: + chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt()); + break; + case FLOAT: + chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat()); + break; + default: + throw new UnsupportedOperationException("Unknown data type " + chunkWriter.getDataType()); + } + } + + private static List collectFileSeries(TsFileSequenceReader sequenceReader) throws IOException { + TsFileMetaData metaData = sequenceReader.readFileMetadata(); + Set deviceIds = metaData.getDeviceMap().keySet(); + Set measurements = metaData.getMeasurementSchema().keySet(); + List paths = new ArrayList<>(); + for (String deviceId : deviceIds) { + for (String measurement : measurements) { + paths.add(new Path(deviceId, measurement)); + } + } + return paths; + } + + public static long collectFileSizes(List seqFiles, List unseqFiles) { + long totalSize = 0; + for (TsFileResource tsFileResource : seqFiles) { + totalSize += tsFileResource.getFileSize(); + } + for (TsFileResource tsFileResource : unseqFiles) { + totalSize += tsFileResource.getFileSize(); + } + return totalSize; + } + + public static int writeChunkWithoutUnseq(Chunk chunk, IChunkWriter chunkWriter) throws IOException { + ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk); + int ptWritten = 0; + while (chunkReader.hasNextBatch()) { + BatchData batchData = chunkReader.nextBatch(); + for (int i = 0; i < batchData.length(); i++) { + writeBatchPoint(batchData, i, chunkWriter); + } + ptWritten += batchData.length(); + } + return ptWritten; + } + + public static void writeBatchPoint(BatchData batchData, int i, IChunkWriter chunkWriter) { + switch (chunkWriter.getDataType()) { + case TEXT: + chunkWriter.write(batchData.getTimeByIndex(i), batchData.getBinaryByIndex(i)); + break; + case DOUBLE: + chunkWriter.write(batchData.getTimeByIndex(i), batchData.getDoubleByIndex(i)); + break; + case BOOLEAN: + chunkWriter.write(batchData.getTimeByIndex(i), batchData.getBooleanByIndex(i)); + break; + case INT64: + chunkWriter.write(batchData.getTimeByIndex(i), batchData.getLongByIndex(i)); + break; + case INT32: + chunkWriter.write(batchData.getTimeByIndex(i), batchData.getIntByIndex(i)); + break; + case FLOAT: + chunkWriter.write(batchData.getTimeByIndex(i), batchData.getFloatByIndex(i)); + break; + default: + throw new UnsupportedOperationException("Unknown data type " + chunkWriter.getDataType()); + } + } + + // returns totalChunkNum of a file and the max number of chunks of a series + public static long[] findTotalAndLargestSeriesChunkNum(TsFileResource tsFileResource, + TsFileSequenceReader sequenceReader) + throws IOException { + long totalChunkNum = 0; + long maxChunkNum = Long.MIN_VALUE; + List paths = collectFileSeries(sequenceReader); + + for (Path path : paths) { + List chunkMetaDataList = sequenceReader.getChunkMetadataList(path); + totalChunkNum += chunkMetaDataList.size(); + maxChunkNum = chunkMetaDataList.size() > maxChunkNum ? chunkMetaDataList.size() : maxChunkNum; + } + logger.debug("In file {}, total chunk num {}, series max chunk num {}", tsFileResource, + totalChunkNum, maxChunkNum); + return new long[] {totalChunkNum, maxChunkNum}; + } + + public static long getFileMetaSize(TsFileResource seqFile, TsFileSequenceReader sequenceReader) throws IOException { + long minPos = Long.MAX_VALUE; + TsFileMetaData fileMetaData = sequenceReader.readFileMetadata(); + Map deviceMap = fileMetaData.getDeviceMap(); + for (TsDeviceMetadataIndex metadataIndex : deviceMap.values()) { + minPos = metadataIndex.getOffset() < minPos ? metadataIndex.getOffset() : minPos; + } + return seqFile.getFileSize() - minPos; + } + + /** + * Reads chunks of paths in unseqResources and put them in separated lists. When reading a + * file, this method follows the order of positions of chunks instead of the order of + * timeseries, which reduce disk seeks. + * @param paths names of the timeseries + * @param unseqResources + * @param mergeResource + * @return + * @throws IOException + */ + public static List[] collectUnseqChunks(List paths, + List unseqResources, MergeResource mergeResource) throws IOException { + List[] ret = new List[paths.size()]; + for (int i = 0; i < paths.size(); i++) { + ret[i] = new ArrayList<>(); + } + PriorityQueue chunkMetaHeap = new PriorityQueue<>(); + + for (TsFileResource tsFileResource : unseqResources) { + + TsFileSequenceReader tsFileReader = mergeResource.getFileReader(tsFileResource); + // prepare metaDataList + buildMetaHeap(paths, tsFileReader, mergeResource, tsFileResource, chunkMetaHeap); + + // read chunks order by their position + collectUnseqChunks(chunkMetaHeap, tsFileReader, ret); + } + return ret; + } + + private static void buildMetaHeap(List paths, TsFileSequenceReader tsFileReader, + MergeResource resource, TsFileResource tsFileResource, PriorityQueue chunkMetaHeap) + throws IOException { + for (int i = 0; i < paths.size(); i++) { + Path path = paths.get(i); + List metaDataList = tsFileReader.getChunkMetadataList(path); + if (metaDataList.isEmpty()) { + continue; + } + List pathModifications = + resource.getModifications(tsFileResource, path); + if (!pathModifications.isEmpty()) { + QueryUtils.modifyChunkMetaData(metaDataList, pathModifications); + } + MetaListEntry entry = new MetaListEntry(i, metaDataList); + if (entry.hasNext()) { + entry.next(); + chunkMetaHeap.add(entry); + } + } + } + + private static void collectUnseqChunks(PriorityQueue chunkMetaHeap, + TsFileSequenceReader tsFileReader, List[] ret) throws IOException { + while (!chunkMetaHeap.isEmpty()) { + MetaListEntry metaListEntry = chunkMetaHeap.poll(); + ChunkMetaData currMeta = metaListEntry.current(); + Chunk chunk = tsFileReader.readMemChunk(currMeta); + ret[metaListEntry.pathId].add(chunk); + if (metaListEntry.hasNext()) { + metaListEntry.next(); + chunkMetaHeap.add(metaListEntry); + } + } + } + + public static boolean isChunkOverflowed(TimeValuePair timeValuePair, ChunkMetaData metaData) { + return timeValuePair != null + && timeValuePair.getTimestamp() < metaData.getEndTime(); + } + + public static boolean isChunkTooSmall(int ptWritten, ChunkMetaData chunkMetaData, + boolean isLastChunk, int minChunkPointNum) { + return ptWritten > 0 || (minChunkPointNum >= 0 && chunkMetaData.getNumOfPoints() < minChunkPointNum + && !isLastChunk); + } + + public static List> splitPathsByDevice(List paths) { + if (paths.isEmpty()) { + return Collections.emptyList(); + } + paths.sort(Comparator.comparing(Path::getDevice)); + + String currDevice = null; + List currList = null; + List> ret = new ArrayList<>(); + for (Path path : paths) { + if (currDevice == null) { + currDevice = path.getDevice(); + currList = new ArrayList<>(); + currList.add(path); + } else if (path.getDevice().equals(currDevice)) { + currList.add(path); + } else { + ret.add(currList); + currDevice = path.getDevice(); + currList = new ArrayList<>(); + currList.add(path); + } + } + ret.add(currList); + return ret; + } + + public static class MetaListEntry implements Comparable{ + private int pathId; + private int listIdx; + private List chunkMetaDataList; + + public MetaListEntry(int pathId, List chunkMetaDataList) { + this.pathId = pathId; + this.listIdx = -1; + this.chunkMetaDataList = chunkMetaDataList; + } + + @Override + public int compareTo(MetaListEntry o) { + return Long.compare(this.current().getOffsetOfChunkHeader(), + o.current().getOffsetOfChunkHeader()); + } + + public ChunkMetaData current() { + return chunkMetaDataList.get(listIdx); + } + + public boolean hasNext() { + return listIdx + 1 < chunkMetaDataList.size(); + } + + public ChunkMetaData next() { + return chunkMetaDataList.get(++listIdx); + } + + public int getPathId() { + return pathId; + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java index 0cd742cc1f9cb88efa79ae98fd4f766f20f2ca48..60e73fdbacf4389586995219f90426c7778b4d2f 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java @@ -22,9 +22,12 @@ import org.apache.iotdb.service.rpc.thrift.TSDataValue; import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; import org.apache.iotdb.service.rpc.thrift.TSRowRecord; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Field; import org.apache.iotdb.tsfile.read.common.RowRecord; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.BytesUtils; /** * TimeValuePairUtils to convert between thrift format and TsFile format. @@ -101,4 +104,87 @@ public class QueryDataSetUtils { } return tsRowRecord; } + + + public static long[] readTimesFromBuffer(ByteBuffer buffer, int size) { + long[] times = new long[size]; + for (int i = 0; i < size; i++) { + times[i] = buffer.getLong(); + } + return times; + } + + + public static Object[] readValuesFromBuffer(ByteBuffer buffer, List types, + int columns, int size) { + TSDataType[] dataTypes = new TSDataType[types.size()]; + for (int i = 0; i < dataTypes.length; i++) { + dataTypes[i] = TSDataType.values()[types.get(i)]; + } + return readValuesFromBuffer(buffer, dataTypes, columns, size); + } + + /** + * @param buffer data values + * @param columns column number + * @param size value count in each column + */ + public static Object[] readValuesFromBuffer(ByteBuffer buffer, TSDataType[] types, + int columns, int size) { + Object[] values = new Object[columns]; + for (int i = 0; i < columns; i++) { + switch (types[i]) { + case BOOLEAN: + boolean[] boolValues = new boolean[size]; + for (int index = 0; index < size; index++) { + boolValues[index] = BytesUtils.byteToBool(buffer.get()); + } + values[i] = boolValues; + break; + case INT32: + int[] intValues = new int[size]; + for (int index = 0; index < size; index++) { + intValues[index] = buffer.getInt(); + } + values[i] = intValues; + break; + case INT64: + long[] longValues = new long[size]; + for (int index = 0; index < size; index++) { + longValues[index] = buffer.getLong(); + } + values[i] = longValues; + break; + case FLOAT: + float[] floatValues = new float[size]; + for (int index = 0; index < size; index++) { + floatValues[index] = buffer.getFloat(); + } + values[i] = floatValues; + break; + case DOUBLE: + double[] doubleValues = new double[size]; + for (int index = 0; index < size; index++) { + doubleValues[index] = buffer.getDouble(); + } + values[i] = doubleValues; + break; + case TEXT: + Binary[] binaryValues = new Binary[size]; + for (int index = 0; index < size; index++) { + int binarySize = buffer.getInt(); + byte[] binaryValue = new byte[binarySize]; + buffer.get(binaryValue); + binaryValues[index] = new Binary(binaryValue); + } + values[i] = binaryValues; + break; + default: + throw new UnSupportedDataTypeException( + String.format("data type %s is not supported when convert data at client", + types[i])); + } + } + return values; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileSchemaUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java similarity index 72% rename from server/src/main/java/org/apache/iotdb/db/utils/FileSchemaUtils.java rename to server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 0b529e004b031ff3559a6694b199e302032e8fe9..cae69e35ada2d7a9719ba89fc59ec5d97ec37bdb 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/FileSchemaUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -21,34 +21,34 @@ package org.apache.iotdb.db.utils; import java.util.List; import org.apache.iotdb.db.metadata.MManager; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; -public class FileSchemaUtils { +public class SchemaUtils { - private FileSchemaUtils(){} + private SchemaUtils(){} /** - * Construct the FileSchema of the FileNode named processorName. + * Construct the Schema of the FileNode named processorName. * @param processorName the name of a FileNode. * @return the schema of the FileNode named processorName. * @throws WriteProcessException when the fileSchema cannot be created. */ - public static FileSchema constructFileSchema(String processorName) { + public static Schema constructSchema(String processorName) { List columnSchemaList; columnSchemaList = MManager.getInstance().getSchemaForStorageGroup(processorName); - return getFileSchemaFromColumnSchema(columnSchemaList); + return getSchemaFromColumnSchema(columnSchemaList); } /** - * getFileSchemaFromColumnSchema construct a FileSchema using the schema of the columns and the + * getSchemaFromColumnSchema construct a Schema using the schema of the columns and the * device type. * @param schemaList the schema of the columns in this file. - * @return a FileSchema contains the provided schemas. + * @return a Schema contains the provided schemas. */ - public static FileSchema getFileSchemaFromColumnSchema(List schemaList) { - FileSchema schema = new FileSchema(); + public static Schema getSchemaFromColumnSchema(List schemaList) { + Schema schema = new Schema(); for (MeasurementSchema measurementSchema : schemaList) { schema.registerMeasurement(measurementSchema); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java index 72fb735a010ab0d9cb03c2f1f52a91024e9d6f55..cd286dd58bd9d89c2d3c613e966f2fa3884e34ad 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java @@ -19,12 +19,22 @@ package org.apache.iotdb.db.utils; import org.apache.iotdb.db.query.aggregation.AggreResultData; +import org.apache.iotdb.db.utils.TsPrimitiveType.TsBinary; +import org.apache.iotdb.db.utils.TsPrimitiveType.TsBoolean; +import org.apache.iotdb.db.utils.TsPrimitiveType.TsDouble; +import org.apache.iotdb.db.utils.TsPrimitiveType.TsFloat; +import org.apache.iotdb.db.utils.TsPrimitiveType.TsInt; +import org.apache.iotdb.db.utils.TsPrimitiveType.TsLong; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.utils.Binary; public class TimeValuePairUtils { - private TimeValuePairUtils(){} + private TimeValuePairUtils() { + } + /** * get given data's current (time,value) pair. * @@ -64,17 +74,93 @@ public class TimeValuePairUtils { case INT32: return new TimeValuePair(data.getTimestamp(), new TsPrimitiveType.TsInt(data.getIntRet())); case INT64: - return new TimeValuePair(data.getTimestamp(), new TsPrimitiveType.TsLong(data.getLongRet())); + return new TimeValuePair(data.getTimestamp(), + new TsPrimitiveType.TsLong(data.getLongRet())); case FLOAT: - return new TimeValuePair(data.getTimestamp(), new TsPrimitiveType.TsFloat(data.getFloatRet())); + return new TimeValuePair(data.getTimestamp(), + new TsPrimitiveType.TsFloat(data.getFloatRet())); case DOUBLE: - return new TimeValuePair(data.getTimestamp(), new TsPrimitiveType.TsDouble(data.getDoubleRet())); + return new TimeValuePair(data.getTimestamp(), + new TsPrimitiveType.TsDouble(data.getDoubleRet())); case TEXT: - return new TimeValuePair(data.getTimestamp(), new TsPrimitiveType.TsBinary(data.getBinaryRet())); + return new TimeValuePair(data.getTimestamp(), + new TsPrimitiveType.TsBinary(data.getBinaryRet())); case BOOLEAN: - return new TimeValuePair(data.getTimestamp(), new TsPrimitiveType.TsBoolean(data.isBooleanRet())); + return new TimeValuePair(data.getTimestamp(), + new TsPrimitiveType.TsBoolean(data.isBooleanRet())); default: throw new UnSupportedDataTypeException(String.valueOf(data.getDataType())); } } -} + + public static void setCurrentTimeValuePair(BatchData data, TimeValuePair current) { + current.setTimestamp(data.currentTime()); + switch (data.getDataType()) { + case INT32: + current.getValue().setInt(data.getInt()); + break; + case INT64: + current.getValue().setLong(data.getLong()); + break; + case FLOAT: + current.getValue().setFloat(data.getFloat()); + break; + case DOUBLE: + current.getValue().setDouble(data.getDouble()); + break; + case TEXT: + current.getValue().setBinary(data.getBinary()); + break; + case BOOLEAN: + current.getValue().setBoolean(data.getBoolean()); + break; + default: + throw new UnSupportedDataTypeException(String.valueOf(data.getDataType())); + } + } + + public static void setTimeValuePair(TimeValuePair from, TimeValuePair to) { + to.setTimestamp(from.getTimestamp()); + switch (from.getValue().getDataType()) { + case INT32: + to.getValue().setInt(from.getValue().getInt()); + break; + case INT64: + to.getValue().setLong(from.getValue().getLong()); + break; + case FLOAT: + to.getValue().setFloat(from.getValue().getFloat()); + break; + case DOUBLE: + to.getValue().setDouble(from.getValue().getDouble()); + break; + case TEXT: + to.getValue().setBinary(from.getValue().getBinary()); + break; + case BOOLEAN: + to.getValue().setBoolean(from.getValue().getBoolean()); + break; + default: + throw new UnSupportedDataTypeException(String.valueOf(from.getValue().getDataType())); + } + } + + public static TimeValuePair getEmptyTimeValuePair(TSDataType dataType) { + switch (dataType) { + case FLOAT: + return new TimeValuePair(0, new TsFloat(0.0f)); + case INT32: + return new TimeValuePair(0, new TsInt(0)); + case INT64: + return new TimeValuePair(0, new TsLong(0)); + case BOOLEAN: + return new TimeValuePair(0, new TsBoolean(false)); + case DOUBLE: + return new TimeValuePair(0, new TsDouble(0.0)); + case TEXT: + return new TimeValuePair(0, new TsBinary(new Binary(""))); + default: + throw new UnsupportedOperationException("Unrecognized datatype: " + dataType); + } + } +} \ No newline at end of file diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TsPrimitiveType.java b/server/src/main/java/org/apache/iotdb/db/utils/TsPrimitiveType.java index 70cd89c95bdc19247bdafef071bff2be426ef1af..67750542d718458e4150ae609febd6871c8ec608 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/TsPrimitiveType.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/TsPrimitiveType.java @@ -74,6 +74,30 @@ public abstract class TsPrimitiveType implements Serializable { throw new UnsupportedOperationException("getBinary() is not supported for current sub-class"); } + public void setBoolean(boolean val) { + throw new UnsupportedOperationException("setBoolean() is not supported for current sub-class"); + } + + public void setInt(int val) { + throw new UnsupportedOperationException("setInt() is not supported for current sub-class"); + } + + public void setLong(long val) { + throw new UnsupportedOperationException("setLong() is not supported for current sub-class"); + } + + public void setFloat(float val) { + throw new UnsupportedOperationException("setFloat() is not supported for current sub-class"); + } + + public void setDouble(double val) { + throw new UnsupportedOperationException("setDouble() is not supported for current sub-class"); + } + + public void setBinary(Binary val) { + throw new UnsupportedOperationException("setBinary() is not supported for current sub-class"); + } + /** * get the size of one instance of current class. * @@ -116,6 +140,11 @@ public abstract class TsPrimitiveType implements Serializable { return value; } + @Override + public void setBoolean(boolean val) { + this.value = val; + } + @Override public int getSize() { return 4 + 1; @@ -167,6 +196,11 @@ public abstract class TsPrimitiveType implements Serializable { return value; } + @Override + public void setInt(int val) { + this.value = val; + } + @Override public int getSize() { return 4 + 4; @@ -218,6 +252,11 @@ public abstract class TsPrimitiveType implements Serializable { return value; } + @Override + public void setLong(long val) { + this.value = val; + } + @Override public int getSize() { return 4 + 8; @@ -269,6 +308,11 @@ public abstract class TsPrimitiveType implements Serializable { return value; } + @Override + public void setFloat(float val) { + this.value = val; + } + @Override public int getSize() { return 4 + 4; @@ -320,6 +364,11 @@ public abstract class TsPrimitiveType implements Serializable { return value; } + @Override + public void setDouble(double val) { + this.value = val; + } + @Override public int getSize() { return 4 + 8; @@ -371,6 +420,11 @@ public abstract class TsPrimitiveType implements Serializable { return value; } + @Override + public void setBinary(Binary val) { + this.value = val; + } + @Override public int getSize() { return 4 + 4 + value.getLength(); diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java index cc176dacc935acffcdde73ce1167b6d20543b699..357f31dd6eafbc29aa281092bc4ef0bf1bdf8260 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java @@ -173,4 +173,35 @@ public class BinaryTVList extends TVList { protected void releaseLastValueArray() { PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1)); } + + @Override + public void putBinaries(long[] time, Binary[] value) { + checkExpansion(); + int idx = 0; + int length = time.length; + + updateMinTimeAndSorted(time); + + while (idx < length) { + int inputRemaining = length - idx; + int arrayIdx = size / ARRAY_SIZE; + int elementIdx = size % ARRAY_SIZE; + int internalRemaining = ARRAY_SIZE - elementIdx; + if (internalRemaining >= inputRemaining) { + // the remaining inputs can fit the last array, copy all remaining inputs into last array + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining); + size += inputRemaining; + break; + } else { + // the remaining inputs cannot fit the last array, fill the last array and create a new + // one and enter the next loop + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining); + idx += internalRemaining; + size += internalRemaining; + checkExpansion(); + } + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java index bdadec9278a720cb2d983b9b710040197eff4e96..f5ab2bb89e2cc09c10ef41cb4ba99d5524ef3309 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java @@ -172,4 +172,35 @@ public class BooleanTVList extends TVList { protected void releaseLastValueArray() { PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1)); } + + @Override + public void putBooleans(long[] time, boolean[] value) { + checkExpansion(); + int idx = 0; + int length = time.length; + + updateMinTimeAndSorted(time); + + while (idx < length) { + int inputRemaining = length - idx; + int arrayIdx = size / ARRAY_SIZE; + int elementIdx = size % ARRAY_SIZE; + int internalRemaining = ARRAY_SIZE - elementIdx; + if (internalRemaining >= inputRemaining) { + // the remaining inputs can fit the last array, copy all remaining inputs into last array + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining); + size += inputRemaining; + break; + } else { + // the remaining inputs cannot fit the last array, fill the last array and create a new + // one and enter the next loop + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining); + idx += internalRemaining; + size += internalRemaining; + checkExpansion(); + } + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java index e9934b5ac714b68001766cd3f9d1a33dd1a2e324..d5e60154da54f7e8cea6cfdf0e79edc6fa077563 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java @@ -172,4 +172,35 @@ public class DoubleTVList extends TVList { protected void releaseLastValueArray() { PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1)); } + + @Override + public void putDoubles(long[] time, double[] value) { + checkExpansion(); + int idx = 0; + int length = time.length; + + updateMinTimeAndSorted(time); + + while (idx < length) { + int inputRemaining = length - idx; + int arrayIdx = size / ARRAY_SIZE; + int elementIdx = size % ARRAY_SIZE; + int internalRemaining = ARRAY_SIZE - elementIdx; + if (internalRemaining >= inputRemaining) { + // the remaining inputs can fit the last array, copy all remaining inputs into last array + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining); + size += inputRemaining; + break; + } else { + // the remaining inputs cannot fit the last array, fill the last array and create a new + // one and enter the next loop + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining); + idx += internalRemaining; + size += internalRemaining; + checkExpansion(); + } + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java index b8235144281032da06a9866c666347abc5541b5c..5f6c38a576e1f2d1effac2779ea2397cb44533e9 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java @@ -172,4 +172,35 @@ public class FloatTVList extends TVList { protected void releaseLastValueArray() { PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1)); } + + @Override + public void putFloats(long[] time, float[] value) { + checkExpansion(); + int idx = 0; + int length = time.length; + + updateMinTimeAndSorted(time); + + while (idx < length) { + int inputRemaining = length - idx; + int arrayIdx = size / ARRAY_SIZE; + int elementIdx = size % ARRAY_SIZE; + int internalRemaining = ARRAY_SIZE - elementIdx; + if (internalRemaining >= inputRemaining) { + // the remaining inputs can fit the last array, copy all remaining inputs into last array + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining); + size += inputRemaining; + break; + } else { + // the remaining inputs cannot fit the last array, fill the last array and create a new + // one and enter the next loop + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining); + idx += internalRemaining; + size += internalRemaining; + checkExpansion(); + } + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java index 9fb699f6e592c23c7c2b41179e92f6ebb7403c12..131b61ec84b16b5f7e6e61f4e7c3254452fd3615 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java @@ -172,4 +172,35 @@ public class IntTVList extends TVList { protected void releaseLastValueArray() { PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1)); } + + @Override + public void putInts(long[] time, int[] value) { + checkExpansion(); + int idx = 0; + int length = time.length; + + updateMinTimeAndSorted(time); + + while (idx < length) { + int inputRemaining = length - idx; + int arrayIdx = size / ARRAY_SIZE; + int elementIdx = size % ARRAY_SIZE; + int internalRemaining = ARRAY_SIZE - elementIdx; + if (internalRemaining >= inputRemaining) { + // the remaining inputs can fit the last array, copy all remaining inputs into last array + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining); + size += inputRemaining; + break; + } else { + // the remaining inputs cannot fit the last array, fill the last array and create a new + // one and enter the next loop + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining); + idx += internalRemaining; + size += internalRemaining; + checkExpansion(); + } + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java index dfa06f5b9f06593295fe1247f254de18421b8fe0..9ad1a56ba002ef335f6bf79977590781661dcbf5 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java @@ -172,4 +172,35 @@ public class LongTVList extends TVList { protected void releaseLastValueArray() { PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1)); } + + @Override + public void putLongs(long[] time, long[] value) { + checkExpansion(); + int idx = 0; + int length = time.length; + + updateMinTimeAndSorted(time); + + while (idx < length) { + int inputRemaining = length - idx; + int arrayIdx = size / ARRAY_SIZE; + int elementIdx = size % ARRAY_SIZE; + int internalRemaining = ARRAY_SIZE - elementIdx; + if (internalRemaining >= inputRemaining) { + // the remaining inputs can fit the last array, copy all remaining inputs into last array + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining); + size += inputRemaining; + break; + } else { + // the remaining inputs cannot fit the last array, fill the last array and create a new + // one and enter the next loop + System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining); + System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining); + idx += internalRemaining; + size += internalRemaining; + checkExpansion(); + } + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 56dbf8c276f2248ec63a70864abd7dd428ce1f88..95d164740f1e4caa176104a9495624a620295e25 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -92,6 +92,30 @@ public abstract class TVList { throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); } + public void putLongs(long[] time, long[] value) { + throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + } + + public void putInts(long[] time, int[] value) { + throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + } + + public void putFloats(long[] time, float[] value) { + throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + } + + public void putDoubles(long[] time, double[] value) { + throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + } + + public void putBinaries(long[] time, Binary[] value) { + throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + } + + public void putBooleans(long[] time, boolean[] value) { + throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + } + public long getLong(int index) { throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); } @@ -393,4 +417,18 @@ public abstract class TVList { setFromSorted(i, i); } } + + void updateMinTimeAndSorted(long[] time) { + int length = time.length; + long inPutMinTime = Long.MAX_VALUE; + boolean inputSorted = true; + for (int i = 0; i < length; i++) { + inPutMinTime = inPutMinTime <= time[i] ? inPutMinTime : time[i]; + if (inputSorted && i < length - 1 && time[i] > time[i+1]) { + inputSorted = false; + } + } + minTime = inPutMinTime < minTime ? inPutMinTime : minTime; + sorted = sorted && inputSorted && (size == 0 || inPutMinTime >= getTime(size - 1)); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/manager/MultiFileLogNodeManager.java b/server/src/main/java/org/apache/iotdb/db/writelog/manager/MultiFileLogNodeManager.java index 9e7d3a016dc0a42ab172cc5fe4fd1309e10da065..b5675a12a1889380c9841913a81eb2d4bba42e46 100644 --- a/server/src/main/java/org/apache/iotdb/db/writelog/manager/MultiFileLogNodeManager.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/manager/MultiFileLogNodeManager.java @@ -106,7 +106,7 @@ public class MultiFileLogNodeManager implements WriteLogNodeManager, IService { @Override public void close() { if (!isActivated(forceThread)) { - logger.error("MultiFileLogNodeManager has not yet started"); + logger.warn("MultiFileLogNodeManager has not yet started"); return; } logger.info("LogNodeManager starts closing.."); diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java index 2e1f7740bb49636eed94d5225e7b94d5e02d8aaf..42c5b58e33242cb25b230ab0abe0ab370917dd77 100644 --- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java @@ -24,10 +24,10 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.memtable.IMemTable; import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.modification.ModificationFile; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.version.VersionController; import org.apache.iotdb.db.exception.ProcessorException; import org.apache.iotdb.db.qp.physical.PhysicalPlan; @@ -39,7 +39,7 @@ import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager; import org.apache.iotdb.db.writelog.node.WriteLogNode; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; /** * LogReplayer finds the logNode of the TsFile given by insertFilePath and logNodePrefix, reads @@ -52,8 +52,8 @@ public class LogReplayer { private ModificationFile modFile; private VersionController versionController; private TsFileResource currentTsFileResource; - // fileSchema is used to get the measurement data type - private FileSchema fileSchema; + // schema is used to get the measurement data type + private Schema schema; private IMemTable recoverMemTable; // unsequence file tolerates duplicated data @@ -66,13 +66,13 @@ public class LogReplayer { ModificationFile modFile, VersionController versionController, TsFileResource currentTsFileResource, - FileSchema fileSchema, IMemTable memTable, boolean acceptDuplication) { + Schema schema, IMemTable memTable, boolean acceptDuplication) { this.logNodePrefix = logNodePrefix; this.insertFilePath = insertFilePath; this.modFile = modFile; this.versionController = versionController; this.currentTsFileResource = currentTsFileResource; - this.fileSchema = fileSchema; + this.schema = schema; this.recoverMemTable = memTable; this.acceptDuplication = acceptDuplication; } @@ -103,8 +103,8 @@ public class LogReplayer { } finally { logReader.close(); } - tempStartTimeMap.forEach((k, v) -> currentTsFileResource.updateTime(k, v)); - tempEndTimeMap.forEach((k, v) -> currentTsFileResource.updateTime(k, v)); + tempStartTimeMap.forEach((k, v) -> currentTsFileResource.updateStartTime(k, v)); + tempEndTimeMap.forEach((k, v) -> currentTsFileResource.updateEndTime(k, v)); } private void replayDelete(DeletePlan deletePlan) throws IOException { @@ -123,7 +123,10 @@ public class LogReplayer { !acceptDuplication) { return; } - tempStartTimeMap.putIfAbsent(insertPlan.getDeviceId(), insertPlan.getTime()); + Long startTime = tempStartTimeMap.get(insertPlan.getDeviceId()); + if (startTime == null || startTime > insertPlan.getTime()) { + tempStartTimeMap.put(insertPlan.getDeviceId(), insertPlan.getTime()); + } Long endTime = tempEndTimeMap.get(insertPlan.getDeviceId()); if (endTime == null || endTime < insertPlan.getTime()) { tempEndTimeMap.put(insertPlan.getDeviceId(), insertPlan.getTime()); @@ -132,7 +135,7 @@ public class LogReplayer { String[] measurementList = insertPlan.getMeasurements(); TSDataType[] dataTypes = new TSDataType[measurementList.length]; for (int i = 0; i < measurementList.length; i++) { - dataTypes[i] = fileSchema.getMeasurementDataType(measurementList[i]); + dataTypes[i] = schema.getMeasurementDataType(measurementList[i]); } insertPlan.setDataTypes(dataTypes); recoverMemTable.insert(insertPlan); diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java index d012e3d14ab9ef768fc4fe026f1c744f8d38fddc..cf8ee79a0745ae3c7441cc987dee1857236c7d4a 100644 --- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java @@ -26,8 +26,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; -import org.apache.iotdb.db.engine.memtable.IMemTable; import org.apache.iotdb.db.engine.flush.MemTableFlushTask; +import org.apache.iotdb.db.engine.memtable.IMemTable; import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.version.VersionController; @@ -39,32 +39,33 @@ import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * TsFileRecoverPerformer recovers a SeqTsFile to correct status, redoes the WALs since last - * crash and removes the redone logs. + * TsFileRecoverPerformer recovers a SeqTsFile to correct status, redoes the WALs since last crash + * and removes the redone logs. */ public class TsFileRecoverPerformer { + private static final Logger logger = LoggerFactory.getLogger(TsFileRecoverPerformer.class); private String insertFilePath; private String logNodePrefix; - private FileSchema fileSchema; + private Schema schema; private VersionController versionController; private LogReplayer logReplayer; private TsFileResource tsFileResource; private boolean acceptUnseq; public TsFileRecoverPerformer(String logNodePrefix, - FileSchema fileSchema, VersionController versionController, + Schema schema, VersionController versionController, TsFileResource currentTsFileResource, boolean acceptUnseq) { this.insertFilePath = currentTsFileResource.getFile().getPath(); this.logNodePrefix = logNodePrefix; - this.fileSchema = fileSchema; + this.schema = schema; this.versionController = versionController; this.tsFileResource = currentTsFileResource; this.acceptUnseq = acceptUnseq; @@ -72,16 +73,14 @@ public class TsFileRecoverPerformer { /** * 1. recover the TsFile by RestorableTsFileIOWriter and truncate the file to remaining corrected - * data - * 2. redo the WALs to recover unpersisted data - * 3. flush and close the file - * 4. clean WALs + * data 2. redo the WALs to recover unpersisted data 3. flush and close the file 4. clean WALs */ public void recover() throws ProcessorException { + IMemTable recoverMemTable = new PrimitiveMemTable(); this.logReplayer = new LogReplayer(logNodePrefix, insertFilePath, tsFileResource.getModFile(), versionController, - tsFileResource, fileSchema, recoverMemTable, acceptUnseq); + tsFileResource, schema, recoverMemTable, acceptUnseq); File insertFile = new File(insertFilePath); if (!insertFile.exists()) { logger.error("TsFile {} is missing, will skip its recovery.", insertFilePath); @@ -95,25 +94,29 @@ public class TsFileRecoverPerformer { throw new ProcessorException(e); } - if (!restorableTsFileIOWriter.hasCrashed()) { + if (!restorableTsFileIOWriter.hasCrashed() && !restorableTsFileIOWriter.canWrite()) { // tsfile is complete try { if (tsFileResource.fileExists()) { // .resource file exists, deserialize it - tsFileResource.deSerialize(); + recoverResourceFromFile(); } else { // .resource file does not exist, read file metadata and recover tsfile resource - try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFileResource.getFile().getAbsolutePath())) { + try (TsFileSequenceReader reader = new TsFileSequenceReader( + tsFileResource.getFile().getAbsolutePath())) { TsFileMetaData metaData = reader.readFileMetadata(); List deviceMetadataIndexList = new ArrayList<>( metaData.getDeviceMap().values()); for (TsDeviceMetadataIndex index : deviceMetadataIndexList) { TsDeviceMetadata deviceMetadata = reader.readTsDeviceMetaData(index); - List chunkGroupMetaDataList = deviceMetadata.getChunkGroupMetaDataList(); + List chunkGroupMetaDataList = deviceMetadata + .getChunkGroupMetaDataList(); for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetaDataList) { for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) { - tsFileResource.updateTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getStartTime()); - tsFileResource.updateTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getEndTime()); + tsFileResource.updateStartTime(chunkGroupMetaData.getDeviceID(), + chunkMetaData.getStartTime()); + tsFileResource + .updateEndTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getEndTime()); } } } @@ -129,40 +132,87 @@ public class TsFileRecoverPerformer { } else { // due to failure, the last ChunkGroup may contain the same data as the WALs, so the time // map must be updated first to avoid duplicated insertion - for (ChunkGroupMetaData chunkGroupMetaData : restorableTsFileIOWriter - .getChunkGroupMetaDatas()) { - for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) { - tsFileResource.updateTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getStartTime()); - tsFileResource.updateTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getEndTime()); + recoverResourceFromWriter(restorableTsFileIOWriter); + } + + // redo logs + redoLogs(restorableTsFileIOWriter); + + // clean logs + try { + MultiFileLogNodeManager.getInstance() + .deleteNode(logNodePrefix + new File(insertFilePath).getName()); + } catch (IOException e) { + throw new ProcessorException(e); + } + } + + private void recoverResourceFromFile() throws IOException { + try { + tsFileResource.deSerialize(); + } catch (IOException e) { + logger.warn("Cannot deserialize TsFileResource {}, construct it using " + + "TsFileSequenceReader", tsFileResource.getFile(), e); + recoverResourceFromReader(); + } + } + + + private void recoverResourceFromReader() throws IOException { + try (TsFileSequenceReader reader = + new TsFileSequenceReader(tsFileResource.getFile().getAbsolutePath(), false)) { + TsFileMetaData metaData = reader.readFileMetadata(); + List deviceMetadataIndexList = new ArrayList<>( + metaData.getDeviceMap().values()); + for (TsDeviceMetadataIndex index : deviceMetadataIndexList) { + TsDeviceMetadata deviceMetadata = reader.readTsDeviceMetaData(index); + for (ChunkGroupMetaData chunkGroupMetaData : deviceMetadata + .getChunkGroupMetaDataList()) { + for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) { + tsFileResource.updateStartTime(chunkGroupMetaData.getDeviceID(), + chunkMetaData.getStartTime()); + tsFileResource + .updateEndTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getEndTime()); + } } } } + // write .resource file + tsFileResource.serialize(); + } - // redo logs + private void recoverResourceFromWriter(RestorableTsFileIOWriter restorableTsFileIOWriter) { + for (ChunkGroupMetaData chunkGroupMetaData : restorableTsFileIOWriter + .getChunkGroupMetaDatas()) { + for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) { + tsFileResource.updateStartTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getStartTime()); + tsFileResource.updateEndTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getEndTime()); + } + } + } + + private void redoLogs(RestorableTsFileIOWriter restorableTsFileIOWriter) throws ProcessorException { + IMemTable recoverMemTable = new PrimitiveMemTable(); + this.logReplayer = new LogReplayer(logNodePrefix, insertFilePath, tsFileResource.getModFile(), + versionController, + tsFileResource, schema, recoverMemTable, acceptUnseq); logReplayer.replayLogs(); try { if (!recoverMemTable.isEmpty()) { // flush logs - MemTableFlushTask tableFlushTask = new MemTableFlushTask(recoverMemTable, fileSchema, + + MemTableFlushTask tableFlushTask = new MemTableFlushTask(recoverMemTable, schema, restorableTsFileIOWriter, logNodePrefix); tableFlushTask.syncFlushMemTable(); } // close file - restorableTsFileIOWriter.endFile(fileSchema); + restorableTsFileIOWriter.endFile(schema); tsFileResource.serialize(); } catch (ExecutionException | InterruptedException | IOException e) { Thread.currentThread().interrupt(); throw new ProcessorException(e); } - - // clean logs - try { - MultiFileLogNodeManager.getInstance() - .deleteNode(logNodePrefix + new File(insertFilePath).getName()); - } catch (IOException e) { - throw new ProcessorException(e); - } } } diff --git a/server/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java b/server/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java index aa02b86b4cc301076ebec21546d8093309fe5df6..eb8a7165693e44422eb7ecfe0ac250e48e2dbb3d 100644 --- a/server/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java +++ b/server/src/test/java/org/apache/iotdb/db/cost/statistic/PerformanceStatTest.java @@ -44,7 +44,7 @@ public class PerformanceStatTest { @Test public void test() { Measurement measurement = Measurement.INSTANCE; - Operation operation = Operation.EXECUTE_BATCH; + Operation operation = Operation.EXECUTE_JDBC_BATCH; measurement.addOperationLatency(operation, System.currentTimeMillis()); measurement.addOperationLatency(operation, System.currentTimeMillis() - 8000000); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java index 63063387b3929a28349310e6fb858e2bfadff992..09b34e88f3d6cd821e5813cb377902bf7d567a2f 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskTest.java @@ -61,7 +61,7 @@ public class MemTableFlushTaskTest { MemTableTestUtils.produceData(memTable, startTime, endTime, MemTableTestUtils.deviceId0, MemTableTestUtils.measurementId0, MemTableTestUtils.dataType0); - MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, MemTableTestUtils.getFileSchema(), writer, storageGroup); + MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, MemTableTestUtils.getSchema(), writer, storageGroup); assertTrue(writer .getVisibleMetadataList(MemTableTestUtils.deviceId0, MemTableTestUtils.measurementId0, MemTableTestUtils.dataType0).isEmpty()); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java index ea5d174ad1cb915fb0f16de4e446978797bd542a..ddf7487facd35df91b0550979e2a1a59bf9ea7f1 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.engine.memtable; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; public class MemTableTestUtils { @@ -30,10 +30,10 @@ public class MemTableTestUtils { public static String measurementId0 = "s0"; public static TSDataType dataType0 = TSDataType.INT32; - private static FileSchema fileSchema = new FileSchema(); + private static Schema schema = new Schema(); static { - fileSchema + schema .registerMeasurement(new MeasurementSchema(measurementId0, dataType0, TSEncoding.PLAIN)); } @@ -47,8 +47,8 @@ public class MemTableTestUtils { } } - public static FileSchema getFileSchema() { - return fileSchema; + public static Schema getSchema() { + return schema; } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..ac6f983aa497f5fd0315e8210e6bf53cc21042bd --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.List; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.selector.MaxFileMergeFileSelector; +import org.apache.iotdb.db.engine.merge.selector.IMergeFileSelector; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MergeException; +import org.junit.Test; + +public class MaxFileMergeFileSelectorTest extends MergeTest{ + + @Test + public void testFullSelection() throws MergeException, IOException { + MergeResource resource = new MergeResource(seqResources, unseqResources); + IMergeFileSelector mergeFileSelector = new MaxFileMergeFileSelector(resource, Long.MAX_VALUE); + List[] result = mergeFileSelector.select(); + List seqSelected = result[0]; + List unseqSelected = result[1]; + assertEquals(seqResources, seqSelected); + assertEquals(unseqResources, unseqSelected); + resource.clear(); + + resource = new MergeResource(seqResources.subList(0, 1), unseqResources); + mergeFileSelector = new MaxFileMergeFileSelector(resource, Long.MAX_VALUE); + result = mergeFileSelector.select(); + seqSelected = result[0]; + unseqSelected = result[1]; + assertEquals(seqResources.subList(0, 1), seqSelected); + assertEquals(unseqResources, unseqSelected); + resource.clear(); + + resource = new MergeResource(seqResources, unseqResources.subList(0, 1)); + mergeFileSelector = new MaxFileMergeFileSelector(resource, Long.MAX_VALUE); + result = mergeFileSelector.select(); + seqSelected = result[0]; + unseqSelected = result[1]; + assertEquals(seqResources.subList(0, 1), seqSelected); + assertEquals(unseqResources.subList(0, 1), unseqSelected); + resource.clear(); + } + + @Test + public void testNonSelection() throws MergeException, IOException { + MergeResource resource = new MergeResource(seqResources, unseqResources); + IMergeFileSelector mergeFileSelector = new MaxFileMergeFileSelector(resource, 1); + List[] result = mergeFileSelector.select(); + assertEquals(0, result.length); + resource.clear(); + } + + @Test + public void testRestrictedSelection() throws MergeException, IOException { + MergeResource resource = new MergeResource(seqResources, unseqResources); + IMergeFileSelector mergeFileSelector = new MaxFileMergeFileSelector(resource, 400000); + List[] result = mergeFileSelector.select(); + List seqSelected = result[0]; + List unseqSelected = result[1]; + assertEquals(seqResources.subList(0, 2), seqSelected); + assertEquals(unseqResources.subList(0, 2), unseqSelected); + resource.clear(); + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..50f5cd091aeaae999e161d1b08388695f7e664ed --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.List; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.selector.MaxSeriesMergeFileSelector; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MergeException; +import org.junit.Test; + +public class MaxSeriesMergeFileSelectorTest extends MergeTest{ + + @Test + public void testFullSelection() throws MergeException, IOException { + MergeResource resource = new MergeResource(seqResources, unseqResources); + MaxSeriesMergeFileSelector mergeFileSelector = new MaxSeriesMergeFileSelector(resource, Long.MAX_VALUE); + List[] result = mergeFileSelector.select(); + List seqSelected = result[0]; + List unseqSelected = result[1]; + assertEquals(seqResources, seqSelected); + assertEquals(unseqResources, unseqSelected); + assertEquals(MaxSeriesMergeFileSelector.MAX_SERIES_NUM, mergeFileSelector.getConcurrentMergeNum()); + resource.clear(); + + resource = new MergeResource(seqResources.subList(0, 1), unseqResources); + mergeFileSelector = new MaxSeriesMergeFileSelector(resource, Long.MAX_VALUE); + result = mergeFileSelector.select(); + seqSelected = result[0]; + unseqSelected = result[1]; + assertEquals(seqResources.subList(0, 1), seqSelected); + assertEquals(unseqResources, unseqSelected); + assertEquals(MaxSeriesMergeFileSelector.MAX_SERIES_NUM, mergeFileSelector.getConcurrentMergeNum()); + resource.clear(); + + resource = new MergeResource(seqResources, unseqResources.subList(0, 1)); + mergeFileSelector = new MaxSeriesMergeFileSelector(resource, Long.MAX_VALUE); + result = mergeFileSelector.select(); + seqSelected = result[0]; + unseqSelected = result[1]; + assertEquals(seqResources.subList(0, 1), seqSelected); + assertEquals(unseqResources.subList(0, 1), unseqSelected); + assertEquals(MaxSeriesMergeFileSelector.MAX_SERIES_NUM, mergeFileSelector.getConcurrentMergeNum()); + resource.clear(); + } + + @Test + public void testNonSelection() throws MergeException, IOException { + MergeResource resource = new MergeResource(seqResources, unseqResources); + MaxSeriesMergeFileSelector mergeFileSelector = new MaxSeriesMergeFileSelector(resource, 1); + List[] result = mergeFileSelector.select(); + assertEquals(0, result.length); + assertEquals(0, mergeFileSelector.getConcurrentMergeNum()); + resource.clear(); + } + + @Test + public void testRestrictedSelection() throws MergeException, IOException { + MergeResource resource = new MergeResource(seqResources, unseqResources); + MaxSeriesMergeFileSelector mergeFileSelector = new MaxSeriesMergeFileSelector(resource, 400000); + List[] result = mergeFileSelector.select(); + List seqSelected = result[0]; + List unseqSelected = result[1]; + assertEquals(seqResources.subList(0, 2), seqSelected); + assertEquals(unseqResources.subList(0, 2), unseqSelected); + assertEquals(MaxSeriesMergeFileSelector.MAX_SERIES_NUM, mergeFileSelector.getConcurrentMergeNum()); + resource.clear(); + } + + @Test + public void testRestrictedSelection2() throws MergeException, IOException { + MergeResource resource = new MergeResource(seqResources, unseqResources); + MaxSeriesMergeFileSelector mergeFileSelector = new MaxSeriesMergeFileSelector(resource, + 100000); + List[] result = mergeFileSelector.select(); + List seqSelected = result[0]; + List unseqSelected = result[1]; + assertEquals(seqResources.subList(0, 1), seqSelected); + assertEquals(unseqResources.subList(0, 1), unseqSelected); + assertEquals(34, mergeFileSelector.getConcurrentMergeNum()); + resource.clear(); + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java new file mode 100644 index 0000000000000000000000000000000000000000..0cf852093bd9660006161bb681efdb557894f882 --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.List; +import org.apache.commons.io.FileUtils; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.task.MergeTask; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MetadataErrorException; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.tsfile.exception.write.WriteProcessException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class MergeLogTest extends MergeTest { + + File tempSGDir; + + @Before + public void setUp() throws IOException, WriteProcessException, MetadataErrorException { + super.setUp(); + tempSGDir = new File("tempSG"); + tempSGDir.mkdirs(); + } + + @After + public void tearDown() throws IOException, StorageEngineException { + super.tearDown(); + FileUtils.deleteDirectory(tempSGDir);FileUtils.deleteDirectory(tempSGDir); + } + + @Test + public void testMergeLog() throws Exception { + MergeTask mergeTask = + new MergeTask(new MergeResource(seqResources.subList(0, 1), unseqResources.subList(0, 1)), + tempSGDir.getPath(), this::testCallBack, "test", false, 1, MERGE_TEST_SG); + mergeTask.call(); + } + + private void testCallBack(List seqFiles, List unseqFiles, + File mergeLog) { + int lineCnt = 0; + try (BufferedReader bufferedReader = new BufferedReader(new FileReader(mergeLog))) { + String line; + while ((line = bufferedReader.readLine()) != null) { + lineCnt ++; + System.out.println(line); + } + } catch (IOException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertEquals(309, lineCnt); + } + +} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java new file mode 100644 index 0000000000000000000000000000000000000000..1f1b0e98c52e03de1911ea94d5c7eb2507e94c95 --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.io.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.task.MergeTask; +import org.apache.iotdb.db.metadata.MManager; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; + +public class MergePerfTest extends MergeTest{ + + private long timeConsumption; + private boolean fullMerge; + private File tempSGDir; + + public void test() throws Exception { + MManager.getInstance().init(); + tempSGDir = new File("tempSG"); + tempSGDir.mkdirs(); + setUp(); + timeConsumption = System.currentTimeMillis(); + MergeResource resource = new MergeResource(seqResources, unseqResources); + resource.setCacheDeviceMeta(true); + MergeTask mergeTask = + new MergeTask(resource, tempSGDir.getPath(), (k, v + , l) -> {}, "test", fullMerge, 100, MERGE_TEST_SG); + mergeTask.call(); + timeConsumption = System.currentTimeMillis() - timeConsumption; + tearDown(); + FileUtils.deleteDirectory(tempSGDir); + } + + public static void main(String[] args) throws Exception { + IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(-1); + + List timeConsumptions = new ArrayList<>(); + MergePerfTest perfTest = new MergePerfTest(); + + perfTest.seqFileNum = 5; + perfTest.unseqFileNum = 5; + perfTest.measurementNum = 100; + perfTest.deviceNum = 10; + perfTest.ptNum = 5000; + perfTest.flushInterval = 1000; + perfTest.fullMerge = true; + perfTest.encoding = TSEncoding.PLAIN; + + for (int i = 0; i < 1; i++) { + // cache warm-up + perfTest.test(); + } + +// int[] intParameters = new int[10]; +// for (int i = 1; i <= 10; i++) { +// intParameters[i-1] = i; +// } +// for (int param : intParameters) { +// perfTest.unseqFileNum = param; +// perfTest.test(); +// timeConsumptions.add(perfTest.timeConsumption); +// } +// long[] longParameters = new long[10]; +// for (int i = 1; i <= 10; i++) { +// longParameters[i-1] = i * 200; +// } +// for (long param : longParameters) { +// perfTest.flushInterval = param; +// perfTest.test(); +// timeConsumptions.add(perfTest.timeConsumption); +// } +// double[] doubleParameters = new double[10]; +// for (int i = 1; i <= 10; i++) { +// doubleParameters[i-1] = 0.1 * i; +// } +// for (double param : doubleParameters) { +// perfTest.unseqRatio = param; +// perfTest.test(); +// timeConsumptions.add(perfTest.timeConsumption); +// } + + System.out.println(timeConsumptions); + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java new file mode 100644 index 0000000000000000000000000000000000000000..6a02d4b0a0cb87a8909b6d8a3f338f7590c86b3f --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import org.apache.commons.io.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.merge.manage.MergeResource; +import org.apache.iotdb.db.engine.merge.task.MergeTask; +import org.apache.iotdb.db.engine.modification.Deletion; +import org.apache.iotdb.db.exception.MetadataErrorException; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.reader.resourceRelated.SeqResourceIterateReader; +import org.apache.iotdb.tsfile.exception.write.WriteProcessException; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.common.Path; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class MergeTaskTest extends MergeTest { + + private File tempSGDir; + + @Before + public void setUp() throws IOException, WriteProcessException, MetadataErrorException { + super.setUp(); + tempSGDir = new File("tempSG"); + tempSGDir.mkdirs(); + } + + @After + public void tearDown() throws IOException, StorageEngineException { + super.tearDown(); + FileUtils.deleteDirectory(tempSGDir); + } + + @Test + public void testMerge() throws Exception { + MergeTask mergeTask = + new MergeTask(new MergeResource(seqResources, unseqResources), tempSGDir.getPath(), (k, v + , l) -> {}, "test", false, 1, MERGE_TEST_SG); + mergeTask.call(); + + QueryContext context = new QueryContext(); + Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); + SeqResourceIterateReader tsFilesReader = new SeqResourceIterateReader(path, + Collections.singletonList(seqResources.get(0)), + null, context); + while (tsFilesReader.hasNext()) { + BatchData batchData = tsFilesReader.nextBatch(); + for (int i = 0; i < batchData.length(); i++) { + assertEquals(batchData.getTimeByIndex(i) + 20000.0, batchData.getDoubleByIndex(i), 0.001); + } + } + tsFilesReader.close(); + } + + @Test + public void testFullMerge() throws Exception { + MergeTask mergeTask = + new MergeTask(new MergeResource(seqResources, unseqResources), tempSGDir.getPath(), (k, v, l) -> {}, "test", + true, 1, MERGE_TEST_SG); + mergeTask.call(); + + QueryContext context = new QueryContext(); + Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); + SeqResourceIterateReader tsFilesReader = new SeqResourceIterateReader(path, + Collections.singletonList(seqResources.get(0)), + null, context); + while (tsFilesReader.hasNext()) { + BatchData batchData = tsFilesReader.nextBatch(); + for (int i = 0; i < batchData.length(); i++) { + assertEquals(batchData.getTimeByIndex(i) + 20000.0, batchData.getDoubleByIndex(i), 0.001); + } + } + tsFilesReader.close(); + } + + @Test + public void testChunkNumThreshold() throws Exception { + IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(Integer.MAX_VALUE); + MergeTask mergeTask = + new MergeTask(new MergeResource(seqResources, unseqResources), tempSGDir.getPath(), (k, v, l) -> {}, "test", + false, 1, MERGE_TEST_SG); + mergeTask.call(); + + QueryContext context = new QueryContext(); + Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); + SeqResourceIterateReader tsFilesReader = new SeqResourceIterateReader(path, + Collections.singletonList(seqResources.get(0)), + null, context); + while (tsFilesReader.hasNext()) { + BatchData batchData = tsFilesReader.nextBatch(); + for (int i = 0; i < batchData.length(); i++) { + assertEquals(batchData.getTimeByIndex(i) + 20000.0, batchData.getDoubleByIndex(i), 0.001); + } + } + tsFilesReader.close(); + } + + @Test + public void testPartialMerge1() throws Exception { + MergeTask mergeTask = + new MergeTask(new MergeResource(seqResources, unseqResources.subList(0, 1)), tempSGDir.getPath(), + (k, v, l) -> {}, "test", false, 1, MERGE_TEST_SG); + mergeTask.call(); + + QueryContext context = new QueryContext(); + Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); + SeqResourceIterateReader tsFilesReader = new SeqResourceIterateReader(path, + Collections.singletonList(seqResources.get(0)), + null, context); + while (tsFilesReader.hasNext()) { + BatchData batchData = tsFilesReader.nextBatch(); + for (int i = 0; i < batchData.length(); i++) { + if (batchData.getTimeByIndex(i) < 20) { + assertEquals(batchData.getTimeByIndex(i) + 10000.0, batchData.getDoubleByIndex(i), 0.001); + } else { + assertEquals(batchData.getTimeByIndex(i) + 0.0, batchData.getDoubleByIndex(i), 0.001); + } + } + } + tsFilesReader.close(); + } + + @Test + public void testPartialMerge2() throws Exception { + MergeTask mergeTask = + new MergeTask(new MergeResource(seqResources, unseqResources.subList(5, 6)), tempSGDir.getPath(), + (k, v, l) -> {}, "test", false, 1, MERGE_TEST_SG); + mergeTask.call(); + + QueryContext context = new QueryContext(); + Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); + SeqResourceIterateReader tsFilesReader = new SeqResourceIterateReader(path, + Collections.singletonList(seqResources.get(0)), + null, context); + while (tsFilesReader.hasNext()) { + BatchData batchData = tsFilesReader.nextBatch(); + for (int i = 0; i < batchData.length(); i++) { + assertEquals(batchData.getTimeByIndex(i) + 20000.0, batchData.getDoubleByIndex(i), 0.001); + } + } + tsFilesReader.close(); + } + + @Test + public void testPartialMerge3() throws Exception { + MergeTask mergeTask = + new MergeTask(new MergeResource(seqResources, unseqResources.subList(0, 5)), tempSGDir.getPath(), + (k, v, l) -> {}, "test", false, 1, MERGE_TEST_SG); + mergeTask.call(); + + QueryContext context = new QueryContext(); + Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); + SeqResourceIterateReader tsFilesReader = new SeqResourceIterateReader(path, + Collections.singletonList(seqResources.get(2)), + null, context); + while (tsFilesReader.hasNext()) { + BatchData batchData = tsFilesReader.nextBatch(); + for (int i = 0; i < batchData.length(); i++) { + if (batchData.getTimeByIndex(i) < 260) { + assertEquals(batchData.getTimeByIndex(i) + 10000.0, batchData.getDoubleByIndex(i), 0.001); + } else { + assertEquals(batchData.getTimeByIndex(i) + 0.0, batchData.getDoubleByIndex(i), 0.001); + } + } + } + tsFilesReader.close(); + } + + @Test + public void mergeWithDeletionTest() throws Exception { + seqResources.get(0).getModFile().write(new Deletion(new Path(deviceIds[0], + measurementSchemas[0].getMeasurementId()), 10000, 49)); + seqResources.get(0).getModFile().close(); + + + MergeTask mergeTask = + new MergeTask(new MergeResource(seqResources, unseqResources.subList(0, 1)), tempSGDir.getPath(), + (k, v, l) -> { + try { + seqResources.get(0).removeModFile(); + } catch (IOException e) { + e.printStackTrace(); + } + }, "test", false, 1, MERGE_TEST_SG); + mergeTask.call(); + + QueryContext context = new QueryContext(); + Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); + SeqResourceIterateReader tsFilesReader = new SeqResourceIterateReader(path, + Collections.singletonList(seqResources.get(0)), + null, context); + int count = 0; + while (tsFilesReader.hasNext()) { + BatchData batchData = tsFilesReader.nextBatch(); + for (int i = 0; i < batchData.length(); i++) { + if (batchData.getTimeByIndex(i) <= 20) { + assertEquals(batchData.getTimeByIndex(i) + 10000.0, batchData.getDoubleByIndex(i), 0.001); + } else { + assertEquals(batchData.getTimeByIndex(i), batchData.getDoubleByIndex(i), 0.001); + } + count ++; + } + } + assertEquals(70, count); + tsFilesReader.close(); + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java new file mode 100644 index 0000000000000000000000000000000000000000..cab3a59a6e4f3c7bed1840cd932ecb36e402f82b --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.engine.merge; + +import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_SEPARATOR; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache; +import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache; +import org.apache.iotdb.db.engine.merge.manage.MergeManager; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.MetadataErrorException; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.metadata.MManager; +import org.apache.iotdb.db.query.control.FileReaderManager; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.tsfile.exception.write.WriteProcessException; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.TsFileWriter; +import org.apache.iotdb.tsfile.write.record.TSRecord; +import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; + +abstract class MergeTest { + + static final String MERGE_TEST_SG = "root.mergeTest"; + + int seqFileNum = 5; + int unseqFileNum = 5; + int measurementNum = 10; + int deviceNum = 10; + long ptNum = 100; + long flushInterval = 20; + TSEncoding encoding = TSEncoding.PLAIN; + + String[] deviceIds; + MeasurementSchema[] measurementSchemas; + + List seqResources = new ArrayList<>(); + List unseqResources = new ArrayList<>(); + + private int prevMergeChunkThreshold; + + @Before + public void setUp() throws IOException, WriteProcessException, MetadataErrorException { + MManager.getInstance().init(); + prevMergeChunkThreshold = + IoTDBDescriptor.getInstance().getConfig().getChunkMergePointThreshold(); + IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(-1); + prepareSeries(); + prepareFiles(seqFileNum, unseqFileNum); + MergeManager.getINSTANCE().start(); + } + + @After + public void tearDown() throws IOException, StorageEngineException { + removeFiles(); + seqResources.clear(); + unseqResources.clear(); + IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(prevMergeChunkThreshold); + TsFileMetaDataCache.getInstance().clear(); + DeviceMetaDataCache.getInstance().clear(); + MManager.getInstance().clear(); + EnvironmentUtils.cleanAllDir(); + MergeManager.getINSTANCE().stop(); + } + + private void prepareSeries() throws MetadataErrorException { + measurementSchemas = new MeasurementSchema[measurementNum]; + for (int i = 0; i < measurementNum; i++) { + measurementSchemas[i] = new MeasurementSchema("sensor" + i, TSDataType.DOUBLE, + encoding, CompressionType.UNCOMPRESSED); + } + deviceIds = new String[deviceNum]; + for (int i = 0; i < deviceNum; i++) { + deviceIds[i] = MERGE_TEST_SG + PATH_SEPARATOR + "device" + i; + } + MManager.getInstance().setStorageLevelToMTree(MERGE_TEST_SG); + for (String device : deviceIds) { + for (MeasurementSchema measurementSchema : measurementSchemas) { + MManager.getInstance().addPathToMTree( + device + PATH_SEPARATOR + measurementSchema.getMeasurementId(), measurementSchema + .getType(), measurementSchema.getEncodingType(), measurementSchema.getCompressor(), + Collections.emptyMap()); + } + } + } + + private void prepareFiles(int seqFileNum, int unseqFileNum) throws IOException, WriteProcessException { + for (int i = 0; i < seqFileNum; i++) { + File file = new File(i + "seq.tsfile"); + TsFileResource tsFileResource = new TsFileResource(file); + seqResources.add(tsFileResource); + prepareFile(tsFileResource, i * ptNum, ptNum, 0); + } + for (int i = 0; i < unseqFileNum; i++) { + File file = new File(i + "unseq.tsfile"); + TsFileResource tsFileResource = new TsFileResource(file); + unseqResources.add(tsFileResource); + prepareFile(tsFileResource, i * ptNum, ptNum * (i + 1) / unseqFileNum, 10000); + } + File file = new File(unseqFileNum + "unseq.tsfile"); + TsFileResource tsFileResource = new TsFileResource(file); + unseqResources.add(tsFileResource); + prepareFile(tsFileResource, 0, ptNum * unseqFileNum, 20000); + } + + private void removeFiles() throws IOException { + for (TsFileResource tsFileResource : seqResources) { + tsFileResource.remove(); + } + for (TsFileResource tsFileResource : unseqResources) { + tsFileResource.remove(); + } + FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders(); + FileReaderManager.getInstance().stop(); + } + + private void prepareFile(TsFileResource tsFileResource, long timeOffset, long ptNum, + long valueOffset) + throws IOException, WriteProcessException { + TsFileWriter fileWriter = new TsFileWriter(tsFileResource.getFile()); + for (MeasurementSchema measurementSchema : measurementSchemas) { + fileWriter.addMeasurement(measurementSchema); + } + for (long i = timeOffset; i < timeOffset + ptNum; i++) { + for (int j = 0; j < deviceNum; j++) { + TSRecord record = new TSRecord(i, deviceIds[j]); + for (int k = 0; k < measurementNum; k++) { + record.addTuple(DataPoint.getDataPoint(measurementSchemas[k].getType(), + measurementSchemas[k].getMeasurementId(), String.valueOf(i + valueOffset))); + } + fileWriter.write(record); + tsFileResource.updateStartTime(deviceIds[j], i); + tsFileResource.updateEndTime(deviceIds[j], i); + } + if ((i + 1) % flushInterval == 0) { + fileWriter.flushForTest(); + } + } + fileWriter.close(); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java index c29b0b21a4286064efbbc61b1b43d2e6fd32d495..39b354782ffc365d9d05b961e24dba1419a61daa 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java @@ -110,7 +110,6 @@ public class DeletionFileNodeTest { SingleSeriesExpression expression = new SingleSeriesExpression(new Path(processorName, measurements[5]), null); - QueryResourceManager.getInstance().beginQueryOfGivenExpression(TEST_QUERY_JOB_ID, expression); QueryDataSource dataSource = QueryResourceManager.getInstance() .getQueryDataSource(expression.getSeriesPath(), TEST_QUERY_CONTEXT); @@ -194,7 +193,6 @@ public class DeletionFileNodeTest { SingleSeriesExpression expression = new SingleSeriesExpression(new Path(processorName, measurements[5]), null); - QueryResourceManager.getInstance().beginQueryOfGivenExpression(TEST_QUERY_JOB_ID, expression); QueryDataSource dataSource = QueryResourceManager.getInstance() .getQueryDataSource(expression.getSeriesPath(), TEST_QUERY_CONTEXT); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java index f04d9a8f7d374617488d025705bbaae074620102..3eb681a48a905bbaff400d255ae0bda8ea645745 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java @@ -40,6 +40,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.common.Path; +import org.apache.iotdb.tsfile.read.common.RowRecord; import org.apache.iotdb.tsfile.read.expression.QueryExpression; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.iotdb.tsfile.write.record.TSRecord; @@ -185,8 +186,9 @@ public class DeletionQueryTest { int count = 0; while (dataSet.hasNext()) { - dataSet.next(); + RowRecord record = dataSet.next(); count++; + System.out.println(record.getTimestamp()); } assertEquals(150, count); } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java index f0ee8f00fd34bf960a939e2dc39afff6ce453883..6539a3ef0b094ac71d3a6944a421a9997eb19a3c 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java @@ -18,11 +18,23 @@ */ package org.apache.iotdb.db.engine.storagegroup; +import static org.junit.Assert.assertFalse; + +import java.io.File; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.ArrayList; + import org.apache.iotdb.db.engine.MetadataManagerHelper; +import org.apache.iotdb.db.engine.merge.manage.MergeManager; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; -import org.apache.iotdb.db.exception.StorageGroupProcessorException; + +import org.apache.iotdb.db.exception.ProcessorException; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; + import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.control.JobFileManager; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.write.record.TSRecord; @@ -34,18 +46,20 @@ import org.junit.Test; public class StorageGroupProcessorTest { - private String storageGroup = "storage_group1"; + private String storageGroup = "root.vehicle.d0"; private String systemDir = "data/info"; private String deviceId = "root.vehicle.d0"; private String measurementId = "s0"; private StorageGroupProcessor processor; private QueryContext context = EnvironmentUtils.TEST_QUERY_CONTEXT; + private AtomicLong mergeLock; @Before public void setUp() throws Exception { MetadataManagerHelper.initMetadata(); EnvironmentUtils.envSetUp(); - processor = new StorageGroupProcessor(systemDir, storageGroup); + processor = new DummySGP(systemDir, storageGroup); + MergeManager.getINSTANCE().start(); } @After @@ -53,6 +67,7 @@ public class StorageGroupProcessorTest { processor.syncDeleteDataFiles(); EnvironmentUtils.cleanEnv(); EnvironmentUtils.cleanDir("data"); + MergeManager.getINSTANCE().stop(); } @@ -66,7 +81,8 @@ public class StorageGroupProcessorTest { } processor.waitForAllCurrentTsFileProcessorsClosed(); - QueryDataSource queryDataSource = processor.query(deviceId, measurementId, context); + QueryDataSource queryDataSource = processor.query(deviceId, measurementId, context, + null); Assert.assertEquals(10, queryDataSource.getSeqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { @@ -74,9 +90,63 @@ public class StorageGroupProcessorTest { } } + @Test + public void testIoTDBRowBatchWriteAndSyncClose() { + + String[] measurements = new String[2]; + measurements[0] = "s0"; + measurements[1] = "s1"; + List dataTypes = new ArrayList<>(); + dataTypes.add(TSDataType.INT32.ordinal()); + dataTypes.add(TSDataType.INT64.ordinal()); + + BatchInsertPlan batchInsertPlan1 = new BatchInsertPlan("root.vehicle.d0", measurements, dataTypes); + + long[] times = new long[100]; + Object[] columns = new Object[2]; + columns[0] = new int[100]; + columns[1] = new long[100]; + + for (int r = 0; r < 100; r++) { + times[r] = r; + ((int[]) columns[0])[r] = 1; + ((long[]) columns[1])[r] = 1; + } + batchInsertPlan1.setTimes(times); + batchInsertPlan1.setColumns(columns); + batchInsertPlan1.setRowCount(times.length); + + processor.insertBatch(batchInsertPlan1); + processor.putAllWorkingTsFileProcessorIntoClosingList(); + + BatchInsertPlan batchInsertPlan2 = new BatchInsertPlan("root.vehicle.d0", measurements, dataTypes); + + for (int r = 50; r < 149; r++) { + times[r-50] = r; + ((int[]) columns[0])[r-50] = 1; + ((long[]) columns[1])[r-50] = 1; + } + batchInsertPlan2.setTimes(times); + batchInsertPlan2.setColumns(columns); + batchInsertPlan2.setRowCount(times.length); + + processor.insertBatch(batchInsertPlan2); + processor.putAllWorkingTsFileProcessorIntoClosingList(); + processor.waitForAllCurrentTsFileProcessorsClosed(); + + QueryDataSource queryDataSource = processor.query(deviceId, measurementId, context, + null); + + Assert.assertEquals(2, queryDataSource.getSeqResources().size()); + Assert.assertEquals(1, queryDataSource.getUnseqResources().size()); + for (TsFileResource resource : queryDataSource.getSeqResources()) { + Assert.assertTrue(resource.isClosed()); + } + } + @Test - public void testSeqAndUnSeqSyncClose() throws StorageGroupProcessorException { + public void testSeqAndUnSeqSyncClose() { for (int j = 21; j <= 30; j++) { TSRecord record = new TSRecord(j, deviceId); @@ -95,7 +165,8 @@ public class StorageGroupProcessorTest { processor.waitForAllCurrentTsFileProcessorsClosed(); - QueryDataSource queryDataSource = processor.query(deviceId, measurementId, context); + QueryDataSource queryDataSource = processor.query(deviceId, measurementId, context, + null); Assert.assertEquals(10, queryDataSource.getSeqResources().size()); Assert.assertEquals(10, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { @@ -106,4 +177,55 @@ public class StorageGroupProcessorTest { } } + @Test + public void testMerge() { + + mergeLock = new AtomicLong(0); + for (int j = 21; j <= 30; j++) { + TSRecord record = new TSRecord(j, deviceId); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); + processor.insert(new InsertPlan(record)); + processor.putAllWorkingTsFileProcessorIntoClosingList(); + } + processor.waitForAllCurrentTsFileProcessorsClosed(); + + for (int j = 10; j >= 1; j--) { + TSRecord record = new TSRecord(j, deviceId); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); + processor.insert(new InsertPlan(record)); + processor.putAllWorkingTsFileProcessorIntoClosingList(); + } + + processor.waitForAllCurrentTsFileProcessorsClosed(); + processor.merge(true); + while (mergeLock.get() == 0) { + // wait + } + + QueryDataSource queryDataSource = processor.query(deviceId, measurementId, context, + null); + Assert.assertEquals(10, queryDataSource.getSeqResources().size()); + Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + for (TsFileResource resource : queryDataSource.getSeqResources()) { + Assert.assertTrue(resource.isClosed()); + } + for (TsFileResource resource : queryDataSource.getUnseqResources()) { + Assert.assertTrue(resource.isClosed()); + } + } + + class DummySGP extends StorageGroupProcessor { + + DummySGP(String systemInfoDir, String storageGroupName) throws ProcessorException { + super(systemInfoDir, storageGroupName); + } + + @Override + protected void mergeEndAction(List seqFiles, List unseqFiles, + File mergeLog) { + super.mergeEndAction(seqFiles, unseqFiles, mergeLog); + mergeLock.incrementAndGet(); + assertFalse(mergeLog.exists()); + } + } } \ No newline at end of file diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java index 9508f4ca6147bc7ca861e65ed444f3e29a18c6c2..a61534ab9d9baa465a7908e17393670692c161c5 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java @@ -36,7 +36,7 @@ import org.apache.iotdb.db.exception.TsFileProcessorException; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.utils.EnvironmentUtils; -import org.apache.iotdb.db.utils.FileSchemaUtils; +import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.iotdb.db.utils.TimeValuePair; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData; @@ -77,7 +77,7 @@ public class TsFileProcessorTest { public void testWriteAndFlush() throws WriteProcessException, IOException, TsFileProcessorException { processor = new TsFileProcessor(storageGroup, new File(filePath), - FileSchemaUtils.constructFileSchema(deviceId), SysTimeVersionController.INSTANCE, x -> { + SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> { }, () -> true, true); @@ -124,7 +124,7 @@ public class TsFileProcessorTest { public void testWriteAndRestoreMetadata() throws IOException { processor = new TsFileProcessor(storageGroup, new File(filePath), - FileSchemaUtils.constructFileSchema(deviceId), SysTimeVersionController.INSTANCE, x -> { + SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> { }, () -> true, true); @@ -191,7 +191,7 @@ public class TsFileProcessorTest { public void testMultiFlush() throws WriteProcessException, IOException, TsFileProcessorException { processor = new TsFileProcessor(storageGroup, new File(filePath), - FileSchemaUtils.constructFileSchema(deviceId), SysTimeVersionController.INSTANCE, x -> { + SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, x -> { }, () -> true, true); @@ -227,7 +227,7 @@ public class TsFileProcessorTest { public void testWriteAndClose() throws WriteProcessException, IOException { processor = new TsFileProcessor(storageGroup, new File(filePath), - FileSchemaUtils.constructFileSchema(deviceId), SysTimeVersionController.INSTANCE, + SchemaUtils.constructSchema(deviceId), SysTimeVersionController.INSTANCE, unsealedTsFileProcessor -> { TsFileResource resource = unsealedTsFileProcessor.getTsFileResource(); synchronized (resource) { diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IOTDBFillIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IOTDBFillIT.java index 334f9438d566b0a1253bfd2ac0b74a6b7f50db7a..d4886f1223fbc94542ec232b27e8849b7f68585e 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IOTDBFillIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IOTDBFillIT.java @@ -127,144 +127,132 @@ public class IOTDBFillIT { "70,null,null,null", "625,null,false,null" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select temperature,status, hardware from " + "root.ln.wf01.wt01 where time = 3 " + "Fill(int32[linear, 5ms, 5ms], double[linear, 5ms, 5ms], boolean[previous, 5ms])"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select temperature,status, hardware " + "from root.ln.wf01.wt01 where time = 70 Fill(int32[linear, 500ms, 500ms], " + "double[linear, 500ms, 500ms], boolean[previous, 500ms])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select temperature,status, hardware " + "from root.ln.wf01.wt01 where time = 70 " + "Fill(int32[linear, 25ms, 25ms], double[linear, 25ms, 25ms], boolean[previous, 5ms])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select temperature,status, hardware " + "from root.ln.wf01.wt01 where time = 625 " + "Fill(int32[linear, 25ms, 25ms], double[linear, 25ms, 25ms], boolean[previous, 5ms])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @Test - public void PreviousFillTest() throws SQLException { + public void PreviousFillTest() { String[] retArray1 = new String[]{ "3,3.3,false,33", "70,50.5,false,550", "70,null,null,null" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select temperature,status, hardware " + "from root.ln.wf01.wt01 where time = 3 " + "Fill(int32[previous, 5ms], double[previous, 5ms], boolean[previous, 5ms])"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select temperature,status, hardware " + "from root.ln.wf01.wt01 where time = 70 " + "Fill(int32[previous, 500ms], double[previous, 500ms], boolean[previous, 500ms])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select temperature,status, hardware " + "from root.ln.wf01.wt01 where time = 70 " + "Fill(int32[previous, 15ms], double[previous, 15ms], boolean[previous, 5ms])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray1.length, cnt); } - statement.close(); - Assert.assertEquals(retArray1.length, cnt); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -274,70 +262,58 @@ public class IOTDBFillIT { "3,3.3,false,33", "70,70.34,false,374" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select temperature,status, hardware " + "from root.ln.wf01.wt01 where time = 3 " + "Fill(int32[linear], double[linear], boolean[previous])"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select temperature,status, hardware " + "from root.ln.wf01.wt01 where time = 70 " + "Fill(int32[linear], double[linear], boolean[previous])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) - + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(TEMPERATURE_STR) + + "," + resultSet.getString(STATUS_STR) + "," + resultSet.getString(HARDWARE_STR); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } } - statement.close(); - } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } private void prepareData() throws SQLException { - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { + for (String sql : dataSet1) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IOTDBGroupByIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IOTDBGroupByIT.java index 07659973fc6533a5bae2a580c40e9fa2c461b643..a27999294a2971e986d0df1ae58069289440a1d2 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IOTDBGroupByIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IOTDBGroupByIT.java @@ -152,57 +152,51 @@ public class IOTDBGroupByIT { "145,1,200.2,200.2", "310,0,0.0,null" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "select count(temperature), sum(temperature), mean(temperature) from " + "root.ln.wf01.wt01 where time > 3 " + "GROUP BY (20ms, 5,[2,30], [35,37], [50, 160], [310, 314])"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(count("root.ln.wf01.wt01.temperature")) + "," + - resultSet.getString(sum("root.ln.wf01.wt01.temperature")) + "," + resultSet - .getString(mean("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(count("root.ln.wf01.wt01.temperature")) + "," + + resultSet.getString(sum("root.ln.wf01.wt01.temperature")) + "," + resultSet + .getString(mean("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray1.length, cnt); } - Assert.assertEquals(retArray1.length, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute( "select count(temperature), sum(temperature), mean(temperature) from " + "root.ln.wf01.wt01 where temperature > 3 " + "GROUP BY (20ms, 5,[2,30], [35,37], [50, 160], [310, 314])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(count("root.ln.wf01.wt01.temperature")) + "," + - resultSet.getString(sum("root.ln.wf01.wt01.temperature")) + "," + resultSet - .getString(mean("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray2[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(count("root.ln.wf01.wt01.temperature")) + "," + + resultSet.getString(sum("root.ln.wf01.wt01.temperature")) + "," + resultSet + .getString(mean("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray2[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray2.length, cnt); } - Assert.assertEquals(retArray2.length, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -232,59 +226,53 @@ public class IOTDBGroupByIT { "145,200.2,200.2,150,150", "310,null,null,null,null" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute( "select max_value(temperature), min_value(temperature), max_time(temperature), " + "min_time(temperature) from root.ln.wf01.wt01 where time > 3 " + "GROUP BY (20ms, 5,[2,30], [35,37], [50, 160], [310, 314])"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(max_value("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(min_value("root.ln.wf01.wt01.temperature")) + "," - + resultSet.getString(max_time("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(min_time("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(max_value("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(min_value("root.ln.wf01.wt01.temperature")) + "," + + resultSet.getString(max_time("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(min_time("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray1.length, cnt); } - Assert.assertEquals(retArray1.length, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute( "select max_value(temperature), min_value(temperature), max_time(temperature), " + "min_time(temperature) from root.ln.wf01.wt01 where temperature > 3 " + "GROUP BY (20ms, 5,[2,30], [35,37], [50, 160], [310, 314])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(max_value("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(min_value("root.ln.wf01.wt01.temperature")) + "," - + resultSet.getString(max_time("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(min_time("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray2[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(max_value("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(min_value("root.ln.wf01.wt01.temperature")) + "," + + resultSet.getString(max_time("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(min_time("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray2[cnt], ans); + cnt++; + } } Assert.assertEquals(retArray2.length, cnt); - statement.close(); - } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -314,54 +302,48 @@ public class IOTDBGroupByIT { "145,200.2,200.2", "310,null,null" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "select last(temperature), first(temperature) from root.ln.wf01.wt01 where time > 3 " + "GROUP BY (20ms, 5,[2,30], [35,37], [50, 160], [310, 314])"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(last("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(first("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(last("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(first("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray1.length, cnt); } - Assert.assertEquals(retArray1.length, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute( "select first(temperature), last(temperature) from root.ln.wf01.wt01 " + "where temperature > 3 " + "GROUP BY (20ms, 5,[2,30], [35,37], [50, 160], [310, 314])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(last("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(first("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray2[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(last("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(first("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray2[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray2.length, cnt); } - Assert.assertEquals(retArray2.length, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -378,59 +360,53 @@ public class IOTDBGroupByIT { "620,500.5,1,620,620" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "select min_value(temperature), count(temperature), max_time(temperature), " + "min_time(temperature) from root.ln.wf01.wt01 where time > 3 GROUP BY " + "(590ms, 30, [2, 30], [30, 120], [100, 120], [123, 125], [155, 550], [540, 680])"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(min_value("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(count("root.ln.wf01.wt01.temperature")) + "," + - resultSet.getString(max_time("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(min_time("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(min_value("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(count("root.ln.wf01.wt01.temperature")) + "," + + resultSet.getString(max_time("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(min_time("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray1.length, cnt); } - Assert.assertEquals(retArray1.length, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute( "select min_value(temperature), count (temperature), max_time(temperature), " + "min_time(temperature) from root.ln.wf01.wt01 where temperature > 3 GROUP BY " + "(590ms, 30, [2, 30], [30, 120], [100, 120], [123, 125], [155, 550],[540, 680])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(min_value("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(count("root.ln.wf01.wt01.temperature")) + "," - + resultSet.getString(max_time("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(min_time("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray2[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(min_value("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(count("root.ln.wf01.wt01.temperature")) + "," + + resultSet.getString(max_time("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(min_time("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray2[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray2.length, cnt); } - Assert.assertEquals(retArray2.length, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -446,79 +422,66 @@ public class IOTDBGroupByIT { "585,null,null,0.0", "590,500.5,200.2,700.7" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute( "select last(temperature), first(temperature), sum(temperature) from " + "root.ln.wf01.wt01 where time > 3 " + "GROUP BY (80ms, 30,[50,100], [615, 650])"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(last("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(first("root.ln.wf01.wt01.temperature")) + "," - + resultSet.getString(sum("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray1[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(last("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(first("root.ln.wf01.wt01.temperature")) + "," + + resultSet.getString(sum("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray1[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray1.length, cnt); } - Assert.assertEquals(retArray1.length, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute( "select first(temperature), last(temperature), sum(temperature) from " + "root.ln.wf01.wt01 where temperature > 3 " + "GROUP BY (80ms, 30,[50,100], [585,590], [615, 650])"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet - .getString(last("root.ln.wf01.wt01.temperature")) - + "," + resultSet.getString(first("root.ln.wf01.wt01.temperature")) + "," - + resultSet.getString(sum("root.ln.wf01.wt01.temperature")); - Assert.assertEquals(retArray2[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet + .getString(last("root.ln.wf01.wt01.temperature")) + + "," + resultSet.getString(first("root.ln.wf01.wt01.temperature")) + "," + + resultSet.getString(sum("root.ln.wf01.wt01.temperature")); + Assert.assertEquals(retArray2[cnt], ans); + cnt++; + } + Assert.assertEquals(retArray2.length, cnt); } - Assert.assertEquals(retArray2.length, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } - public void prepareData() throws SQLException { - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + private void prepareData() { + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement();) { for (String sql : dataSet1) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java index be1eee8404b5110719079fc46ca7dac5f1b8125c..ab613f3c993c1cad2f6f57e796cb7631a34cec6f 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java @@ -112,63 +112,57 @@ public class IoTDBAggregationIT { "0,4", "0,3" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute( "select count(temperature) from root.ln.wf01.wt01 where time > 3"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + - resultSet.getString(count(TEMPERATURE_STR)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet();) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + + resultSet.getString(count(TEMPERATURE_STR)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute( "select min_time(temperature) from root.ln.wf01.wt01 where time > 3"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + - resultSet.getString(min_time(TEMPERATURE_STR)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + + resultSet.getString(min_time(TEMPERATURE_STR)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(2, cnt); } - Assert.assertEquals(2, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute( "select min_time(temperature) from root.ln.wf01.wt01 where temperature > 3"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + - resultSet.getString(min_time(TEMPERATURE_STR)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + + resultSet.getString(min_time(TEMPERATURE_STR)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(3, cnt); } - Assert.assertEquals(3, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -178,103 +172,91 @@ public class IoTDBAggregationIT { "0,2001,2001,2001,2001", "0,7500,7500,7500,7500" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select count(s0),count(s1),count(s2),count(s3) " + "from root.vehicle.d0 where time >= 6000 and time <= 9000"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) - + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)) - + "," + resultSet.getString(count(d0s3)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet();) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) + + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)) + + "," + resultSet.getString(count(d0s3)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select count(s0),count(s1),count(s2),count(s3) " + "from root.vehicle.d0"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) - + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)) - + "," + resultSet.getString(count(d0s3)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) + + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)) + + "," + resultSet.getString(count(d0s3)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(2, cnt); } - Assert.assertEquals(2, cnt); - statement.close(); - } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @Test - public void firstTest() throws SQLException { + public void firstTest() { String[] retArray = new String[]{ "0,2000,2000,2000.0,2000", "0,500,500,500.0,500" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select first(s0),first(s1),first(s2),first(s3) " + "from root.vehicle.d0 where time >= 1500 and time <= 9000"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(first(d0s0)) - + "," + resultSet.getString(first(d0s1)) + "," + resultSet.getString(first(d0s2)) - + "," + resultSet.getString(first(d0s3)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(first(d0s0)) + + "," + resultSet.getString(first(d0s1)) + "," + resultSet.getString(first(d0s2)) + + "," + resultSet.getString(first(d0s3)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select first(s0),first(s1),first(s2),first(s3) " + "from root.vehicle.d0"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(first(d0s0)) - + "," + resultSet.getString(first(d0s1)) + "," + resultSet.getString(first(d0s2)) - + "," + resultSet.getString(first(d0s3)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(first(d0s0)) + + "," + resultSet.getString(first(d0s1)) + "," + resultSet.getString(first(d0s2)) + + "," + resultSet.getString(first(d0s3)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(2, cnt); } - Assert.assertEquals(2, cnt); - statement.close(); - } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -285,63 +267,56 @@ public class IoTDBAggregationIT { "0,1499,1499.0", "0,2200,2200.0" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select last(s0),last(s2) " + "from root.vehicle.d0 where time >= 1500 and time < 9000"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) - + "," + resultSet.getString(last(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) + + "," + resultSet.getString(last(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select last(s0),last(s2) " + "from root.vehicle.d0 where time <= 1600"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) - + "," + resultSet.getString(last(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) + + "," + resultSet.getString(last(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(2, cnt); } - Assert.assertEquals(2, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select last(s0),last(s2) " + "from root.vehicle.d0 where time <= 2200"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) - + "," + resultSet.getString(last(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) + + "," + resultSet.getString(last(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(3, cnt); } - Assert.assertEquals(3, cnt); - statement.close(); - } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -351,47 +326,44 @@ public class IoTDBAggregationIT { "0,8499,500", "0,2499,2000" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select max_time(s0),min_time(s2) " + "from root.vehicle.d0 where time >= 100 and time < 9000"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_time(d0s0)) - + "," + resultSet.getString(min_time(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_time(d0s0)) + + "," + resultSet.getString(min_time(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select max_time(s0),min_time(s2) " + "from root.vehicle.d0 where time <= 2500 and time > 1800"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_time(d0s0)) - + "," + resultSet.getString(min_time(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_time(d0s0)) + + "," + resultSet.getString(min_time(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(2, cnt); } - Assert.assertEquals(2, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -401,174 +373,149 @@ public class IoTDBAggregationIT { "0,8499,500.0", "0,2499,500.0" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select max_value(s0),min_value(s2) " + "from root.vehicle.d0 where time >= 100 and time < 9000"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_value(d0s0)) - + "," + resultSet.getString(min_value(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_value(d0s0)) + + "," + resultSet.getString(min_value(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select max_value(s0),min_value(s2) " + "from root.vehicle.d0 where time < 2500"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_value(d0s0)) - + "," + resultSet.getString(min_value(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_value(d0s0)) + + "," + resultSet.getString(min_value(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(2, cnt); } - Assert.assertEquals(2, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @Test - public void meanSumTest() throws SQLException { + public void meanSumTest() { String[] retArray = new String[]{ "0,1.4508E7,7250.374812593703", "0,626750.0,1250.998003992016" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select sum(s0),mean(s2)" + "from root.vehicle.d0 where time >= 6000 and time <= 9000"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) - + "," + resultSet.getString(mean(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) + + "," + resultSet.getString(mean(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select sum(s0),mean(s2)" + "from root.vehicle.d0 where time >= 1000 and time <= 2000"); - Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) - + "," + resultSet.getString(mean(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) + + "," + resultSet.getString(mean(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(2, cnt); } - Assert.assertEquals(2, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @Test public void meanSumErrorTest() throws SQLException { - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select mean(s3)" + - "from root.vehicle.d0 where time >= 6000 and time <= 9000"); - Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - try { + "from root.vehicle.d0 where time >= 6000 and time <= 9000"); + Assert.assertTrue(hasResultSet); + try (ResultSet resultSet = statement.getResultSet()) { resultSet.next(); fail(); } catch (Exception e) { - Assert.assertEquals("Unsupported data type in aggregation MEAN : TEXT", e.getMessage()); + Assert.assertEquals("Internal server error: Unsupported data type in aggregation MEAN : TEXT", e.getMessage()); } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select sum(s3)" + "from root.vehicle.d0 where time >= 6000 and time <= 9000"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - try { + try (ResultSet resultSet = statement.getResultSet()) { resultSet.next(); fail(); } catch (Exception e) { - Assert.assertEquals("Unsupported data type in aggregation SUM : TEXT", e.getMessage()); + Assert.assertEquals("Internal server error: Unsupported data type in aggregation SUM : TEXT", e.getMessage()); } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select mean(s4)" + "from root.vehicle.d0 where time >= 6000 and time <= 9000"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - try { + + try (ResultSet resultSet = statement.getResultSet()) { resultSet.next(); fail(); } catch (Exception e) { - Assert.assertEquals("Unsupported data type in aggregation MEAN : BOOLEAN", e.getMessage()); + Assert.assertEquals("Internal server error: Unsupported data type in aggregation MEAN : BOOLEAN", e.getMessage()); } - statement.close(); - statement = connection.createStatement(); hasResultSet = statement.execute("select sum(s4)" + "from root.vehicle.d0 where time >= 6000 and time <= 9000"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - try { + try (ResultSet resultSet = statement.getResultSet()) { resultSet.next(); fail(); } catch (Exception e) { - Assert.assertEquals("Unsupported data type in aggregation SUM : BOOLEAN", e.getMessage()); + Assert.assertEquals("Internal server error: Unsupported data type in aggregation SUM : BOOLEAN", e.getMessage()); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } private void prepareData() throws SQLException { - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { + for (String sql : creationSqls) { statement.execute(sql); } @@ -576,44 +523,43 @@ public class IoTDBAggregationIT { for (String sql : dataSet2) { statement.execute(sql); } - statement.close(); - statement = connection.createStatement(); // prepare BufferWrite file for (int i = 5000; i < 7000; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "true")); + statement.execute(String + .format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "true")); } statement.execute("flush"); for (int i = 7500; i < 8500; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "false")); + statement.execute(String + .format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "false")); } statement.execute("flush"); // prepare Unseq-File for (int i = 500; i < 1500; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "true")); + statement.execute(String + .format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "true")); } statement.execute("flush"); for (int i = 3000; i < 6500; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "false")); + statement.execute(String + .format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "false")); } -// statement.execute("merge"); + statement.execute("merge"); // prepare BufferWrite cache for (int i = 9000; i < 10000; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "true")); + statement.execute(String + .format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "true")); } // prepare Overflow cache for (int i = 2000; i < 2500; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "false")); + statement.execute(String + .format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", "false")); } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationLargeDataIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationLargeDataIT.java index 3508df9b84da13d8c12f1ce16980a509b410c69b..7de74bc68f94224d241b84e722e42e23f5d555e9 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationLargeDataIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationLargeDataIT.java @@ -129,28 +129,27 @@ public class IoTDBAggregationLargeDataIT { public void test() throws ClassNotFoundException, SQLException { insertSQL(); - Connection connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - lastAggreWithSingleFilterTest(); - meanAggreWithSingleFilterTest(); - sumAggreWithSingleFilterTest(); - firstAggreWithSingleFilterTest(); - countAggreWithSingleFilterTest(); - minMaxTimeAggreWithSingleFilterTest(); - minValueAggreWithSingleFilterTest(); - maxValueAggreWithSingleFilterTest(); - - lastAggreWithMultiFilterTest(); - countAggreWithMultiFilterTest(); - minTimeAggreWithMultiFilterTest(); - maxTimeAggreWithMultiFilterTest(); - minValueAggreWithMultiFilterTest(); - maxValueAggreWithMultiFilterTest(); - meanAggreWithMultiFilterTest(); - sumAggreWithMultiFilterTest(); - firstAggreWithMultiFilterTest(); - connection.close(); - + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");) { + lastAggreWithSingleFilterTest(); + meanAggreWithSingleFilterTest(); + sumAggreWithSingleFilterTest(); + firstAggreWithSingleFilterTest(); + countAggreWithSingleFilterTest(); + minMaxTimeAggreWithSingleFilterTest(); + minValueAggreWithSingleFilterTest(); + maxValueAggreWithSingleFilterTest(); + + lastAggreWithMultiFilterTest(); + countAggreWithMultiFilterTest(); + minTimeAggreWithMultiFilterTest(); + maxTimeAggreWithMultiFilterTest(); + minValueAggreWithMultiFilterTest(); + maxValueAggreWithMultiFilterTest(); + meanAggreWithMultiFilterTest(); + sumAggreWithMultiFilterTest(); + firstAggreWithMultiFilterTest(); + } } private void lastAggreWithMultiFilterTest() throws SQLException { @@ -158,37 +157,31 @@ public class IoTDBAggregationLargeDataIT { "0,9,39,63.0,E,true" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "select last(s0),last(s1),last(s2),last(s3),last(s4)" + " from root.vehicle.d0 where s1 >= 0"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + - resultSet.getString(Constant.last(d0s0)) + "," + resultSet - .getString(Constant.last(d0s1)) - + "," + resultSet.getString(Constant.last(d0s2)) + "," + - resultSet.getString(Constant.last(d0s3)) + "," + resultSet - .getString(Constant.last(d0s4)); - // System.out.println("============ " + ans); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + + resultSet.getString(Constant.last(d0s0)) + "," + resultSet + .getString(Constant.last(d0s1)) + + "," + resultSet.getString(Constant.last(d0s2)) + "," + + resultSet.getString(Constant.last(d0s3)) + "," + resultSet + .getString(Constant.last(d0s4)); + // System.out.println("============ " + ans); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -197,139 +190,118 @@ public class IoTDBAggregationLargeDataIT { "0,9,39,63.0" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select last(s0),last(s1),last(s2)" + " from root.vehicle.d0 where s1 >= 0"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + - resultSet.getString(Constant.last(d0s0)) + "," - + resultSet.getString(Constant.last(d0s1)) + "," + - resultSet.getString(Constant.last(d0s2)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + + resultSet.getString(Constant.last(d0s0)) + "," + + resultSet.getString(Constant.last(d0s1)) + "," + + resultSet.getString(Constant.last(d0s2)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } - private void sumAggreWithSingleFilterTest() throws SQLException { + private void sumAggreWithSingleFilterTest() { String[] retArray = new String[]{ "0,55061.0,156752.0,20254" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select sum(s0),sum(s1),sum(s2)" + " from root.vehicle.d0 where s1 >= 0"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) + "," - + resultSet.getString(sum(d0s1)) + "," + Math - .round(resultSet.getDouble(Constant.sum(d0s2))); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) + "," + + resultSet.getString(sum(d0s1)) + "," + Math + .round(resultSet.getDouble(Constant.sum(d0s2))); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } - private void firstAggreWithSingleFilterTest() throws SQLException { + private void firstAggreWithSingleFilterTest() { String[] retArray = new String[]{ "0,90,1101,2.22,ddddd,true" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select first(s0),first(s1),first(s2),first(s3)," + "first(s4) from root.vehicle.d0 where s1 >= 0"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + - resultSet.getString(Constant.first(d0s0)) + "," - + resultSet.getString(Constant.first(d0s1)) + "," + - resultSet.getString(Constant.first(d0s2)) - + "," + resultSet.getString(Constant.first(d0s3)) + "," + - resultSet.getString(Constant.first(d0s4)); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + + resultSet.getString(Constant.first(d0s0)) + "," + + resultSet.getString(Constant.first(d0s1)) + "," + + resultSet.getString(Constant.first(d0s2)) + + "," + resultSet.getString(Constant.first(d0s3)) + "," + + resultSet.getString(Constant.first(d0s4)); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } - private void meanAggreWithSingleFilterTest() throws SQLException { + private void meanAggreWithSingleFilterTest() { String[] retArray = new String[]{ "0,75,212,28" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement();) { + boolean hasResultSet = statement.execute( "select mean(s0),mean(s1),mean(s2) from root.vehicle.d0 where s1 >= 0"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + Math.round(resultSet.getDouble(mean(d0s0))) - + "," + Math.round(resultSet.getDouble(mean(d0s1))) + "," + - Math.round(resultSet.getDouble(mean(d0s2))); - //System.out.println("!!!!!============ " + ans); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + "," + Math.round(resultSet.getDouble(mean(d0s0))) + + "," + Math.round(resultSet.getDouble(mean(d0s1))) + "," + + Math.round(resultSet.getDouble(mean(d0s2))); + //System.out.println("!!!!!============ " + ans); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -338,32 +310,27 @@ public class IoTDBAggregationLargeDataIT { "0,733,740,734" }; - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute( "select count(s0),count(s1),count(s2) from root.vehicle.d0 where s1 >= 0"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) - + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)); - //System.out.println("!!!!!============ " + ans); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) + + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)); + //System.out.println("!!!!!============ " + ans); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -373,11 +340,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute( "select min_time(s0),min_time(s1),min_time(s2),min_time(s3),min_time(s4)" + " from root.vehicle.d0 where s1 >= 0"); @@ -395,12 +361,10 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); retArray = new String[]{ "0,3999,3999,3999,3599,100" }; - statement = connection.createStatement(); hasResultSet = statement.execute("select max_time(s0),max_time(s1),max_time(s2)," + "max_time(s3),max_time(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); @@ -415,15 +379,10 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -433,10 +392,9 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select min_value(s0),min_value(s1),min_value(s2)," + "min_value(s3),min_value(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); @@ -457,14 +415,9 @@ public class IoTDBAggregationLargeDataIT { } Assert.assertEquals(1, cnt); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -474,11 +427,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select max_value(s0),max_value(s1),max_value(s2)," + "max_value(s3),max_value(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); @@ -499,14 +451,9 @@ public class IoTDBAggregationLargeDataIT { } Assert.assertEquals(1, cnt); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -516,11 +463,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select sum(s0),count(s0),mean(s0),mean(s1)," + "mean(s2) from root.vehicle.d0 " + "where s1 >= 0 or s2 < 10"); @@ -539,14 +485,9 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -556,11 +497,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select sum(s0),sum(s1),sum(s2) from root.vehicle.d0" + " where s1 >= 0 or s2 < 10"); Assert.assertTrue(hasResultSet); @@ -576,14 +516,9 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -593,11 +528,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select first(s0),first(s1),first(s2),first(s3)," + "first(s4) from root.vehicle.d0 where s1 >= 0 or s2 < 10"); Assert.assertTrue(hasResultSet); @@ -613,14 +547,9 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -630,11 +559,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select count(s0),count(s1),count(s2),count(s3)," + "count(s4) from root.vehicle.d0 where s1 >= 0 or s2 < 10"); Assert.assertTrue(hasResultSet); @@ -650,14 +578,9 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -667,11 +590,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select min_time(s0),min_time(s1),min_time(s2)," + "min_time(s3),min_time(s4) from root.vehicle.d0 where s1 >= 0"); Assert.assertTrue(hasResultSet); @@ -689,14 +611,9 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -706,11 +623,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select max_time(s0),max_time(s1),max_time(s2)," + "max_time(s3),max_time(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); @@ -726,14 +642,9 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -743,11 +654,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select min_value(s0),min_value(s1),min_value(s2)," + "min_value(s3),min_value(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); @@ -765,14 +675,9 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -782,11 +687,10 @@ public class IoTDBAggregationLargeDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select max_value(s0),max_value(s1),max_value(s2)," + "max_value(s3),max_value(s4) from root.vehicle.d0 where s1 < 50000 and s1 != 100"); Assert.assertTrue(hasResultSet); @@ -802,14 +706,9 @@ public class IoTDBAggregationLargeDataIT { cnt++; } Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -817,13 +716,12 @@ public class IoTDBAggregationLargeDataIT { public static void insertSQL() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; double d0s0sum = 0.0, d0s1sum = 0.0, d0s2sum = 0.0; int cnt = 0; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { + for (String sql : createSql) { statement.execute(sql); } @@ -876,19 +774,14 @@ public class IoTDBAggregationLargeDataIT { d0s2sum += time % 123; } -// statement.execute("merge"); + statement.execute("merge"); for (String sql : insertSql) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationSmallDataIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationSmallDataIT.java index b7c6e4ea1a941c8921dde8589a399bb76919ec13..de418488647f719c545502c82a13c880cf37bd9c 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationSmallDataIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationSmallDataIT.java @@ -145,34 +145,28 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "select count(s0),count(s1),count(s2),count(s3),count(s4) " + "from root.vehicle.d0 where time >= 3 and time <= 106"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) - + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)) - + "," + resultSet.getString(count(d0s3)) + "," + resultSet.getString(count(d0s4)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) + + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)) + + "," + resultSet.getString(count(d0s3)) + "," + resultSet.getString(count(d0s4)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -187,97 +181,87 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { //select count(d0.s0),count(d1.s1),count(d0.s3),count(d0.s4) from root.vehicle - Statement statement = connection.createStatement(); boolean hasResultSet = statement.execute( "select count(d0.s0),count(d1.s1),count(d0.s3),count(d0.s4) from root.vehicle"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) - + "," + resultSet.getString(count(d1s1)) + "," + resultSet.getString(count(d0s3)) - + "," + resultSet.getString(count(d0s4)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) + + "," + resultSet.getString(count(d1s1)) + "," + resultSet.getString(count(d0s3)) + + "," + resultSet.getString(count(d0s4)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); //select max_value(d0.s0),max_value(d1.s1),max_value(d0.s3) from root.vehicle - statement = connection.createStatement(); hasResultSet = statement.execute( "select max_value(d0.s0),max_value(d1.s1),max_value(d0.s3) from root.vehicle"); - resultSet = statement.getResultSet(); Assert.assertTrue(hasResultSet); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_value(d0s0)) - + "," + resultSet.getString(max_value(d1s1)) + "," - + resultSet.getString(max_value(d0s3)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_value(d0s0)) + + "," + resultSet.getString(max_value(d1s1)) + "," + + resultSet.getString(max_value(d0s3)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(2, cnt); } - Assert.assertEquals(2, cnt); - statement.close(); //select first(d0.s0),first(d1.s1),first(d0.s3) from root.vehicle - statement = connection.createStatement(); hasResultSet = statement.execute( "select first(d0.s0),first(d1.s1),first(d0.s3) from root.vehicle"); - resultSet = statement.getResultSet(); Assert.assertTrue(hasResultSet); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(first(d0s0)) - + "," + resultSet.getString(first(d1s1)) + "," + resultSet.getString(first(d0s3)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(first(d0s0)) + + "," + resultSet.getString(first(d1s1)) + "," + resultSet.getString(first(d0s3)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(3, cnt); } - Assert.assertEquals(3, cnt); - statement.close(); //select last(d0.s0),last(d1.s1),last(d0.s3) from root.vehicle - statement = connection.createStatement(); hasResultSet = statement.execute( "select last(d0.s0),last(d1.s1),last(d0.s3) from root.vehicle"); - resultSet = statement.getResultSet(); Assert.assertTrue(hasResultSet); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) - + "," + resultSet.getString(last(d1s1)) + "," + resultSet.getString(last(d0s3)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) + + "," + resultSet.getString(last(d1s1)) + "," + resultSet.getString(last(d0s3)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(4, cnt); } - Assert.assertEquals(4, cnt); - statement.close(); //select sum(d0.s0),sum(d1.s1),sum(d0.s3) from root.vehicle - statement = connection.createStatement(); hasResultSet = statement.execute("select sum(d0.s0),sum(d1.s1) from root.vehicle"); - resultSet = statement.getResultSet(); Assert.assertTrue(hasResultSet); - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) - + "," + resultSet.getString(sum(d1s1)); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) + + "," + resultSet.getString(sum(d1s1)); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(5, cnt); } - Assert.assertEquals(5, cnt); - statement.close(); - } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -287,33 +271,28 @@ public class IoTDBAggregationSmallDataIT { "0,22222,55555" }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute( "select last(s0),last(s1) from root.vehicle.d0 where s2 >= 3.33"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) + "," - + resultSet.getString(last(d0s1)); - //System.out.println("!!!!!============ " + ans); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(last(d0s0)) + "," + + resultSet.getString(last(d0s1)); + //System.out.println("!!!!!============ " + ans); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -323,33 +302,28 @@ public class IoTDBAggregationSmallDataIT { "0,99,180" }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement();) { + boolean hasResultSet = statement.execute( "select first(s0),first(s1) from root.vehicle.d0 where s2 >= 3.33"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(first(d0s0)) + "," - + resultSet.getString(first(d0s1)); - //System.out.println("!!!!!============ " + ans); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(first(d0s0)) + "," + + resultSet.getString(first(d0s1)); + //System.out.println("!!!!!============ " + ans); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -359,33 +333,27 @@ public class IoTDBAggregationSmallDataIT { "0,22321.0,55934.0,1029" }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "select sum(s0),sum(s1),sum(s2) from root.vehicle.d0 where s2 >= 3.33"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) - + "," + resultSet.getString(sum(d0s1)) + "," + Math - .round(resultSet.getDouble(sum(d0s2))); - //System.out.println("!!!!!============ " + ans); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(sum(d0s0)) + + "," + resultSet.getString(sum(d0s1)) + "," + Math + .round(resultSet.getDouble(sum(d0s2))); + //System.out.println("!!!!!============ " + ans); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -395,33 +363,27 @@ public class IoTDBAggregationSmallDataIT { "0,11160.5,18645,206" }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "select mean(s0),mean(s1),mean(s2) from root.vehicle.d0 where s2 >= 3.33"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(mean(d0s0)) - + "," + Math.round(resultSet.getDouble(mean(d0s1))) + "," - + Math.round(resultSet.getDouble(mean(d0s2))); - //System.out.println("!!!!!============ " + ans); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(mean(d0s0)) + + "," + Math.round(resultSet.getDouble(mean(d0s1))) + "," + + Math.round(resultSet.getDouble(mean(d0s2))); + //System.out.println("!!!!!============ " + ans); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -432,34 +394,28 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select count(s0),count(s1),count(s2),count(s3)," + "count(s4) from root.vehicle.d0 where s2 >= 3.33"); // System.out.println(hasResultSet + "..."); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) - + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)) - + "," + resultSet.getString(count(d0s3)) + "," + resultSet.getString(count(d0s4)); - // System.out.println("============ " + ans); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)) + + "," + resultSet.getString(count(d0s1)) + "," + resultSet.getString(count(d0s2)) + + "," + resultSet.getString(count(d0s3)) + "," + resultSet.getString(count(d0s4)); + // System.out.println("============ " + ans); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -470,35 +426,29 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select min_time(s0),min_time(s1),min_time(s2)" + ",min_time(s3),min_time(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(min_time(d0s0)) - + "," + resultSet.getString(min_time(d0s1)) + "," + resultSet.getString(min_time(d0s2)) - + "," + resultSet.getString(min_time(d0s3)) + "," + resultSet.getString(min_time(d0s4)); - // System.out.println("============ " + ans); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; - Assert.assertEquals(1, cnt); + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(min_time(d0s0)) + + "," + resultSet.getString(min_time(d0s1)) + "," + resultSet.getString(min_time(d0s2)) + + "," + resultSet.getString(min_time(d0s3)) + "," + resultSet.getString(min_time(d0s4)); + // System.out.println("============ " + ans); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + Assert.assertEquals(1, cnt); + } } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -509,35 +459,29 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select max_time(s0),max_time(s1),max_time(s2)" + ",max_time(s3),max_time(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_time(d0s0)) - + "," + resultSet.getString(max_time(d0s1)) + "," + resultSet.getString(max_time(d0s2)) - + "," + resultSet.getString(max_time(d0s3)) + "," + resultSet.getString(max_time(d0s4)); - // System.out.println("============ " + ans); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_time(d0s0)) + + "," + resultSet.getString(max_time(d0s1)) + "," + resultSet.getString(max_time(d0s2)) + + "," + resultSet.getString(max_time(d0s3)) + "," + resultSet.getString(max_time(d0s4)); + // System.out.println("============ " + ans); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -548,38 +492,32 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select min_value(s0),min_value(s1),min_value(s2)" + ",min_value(s3),min_value(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(min_value(d0s0)) - + "," + resultSet.getString(min_value(d0s1)) + - "," + resultSet.getString(min_value(d0s2)) - + "," + resultSet.getString(min_value(d0s3)) + "," - + resultSet.getString(min_value(d0s4)); - // System.out.println("============ " + ans); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(min_value(d0s0)) + + "," + resultSet.getString(min_value(d0s1)) + + "," + resultSet.getString(min_value(d0s2)) + + "," + resultSet.getString(min_value(d0s3)) + "," + + resultSet.getString(min_value(d0s4)); + // System.out.println("============ " + ans); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -590,37 +528,31 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select max_value(s0),max_value(s1),max_value(s2)," + "max_value(s3),max_value(s4) from root.vehicle.d0 " + "where s1 < 50000 and s1 != 100"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_value(d0s0)) - + "," + resultSet.getString(max_value(d0s1)) + "," + resultSet - .getString(max_value(d0s2)) - + "," + resultSet.getString(max_value(d0s3)) + "," + resultSet - .getString(max_value(d0s4)); - //System.out.println("============ " + ans); - //Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(max_value(d0s0)) + + "," + resultSet.getString(max_value(d0s1)) + "," + resultSet + .getString(max_value(d0s2)) + + "," + resultSet.getString(max_value(d0s3)) + "," + resultSet + .getString(max_value(d0s4)); + //System.out.println("============ " + ans); + //Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -631,32 +563,26 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute( "select count(s0) from root.vehicle.d0 where s2 >= 3.33"); // System.out.println(hasResultSet + "..."); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)); - //System.out.println("============ " + ans); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(count(d0s0)); + //System.out.println("============ " + ans); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -684,72 +610,59 @@ public class IoTDBAggregationSmallDataIT { }; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager. - getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager. + getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select * from root"); // System.out.println(hasResultSet + "..."); if (hasResultSet) { - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," - + resultSet.getString(d0s1) + "," + resultSet.getString(d0s2) + "," + - resultSet.getString(d0s3) + "," + resultSet.getString(d1s0); - // System.out.println(ans); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," + + resultSet.getString(d0s1) + "," + resultSet.getString(d0s2) + "," + + resultSet.getString(d0s3) + "," + resultSet.getString(d1s0); + // System.out.println(ans); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(17, cnt); } - Assert.assertEquals(17, cnt); } - statement.close(); retArray = new String[]{ "100,true" }; - statement = connection.createStatement(); hasResultSet = statement.execute("select s4 from root.vehicle.d0"); if (hasResultSet) { - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s4); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s4); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } - public static void insertSQL() throws ClassNotFoundException, SQLException { + public static void insertSQL() throws ClassNotFoundException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager.getConnection - (Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager.getConnection + (Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement();) { + for (String sql : sqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java index e993a9daa165cf4182415f1e2ae8eefb082b4f13..b18049dfd30a6db98919494e3c3beaaf6eb4e22f 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBCompleteIT.java @@ -370,13 +370,13 @@ public class IoTDBCompleteIT { private void executeSQL(String[] sqls) throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { String result = ""; Long now_start = 0L; boolean cmp = false; - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + for (String sql : sqls) { if (cmp) { Assert.assertEquals(sql, result); @@ -389,7 +389,7 @@ public class IoTDBCompleteIT { if (sql.contains("NOW()") && now_start == 0L) { now_start = System.currentTimeMillis(); } - Statement statement = connection.createStatement(); + statement.execute(sql); if (sql.split(" ")[0].equals("SELECT")) { ResultSet resultSet = statement.getResultSet(); @@ -418,15 +418,10 @@ public class IoTDBCompleteIT { } cmp = true; } - statement.close(); } } } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDaemonIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDaemonIT.java index 67e53afb7b817635f062c862907c6236a66c0ffe..ea308e34b57f861c3c0d515e6902cfdba04c73a2 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDaemonIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDaemonIT.java @@ -127,21 +127,15 @@ public class IoTDBDaemonIT { private static void insertData() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -157,48 +151,42 @@ public class IoTDBDaemonIT { "946684800000,null,100,null,good,null"}; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select * from root"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," - + resultSet.getString(d0s1) + "," + resultSet.getString(d0s2) + "," + resultSet - .getString(d0s3) - + "," + resultSet.getString(d1s0); - Assert.assertEquals(retArray[cnt], ans); - cnt++; + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," + + resultSet.getString(d0s1) + "," + resultSet.getString(d0s2) + "," + resultSet + .getString(d0s3) + + "," + resultSet.getString(d1s0); + Assert.assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(17, cnt); } - Assert.assertEquals(17, cnt); - statement.close(); retArray = new String[]{"100,true"}; - statement = connection.createStatement(); hasResultSet = statement.execute("select s4 from root.vehicle.d0"); Assert.assertTrue(hasResultSet); - resultSet = statement.getResultSet(); - cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s4); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s4); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -208,31 +196,25 @@ public class IoTDBDaemonIT { "1000,1000.11"}; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select s2 from root.vehicle.*"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s2); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; - } - Assert.assertEquals(6, cnt); - statement.close(); + try (ResultSet resultSet = statement.getResultSet();) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s2); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(6, cnt); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -243,34 +225,27 @@ public class IoTDBDaemonIT { "102,80,180", "103,99,199", "104,90,190", "105,99,199"}; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { boolean hasResultSet = statement .execute( "select s0,s1 from root.vehicle.d0 where time < 106 and (s0 >= 60 or s1 <= 200)"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," - + resultSet.getString(d0s1); - assertEquals(retArray[cnt], ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," + + resultSet.getString(d0s1); + assertEquals(retArray[cnt], ans); + cnt++; + } + Assert.assertEquals(9, cnt); } - Assert.assertEquals(9, cnt); - - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -279,32 +254,27 @@ public class IoTDBDaemonIT { String[] retArray = new String[]{"1000,22222,55555,888"}; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { // TODO select s0,s1 from root.vehicle.d0 where time > 106 and root.vehicle.d1.s0 > 100; boolean hasResultSet = statement.execute( "select s0,s1 from root.vehicle.d0,root.vehicle.d1 where time > 106 and root.vehicle.d0.s0 > 100"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," - + resultSet.getString(d0s1) + "," + resultSet.getString(d1s0); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," + + resultSet.getString(d0s1) + "," + resultSet.getString(d1s0); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -313,32 +283,27 @@ public class IoTDBDaemonIT { String[] retArray = new String[]{"1000,22222,55555"}; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { boolean hasResultSet = statement .execute( "select s0,s1 from root.vehicle.d0 where time > 106 and root.vehicle.d1.s0 > 100"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," - + resultSet.getString(d0s1); - Assert.assertEquals(ans, retArray[cnt]); - cnt++; + + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s0) + "," + + resultSet.getString(d0s1); + Assert.assertEquals(ans, retArray[cnt]); + cnt++; + } + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -347,31 +312,26 @@ public class IoTDBDaemonIT { String[] retArray = new String[]{"102,180", "104,190", "946684800000,100"}; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { boolean hasTextMaxResultSet = statement .execute("select s1 from root.vehicle.d0 where s1 < 199"); Assert.assertTrue(hasTextMaxResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s1); - Assert.assertEquals(retArray[cnt++], ans); + + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = resultSet.getString(TIMESTAMP_STR) + "," + resultSet.getString(d0s1); + Assert.assertEquals(retArray[cnt++], ans); + } + Assert.assertEquals(3, cnt); } - Assert.assertEquals(3, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDeletionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDeletionIT.java index eef24e3b5dc18d8ca19e2adb992561ed0ac38b17..61236798e3e865b202383e96c3b26df16015c666 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDeletionIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBDeletionIT.java @@ -75,21 +75,20 @@ public class IoTDBDeletionIT { @Test public void test() throws SQLException { prepareData(); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { + statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time <= 300"); statement.execute("DELETE FROM root.vehicle.d0.s1,root.vehicle.d0.s2,root.vehicle.d0.s3" - + " WHERE time <= 350"); + + " WHERE time <= 350"); statement.execute("DELETE FROM root.vehicle.d0 WHERE time <= 150"); ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle.d0"); int cnt = 0; while (set.next()) { - cnt ++; + cnt++; } assertEquals(250, cnt); set.close(); @@ -97,7 +96,7 @@ public class IoTDBDeletionIT { set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0"); cnt = 0; while (set.next()) { - cnt ++; + cnt++; } assertEquals(100, cnt); set.close(); @@ -105,174 +104,147 @@ public class IoTDBDeletionIT { set = statement.executeQuery("SELECT s1,s2,s3 FROM root.vehicle.d0"); cnt = 0; while (set.next()) { - cnt ++; + cnt++; } assertEquals(50, cnt); set.close(); - statement.close(); - } finally { - if (connection != null) { - connection.close(); - } } cleanData(); } - @Ignore @Test - public void testMerge() throws SQLException, InterruptedException { + public void testMerge() throws SQLException { prepareMerge(); - Connection connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); -// statement.execute("merge"); - statement.execute("DELETE FROM root.vehicle.d0 WHERE time <= 15000"); - - // before merge completes - ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle.d0"); - int cnt = 0; - while (set.next()) { - cnt ++; - } - assertEquals(5000, cnt); - set.close(); - - // after merge completes - set = statement.executeQuery("SELECT * FROM root.vehicle.d0"); - cnt = 0; - while (set.next()) { - cnt ++; + + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { + statement.execute("merge"); + statement.execute("DELETE FROM root.vehicle.d0 WHERE time <= 15000"); + + // before merge completes + ResultSet set = statement.executeQuery("SELECT * FROM root.vehicle.d0"); + int cnt = 0; + while (set.next()) { + cnt ++; + } + assertEquals(5000, cnt); + set.close(); + + // after merge completes + set = statement.executeQuery("SELECT * FROM root.vehicle.d0"); + cnt = 0; + while (set.next()) { + cnt ++; + } + assertEquals(5000, cnt); + set.close(); + cleanData(); } - assertEquals(5000, cnt); - set.close(); - cleanData(); } @Test public void testDelAfterFlush() throws SQLException { - Connection connection = DriverManager + try (Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); - - statement.execute("SET STORAGE GROUP TO root.ln.wf01.wt01"); - statement.execute("CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN," - + " ENCODING=PLAIN"); - statement.execute("INSERT INTO root.ln.wf01.wt01(timestamp,status) " - + "values(1509465600000,true)"); - statement.execute("INSERT INTO root.ln.wf01.wt01(timestamp,status) VALUES(NOW(), false)"); - - statement.execute("delete from root.ln.wf01.wt01.status where time < NOW()"); - statement.execute("flush"); - statement.execute("delete from root.ln.wf01.wt01.status where time < NOW()"); - ResultSet resultSet = statement.executeQuery("select status from root.ln.wf01.wt01"); - assertFalse(resultSet.next()); - - statement.close(); - connection.close(); + Statement statement = connection.createStatement()) { + statement.execute("SET STORAGE GROUP TO root.ln.wf01.wt01"); + statement.execute("CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN," + + " ENCODING=PLAIN"); + statement.execute("INSERT INTO root.ln.wf01.wt01(timestamp,status) " + + "values(1509465600000,true)"); + statement.execute("INSERT INTO root.ln.wf01.wt01(timestamp,status) VALUES(NOW(), false)"); + + statement.execute("delete from root.ln.wf01.wt01.status where time < NOW()"); + statement.execute("flush"); + statement.execute("delete from root.ln.wf01.wt01.status where time < NOW()"); + + try (ResultSet resultSet = statement.executeQuery("select status from root.ln.wf01.wt01")) { + assertFalse(resultSet.next()); + } + } } - private static void prepareSeries() throws SQLException { - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + private static void prepareSeries() { + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { + for (String sql : creationSqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } private void prepareData() throws SQLException { - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { + // prepare BufferWrite file for (int i = 201; i <= 300; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", + statement.execute( + String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", i % 2 == 0)); } -// statement.execute("merge"); + statement.execute("merge"); // prepare Unseq-File for (int i = 1; i <= 100; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", + statement.execute( + String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", i % 2 == 0)); } -// statement.execute("merge"); + statement.execute("merge"); // prepare BufferWrite cache for (int i = 301; i <= 400; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", + statement.execute( + String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", i % 2 == 0)); } // prepare Overflow cache for (int i = 101; i <= 200; i++) { - statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", + statement.execute( + String.format(Locale.ENGLISH, insertTemplate, i, i, i, (double) i, "\'" + i + "\'", i % 2 == 0)); } - statement.close(); - } finally { - if (connection != null) { - connection.close(); - } } } private void cleanData() throws SQLException { - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { statement.execute(deleteAllTemplate); - - statement.close(); - } finally { - if (connection != null) { - connection.close(); - } } } public void prepareMerge() throws SQLException { - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { + // prepare BufferWrite data for (int i = 10001; i <= 20000; i++) { statement.execute(String.format(insertTemplate, i, i, i, (double) i, "\'" + i + "\'", - i % 2 == 0)); + i % 2 == 0)); } // prepare Overflow data for (int i = 1; i <= 10000; i++) { statement.execute(String.format(insertTemplate, i, i, i, (double) i, "\'" + i + "\'", - i % 2 == 0)); + i % 2 == 0)); } - statement.close(); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBEngineTimeGeneratorIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBEngineTimeGeneratorIT.java index 11d5ffb30b1d42c49340d3b7b7c9beb56ebd7858..d4dc501bdcc0aeb6d177e5f43807ed7a6e254ed6 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBEngineTimeGeneratorIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBEngineTimeGeneratorIT.java @@ -82,7 +82,6 @@ public class IoTDBEngineTimeGeneratorIT { daemon.active(); EnvironmentUtils.envSetUp(); - Thread.sleep(5000); insertData(); } @@ -101,11 +100,9 @@ public class IoTDBEngineTimeGeneratorIT { private static void insertData() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { // create storage group and measurement for (String sql : Constant.create_sql) { @@ -160,14 +157,9 @@ public class IoTDBEngineTimeGeneratorIT { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBFloatPrecisionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBFloatPrecisionIT.java index 432bfcda79604151a66c275b11df740f4ece7bbe..117621d6a18d9399981d14701a926dbe6c8b70b7 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBFloatPrecisionIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBFloatPrecisionIT.java @@ -93,89 +93,77 @@ public class IoTDBFloatPrecisionIT { private static void insertData() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement();) { + for (String sql : sqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } @Test public void selectAllSQLTest() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select * from root"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - assertEquals(TIMESTAMP + "", resultSet.getString(TIMESTAMP_STR)); - for (int i = 0; i < 10; i++) { - Assert.assertEquals(MathUtils.roundWithGivenPrecision(Float.parseFloat(VALUE), i), - resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0", "s" + i + "rle")), - DELTA_FLOAT); - Assert.assertEquals(MathUtils.roundWithGivenPrecision(Float.parseFloat(VALUE), i), - resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0", "s" + i + "2f")), - DELTA_FLOAT); - Assert.assertEquals(MathUtils.roundWithGivenPrecision(Double.parseDouble(VALUE), i), - resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0", "s" + i + "rle")), - DELTA_DOUBLE); - Assert.assertEquals(MathUtils.roundWithGivenPrecision(Double.parseDouble(VALUE), i), - resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0", "s" + i + "2f")), - DELTA_DOUBLE); + int cnt; + try (ResultSet resultSet = statement.getResultSet()) { + cnt = 0; + while (resultSet.next()) { + assertEquals(TIMESTAMP + "", resultSet.getString(TIMESTAMP_STR)); + for (int i = 0; i < 10; i++) { + Assert.assertEquals(MathUtils.roundWithGivenPrecision(Float.parseFloat(VALUE), i), + resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0", "s" + i + "rle")), + DELTA_FLOAT); + Assert.assertEquals(MathUtils.roundWithGivenPrecision(Float.parseFloat(VALUE), i), + resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0", "s" + i + "2f")), + DELTA_FLOAT); + Assert.assertEquals(MathUtils.roundWithGivenPrecision(Double.parseDouble(VALUE), i), + resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0", "s" + i + "rle")), + DELTA_DOUBLE); + Assert.assertEquals(MathUtils.roundWithGivenPrecision(Double.parseDouble(VALUE), i), + resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0", "s" + i + "2f")), + DELTA_DOUBLE); + } + cnt++; } - cnt++; + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); - statement = connection.createStatement(); statement.execute("flush"); - resultSet = statement.executeQuery("select * from root"); - cnt = 0; - while (resultSet.next()) { - assertEquals(TIMESTAMP + "", resultSet.getString(TIMESTAMP_STR)); - for (int i = 0; i < 10; i++) { - BigDecimal b = new BigDecimal(VALUE); - Assert.assertEquals(b.setScale(i, RoundingMode.HALF_UP).floatValue(), - resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0", "s" + i + "rle")), - DELTA_FLOAT); - Assert.assertEquals(b.setScale(i, RoundingMode.HALF_UP).floatValue(), - resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0", "s" + i + "2f")), - DELTA_FLOAT); - Assert.assertEquals(b.setScale(i, RoundingMode.HALF_UP).doubleValue(), - resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0", "s" + i + "rle")), - DELTA_DOUBLE); - Assert.assertEquals(b.setScale(i, RoundingMode.HALF_UP).doubleValue(), - resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0", "s" + i + "2f")), - DELTA_DOUBLE); + try (ResultSet resultSet =statement.executeQuery("select * from root")) { + cnt = 0; + while (resultSet.next()) { + assertEquals(TIMESTAMP + "", resultSet.getString(TIMESTAMP_STR)); + for (int i = 0; i < 10; i++) { + BigDecimal b = new BigDecimal(VALUE); + Assert.assertEquals(b.setScale(i, RoundingMode.HALF_UP).floatValue(), + resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0", "s" + i + "rle")), + DELTA_FLOAT); + Assert.assertEquals(b.setScale(i, RoundingMode.HALF_UP).floatValue(), + resultSet.getFloat(String.format("root.vehicle.%s.%s", "f0", "s" + i + "2f")), + DELTA_FLOAT); + Assert.assertEquals(b.setScale(i, RoundingMode.HALF_UP).doubleValue(), + resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0", "s" + i + "rle")), + DELTA_DOUBLE); + Assert.assertEquals(b.setScale(i, RoundingMode.HALF_UP).doubleValue(), + resultSet.getDouble(String.format("root.vehicle.%s.%s", "d0", "s" + i + "2f")), + DELTA_DOUBLE); + } + cnt++; } - cnt++; + Assert.assertEquals(1, cnt); } - Assert.assertEquals(1, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBFlushQueryMergeTest.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBFlushQueryMergeTest.java index 427714685d7addc3ffb83ea186a8b4b7241872fe..d845aca8307b8d5dbed125cd84640eb6d8b46189 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBFlushQueryMergeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBFlushQueryMergeTest.java @@ -66,23 +66,17 @@ public class IoTDBFlushQueryMergeTest { EnvironmentUtils.cleanEnv(); } - private static void insertData() throws ClassNotFoundException, SQLException { + private static void insertData() throws ClassNotFoundException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -90,32 +84,22 @@ public class IoTDBFlushQueryMergeTest { public void selectAllSQLTest() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select * from root"); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - cnt++; + try (ResultSet resultSet = statement.getResultSet()){ + int cnt = 0; + while (resultSet.next()) { + cnt++; + } } - statement.close(); - - statement = connection.createStatement(); -// statement.execute("merge"); - statement.close(); - + statement.execute("merge"); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLargeDataIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLargeDataIT.java index 093c0a9d4b7c59ef1e1014bff69c7c501cea6c36..5bca581d6f92ff1933b7631cea95eba060d84813 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLargeDataIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLargeDataIT.java @@ -71,7 +71,6 @@ public class IoTDBLargeDataIT { daemon.active(); EnvironmentUtils.envSetUp(); - Thread.sleep(5000); insertData(); } @@ -91,11 +90,9 @@ public class IoTDBLargeDataIT { private static void insertData() throws ClassNotFoundException, SQLException, InterruptedException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { for (String sql : Constant.create_sql) { statement.execute(sql); @@ -138,9 +135,7 @@ public class IoTDBLargeDataIT { } statement.execute("flush"); - // statement.execute("merge"); - - Thread.sleep(5000); + statement.execute("merge"); // buffwrite data, unsealed file for (int time = 100000; time < 101000; time++) { @@ -209,14 +204,9 @@ public class IoTDBLargeDataIT { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -226,36 +216,32 @@ public class IoTDBLargeDataIT { String selectSql = "select * from root.vehicle"; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute(selectSql); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0) - + "," + resultSet.getString(Constant.d0s1) + "," + resultSet - .getString(Constant.d0s2) + "," - + resultSet.getString(Constant.d0s3) + "," + resultSet.getString(Constant.d0s4) - + "," - + resultSet.getString(Constant.d0s5); - cnt++; - } - assertEquals(23400, cnt); - statement.close(); + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0) + + "," + resultSet.getString(Constant.d0s1) + "," + resultSet + .getString(Constant.d0s2) + "," + + resultSet.getString(Constant.d0s3) + "," + resultSet.getString(Constant.d0s4) + + "," + + resultSet.getString(Constant.d0s5); + cnt++; + } + + assertEquals(23400, cnt); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -266,31 +252,26 @@ public class IoTDBLargeDataIT { String selectSql = "select s0 from root.vehicle.d0 where s0 >= 20"; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); +Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute(selectSql); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0); - // System.out.println("===" + ans); - cnt++; + + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0); + // System.out.println("===" + ans); + cnt++; + } + assertEquals(16440, cnt); } - assertEquals(16440, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -299,37 +280,29 @@ public class IoTDBLargeDataIT { public void seriesGlobalTimeFilterTest() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); + boolean hasResultSet; - Statement statement; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement();) { hasResultSet = statement.execute("select s0 from root.vehicle.d0 where time > 22987"); assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0); - // System.out.println(ans); - cnt++; - } - assertEquals(3012, cnt); - statement.close(); + int cnt = 0; + try (ResultSet resultSet = statement.getResultSet();) { + while (resultSet.next()) { + String ans = + resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0); + // System.out.println(ans); + cnt++; + } + assertEquals(3012, cnt); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -337,39 +310,31 @@ public class IoTDBLargeDataIT { @Test public void crossSeriesReadUpdateTest() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - boolean hasResultSet; - Statement statement; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - statement = connection.createStatement(); + boolean hasResultSet; + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { hasResultSet = statement.execute("select s1 from root.vehicle.d0 where s0 < 111"); assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - long time = Long.valueOf(resultSet.getString(Constant.TIMESTAMP_STR)); - String value = resultSet.getString(Constant.d0s1); - if (time > 200900) { - assertEquals("7777", value); + + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + long time = Long.valueOf(resultSet.getString(Constant.TIMESTAMP_STR)); + String value = resultSet.getString(Constant.d0s1); + if (time > 200900) { + assertEquals("7777", value); + } + // String ans = resultSet.getString(d0s1); + cnt++; } - // String ans = resultSet.getString(d0s1); - cnt++; + assertEquals(22800, cnt); } - assertEquals(22800, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLimitSlimitIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLimitSlimitIT.java index 4435af6aca7bad5ae8bcfb029c0bff277d67284c..c8cee20575b78e424e2114f8c238647e1e2f7162 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLimitSlimitIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLimitSlimitIT.java @@ -101,22 +101,16 @@ public class IoTDBLimitSlimitIT { private static void insertData() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + for (String sql : insertSqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -158,13 +152,14 @@ public class IoTDBLimitSlimitIT { private void executeSQL(String[] sqls) throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { + + try ( Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { String result = ""; Long now_start = 0L; boolean cmp = false; - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + for (String sql : sqls) { // System.out.println("----" + sql); if (cmp) { @@ -178,44 +173,40 @@ public class IoTDBLimitSlimitIT { if (sql.contains("NOW()") && now_start == 0L) { now_start = System.currentTimeMillis(); } - Statement statement = connection.createStatement(); + statement.execute(sql); if (sql.split(" ")[0].equals("SELECT") | sql.split(" ")[0].equals("select")) { - ResultSet resultSet = statement.getResultSet(); - ResultSetMetaData metaData = resultSet.getMetaData(); - int count = metaData.getColumnCount(); - String[] column = new String[count]; - for (int i = 0; i < count; i++) { - column[i] = metaData.getColumnName(i + 1); - } - result = ""; - while (resultSet.next()) { - for (int i = 1; i <= count; i++) { - if (now_start > 0L && column[i - 1] == Constant.TIMESTAMP_STR) { - String timestr = resultSet.getString(i); - Long tn = Long.valueOf(timestr); - Long now = System.currentTimeMillis(); - if (tn >= now_start && tn <= now) { - timestr = "NOW()"; + try (ResultSet resultSet = statement.getResultSet()) { + ResultSetMetaData metaData = resultSet.getMetaData(); + int count = metaData.getColumnCount(); + String[] column = new String[count]; + for (int i = 0; i < count; i++) { + column[i] = metaData.getColumnName(i + 1); + } + result = ""; + while (resultSet.next()) { + for (int i = 1; i <= count; i++) { + if (now_start > 0L && column[i - 1] == Constant.TIMESTAMP_STR) { + String timestr = resultSet.getString(i); + Long tn = Long.valueOf(timestr); + Long now = System.currentTimeMillis(); + if (tn >= now_start && tn <= now) { + timestr = "NOW()"; + } + result += timestr + ','; + } else { + result += resultSet.getString(i) + ','; } - result += timestr + ','; - } else { - result += resultSet.getString(i) + ','; } + result += '\n'; } - result += '\n'; + cmp = true; } - cmp = true; } - statement.close(); } } } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - connection.close(); - } } } } \ No newline at end of file diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMergeTest.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMergeTest.java new file mode 100644 index 0000000000000000000000000000000000000000..422cb7730da7781921b352b898c6e26f226571f3 --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMergeTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.integration; + +import static org.junit.Assert.assertEquals; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import org.apache.iotdb.db.service.IoTDB; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.jdbc.Config; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class IoTDBMergeTest { + + private static IoTDB daemon; + + @Before + public void setUp() throws Exception { + EnvironmentUtils.closeStatMonitor(); + + daemon = IoTDB.getInstance(); + daemon.active(); + EnvironmentUtils.envSetUp(); + Class.forName(Config.JDBC_DRIVER_NAME); + } + + @After + public void tearDown() throws Exception { + daemon.stop(); + EnvironmentUtils.cleanEnv(); + } + + @Test + public void test() throws SQLException { + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + statement.execute("SET STORAGE GROUP TO root.mergeTest"); + for (int i = 1; i <= 3; i++) { + statement.execute("CREATE TIMESERIES root.mergeTest.s" + i + " WITH DATATYPE=INT64," + + "ENCODING=PLAIN"); + } + + for (int i = 0; i < 10; i++) { + for (int j = i * 10 + 1; j <= (i+1) * 10; j++) { + statement.execute(String.format("INSERT INTO root.mergeTest(timestamp,s1,s2,s3) VALUES (%d,%d," + + "%d,%d)", j, j+1, j+2, j+3)); + } + statement.execute("FLUSH"); + for (int j = i * 10 + 1; j <= (i+1) * 10; j++) { + statement.execute(String.format("INSERT INTO root.mergeTest(timestamp,s1,s2,s3) VALUES (%d,%d," + + "%d,%d)", j, j+10, j+20, j+30)); + } + statement.execute("FLUSH"); + statement.execute("MERGE"); + + ResultSet resultSet = statement.executeQuery("SELECT * FROM root.mergeTest"); + int cnt = 0; + while (resultSet.next()) { + long time = resultSet.getLong("Time"); + long s1 = resultSet.getLong("root.mergeTest.s1"); + long s2 = resultSet.getLong("root.mergeTest.s2"); + long s3 = resultSet.getLong("root.mergeTest.s3"); + assertEquals(time + 10, s1); + assertEquals(time + 20, s2); + assertEquals(time + 30, s3); + cnt++; + } + assertEquals((i + 1) * 10, cnt); + } + } + } + +} diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java index ae24bd1de6fe0743a1e5c63d6e5ea4edc82cbb26..acdf8af85b8407869d65158200b7e085e5fcc214 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java @@ -49,11 +49,9 @@ public class IoTDBMetadataFetchIT { private static void insertSQL() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { String[] insertSqls = new String[]{"SET STORAGE GROUP TO root.ln.wf01.wt01", "CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE = BOOLEAN, ENCODING = PLAIN", @@ -63,14 +61,9 @@ public class IoTDBMetadataFetchIT { for (String sql : insertSqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -94,11 +87,10 @@ public class IoTDBMetadataFetchIT { @Test public void showTimeseriesTest1() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + String[] sqls = new String[]{"show timeseries root.ln.wf01.wt01.status", // full seriesPath "show timeseries root.ln", // prefix seriesPath "show timeseries root.ln.*.wt01", // seriesPath with stars @@ -127,13 +119,14 @@ public class IoTDBMetadataFetchIT { try { boolean hasResultSet = statement.execute(sql); if (hasResultSet) { - ResultSet resultSet = statement.getResultSet(); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - while (resultSet.next()) { - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - builder.append(resultSet.getString(i)).append(","); + try (ResultSet resultSet = statement.getResultSet()) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(resultSet.getString(i)).append(","); + } + builder.append("\n"); } - builder.append("\n"); } } Assert.assertEquals(builder.toString(), standard); @@ -142,40 +135,26 @@ public class IoTDBMetadataFetchIT { fail(e.getMessage()); } } - statement.close(); - } finally { - connection.close(); } } - @Test + @Test(expected = SQLException.class) public void showTimeseriesTest2() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - Statement statement = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); Statement statement = connection.createStatement()) { String sql = "show timeseries"; // not supported in jdbc, thus expecting SQLException statement.execute(sql); - } catch (SQLException e) { - } catch (Exception e) { - fail(e.getMessage()); - } finally { - statement.close(); - connection.close(); } } @Test public void showStorageGroupTest() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { String[] sqls = new String[]{"show storage group"}; String[] standards = new String[]{"root.ln.wf01.wt01,\n"}; for (int n = 0; n < sqls.length; n++) { @@ -185,13 +164,14 @@ public class IoTDBMetadataFetchIT { try { boolean hasResultSet = statement.execute(sql); if (hasResultSet) { - ResultSet resultSet = statement.getResultSet(); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - while (resultSet.next()) { - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - builder.append(resultSet.getString(i)).append(","); + try (ResultSet resultSet = statement.getResultSet()) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(resultSet.getString(i)).append(","); + } + builder.append("\n"); } - builder.append("\n"); } } Assert.assertEquals(builder.toString(), standard); @@ -200,11 +180,6 @@ public class IoTDBMetadataFetchIT { fail(e.getMessage()); } } - statement.close(); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -241,21 +216,22 @@ public class IoTDBMetadataFetchIT { String standard = "Column,\n" + "root.ln.wf01.wt01.status,\n" + "root.ln.wf01.wt01.temperature,\n"; - ResultSet resultSet = databaseMetaData.getColumns(Constant.CATALOG_COLUMN, "root", null, null); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - int colCount = resultSetMetaData.getColumnCount(); - StringBuilder resultStr = new StringBuilder(); - for (int i = 1; i < colCount + 1; i++) { - resultStr.append(resultSetMetaData.getColumnName(i)).append(","); - } - resultStr.append("\n"); - while (resultSet.next()) { - for (int i = 1; i <= colCount; i++) { - resultStr.append(resultSet.getString(i)).append(","); + try (ResultSet resultSet = databaseMetaData.getColumns(Constant.CATALOG_COLUMN, "root", null, null);) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + int colCount = resultSetMetaData.getColumnCount(); + StringBuilder resultStr = new StringBuilder(); + for (int i = 1; i < colCount + 1; i++) { + resultStr.append(resultSetMetaData.getColumnName(i)).append(","); } resultStr.append("\n"); + while (resultSet.next()) { + for (int i = 1; i <= colCount; i++) { + resultStr.append(resultSet.getString(i)).append(","); + } + resultStr.append("\n"); + } + Assert.assertEquals(resultStr.toString(), standard); } - Assert.assertEquals(resultStr.toString(), standard); } /** @@ -264,21 +240,24 @@ public class IoTDBMetadataFetchIT { private void device() throws SQLException { String standard = "Column,\n" + "root.ln.wf01.wt01,\n"; - ResultSet resultSet = databaseMetaData.getColumns(Constant.CATALOG_DEVICE, "ln", null, null); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - int colCount = resultSetMetaData.getColumnCount(); - StringBuilder resultStr = new StringBuilder(); - for (int i = 1; i < colCount + 1; i++) { - resultStr.append(resultSetMetaData.getColumnName(i)).append(","); - } - resultStr.append("\n"); - while (resultSet.next()) { - for (int i = 1; i <= colCount; i++) { - resultStr.append(resultSet.getString(i)).append(","); + + try (ResultSet resultSet = databaseMetaData.getColumns(Constant.CATALOG_DEVICE, "ln", null, + null)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + int colCount = resultSetMetaData.getColumnCount(); + StringBuilder resultStr = new StringBuilder(); + for (int i = 1; i < colCount + 1; i++) { + resultStr.append(resultSetMetaData.getColumnName(i)).append(","); } resultStr.append("\n"); + while (resultSet.next()) { + for (int i = 1; i <= colCount; i++) { + resultStr.append(resultSet.getString(i)).append(","); + } + resultStr.append("\n"); + } + Assert.assertEquals(resultStr.toString(), standard); } - Assert.assertEquals(resultStr.toString(), standard); } /** @@ -289,22 +268,23 @@ public class IoTDBMetadataFetchIT { + "root.ln.wf01.wt01.status,root.ln.wf01.wt01,BOOLEAN,PLAIN,\n" + "root.ln.wf01.wt01.temperature,root.ln.wf01.wt01,FLOAT,RLE,\n"; - ResultSet resultSet = databaseMetaData - .getColumns(Constant.CATALOG_TIMESERIES, "root", null, null); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - int colCount = resultSetMetaData.getColumnCount(); - StringBuilder resultStr = new StringBuilder(); - for (int i = 1; i < colCount + 1; i++) { - resultStr.append(resultSetMetaData.getColumnName(i)).append(","); - } - resultStr.append("\n"); - while (resultSet.next()) { - for (int i = 1; i <= colCount; i++) { - resultStr.append(resultSet.getString(i)).append(","); + try (ResultSet resultSet = databaseMetaData + .getColumns(Constant.CATALOG_TIMESERIES, "root", null, null);) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + int colCount = resultSetMetaData.getColumnCount(); + StringBuilder resultStr = new StringBuilder(); + for (int i = 1; i < colCount + 1; i++) { + resultStr.append(resultSetMetaData.getColumnName(i)).append(","); } resultStr.append("\n"); + while (resultSet.next()) { + for (int i = 1; i <= colCount; i++) { + resultStr.append(resultSet.getString(i)).append(","); + } + resultStr.append("\n"); + } + Assert.assertEquals(resultStr.toString(), standard); } - Assert.assertEquals(resultStr.toString(), standard); } /** @@ -313,18 +293,19 @@ public class IoTDBMetadataFetchIT { private void showTimeseriesPath2() throws SQLException { String standard = "DataType,\n" + "BOOLEAN,\n"; - ResultSet resultSet = databaseMetaData + try (ResultSet resultSet = databaseMetaData .getColumns(Constant.CATALOG_TIMESERIES, "root.ln.wf01.wt01.status", null, - null); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - StringBuilder resultStr = new StringBuilder(); - resultStr.append(resultSetMetaData.getColumnName(3)).append(",\n"); - while (resultSet.next()) { - resultStr.append(resultSet.getString(IoTDBMetadataResultSet.GET_STRING_TIMESERIES_DATATYPE)) - .append(","); - resultStr.append("\n"); + null);) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + StringBuilder resultStr = new StringBuilder(); + resultStr.append(resultSetMetaData.getColumnName(3)).append(",\n"); + while (resultSet.next()) { + resultStr.append(resultSet.getString(IoTDBMetadataResultSet.GET_STRING_TIMESERIES_DATATYPE)) + .append(","); + resultStr.append("\n"); + } + Assert.assertEquals(resultStr.toString(), standard); } - Assert.assertEquals(resultStr.toString(), standard); } /** @@ -333,22 +314,23 @@ public class IoTDBMetadataFetchIT { private void showStorageGroup() throws SQLException { String standard = "Storage Group,\n" + "root.ln.wf01.wt01,\n"; - ResultSet resultSet = databaseMetaData - .getColumns(Constant.CATALOG_STORAGE_GROUP, null, null, null); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - int colCount = resultSetMetaData.getColumnCount(); - StringBuilder resultStr = new StringBuilder(); - for (int i = 1; i < colCount + 1; i++) { - resultStr.append(resultSetMetaData.getColumnName(i)).append(","); - } - resultStr.append("\n"); - while (resultSet.next()) { - for (int i = 1; i <= colCount; i++) { - resultStr.append(resultSet.getString(i)).append(","); + try (ResultSet resultSet = databaseMetaData + .getColumns(Constant.CATALOG_STORAGE_GROUP, null, null, null);) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + int colCount = resultSetMetaData.getColumnCount(); + StringBuilder resultStr = new StringBuilder(); + for (int i = 1; i < colCount + 1; i++) { + resultStr.append(resultSetMetaData.getColumnName(i)).append(","); } resultStr.append("\n"); + while (resultSet.next()) { + for (int i = 1; i <= colCount; i++) { + resultStr.append(resultSet.getString(i)).append(","); + } + resultStr.append("\n"); + } + Assert.assertEquals(resultStr.toString(), standard); } - Assert.assertEquals(resultStr.toString(), standard); } /** diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java index 2423bd354697c25f2a7825e90f74728b86c77516..13ab399400278d7cb7b9bc2f710915795a2173af 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiSeriesIT.java @@ -73,7 +73,6 @@ public class IoTDBMultiSeriesIT { daemon.active(); EnvironmentUtils.envSetUp(); - Thread.sleep(5000); insertData(); } @@ -90,13 +89,11 @@ public class IoTDBMultiSeriesIT { } private static void insertData() - throws ClassNotFoundException, SQLException, InterruptedException { + throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { for (String sql : Constant.create_sql) { statement.execute(sql); @@ -139,9 +136,7 @@ public class IoTDBMultiSeriesIT { } statement.execute("flush"); - // statement.execute("merge"); - - Thread.sleep(5000); + statement.execute("merge"); // buffwrite data, unsealed file for (int time = 100000; time < 101000; time++) { @@ -210,53 +205,40 @@ public class IoTDBMultiSeriesIT { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } // "select * from root.vehicle" : test select wild data @Test - public void selectAllTest() throws ClassNotFoundException, SQLException { + public void selectAllTest() throws ClassNotFoundException { String selectSql = "select * from root.vehicle"; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute(selectSql); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0) - + "," + resultSet.getString(Constant.d0s1) + "," + resultSet - .getString(Constant.d0s2) + "," - + resultSet.getString(Constant.d0s3) + "," + resultSet.getString(Constant.d0s4) - + "," - + resultSet.getString(Constant.d0s5); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0) + + "," + resultSet.getString(Constant.d0s1) + "," + resultSet + .getString(Constant.d0s2) + "," + + resultSet.getString(Constant.d0s3) + "," + resultSet.getString(Constant.d0s4) + + "," + + resultSet.getString(Constant.d0s5); + cnt++; + } + assertEquals(23400, cnt); } - - assertEquals(23400, cnt); - statement.close(); - } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -267,31 +249,26 @@ public class IoTDBMultiSeriesIT { String selectSql = "select s0 from root.vehicle.d0 where s0 >= 20"; Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute(selectSql); Assert.assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0); - // System.out.println("===" + ans); - cnt++; + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0); + // System.out.println("===" + ans); + cnt++; + } + assertEquals(16440, cnt); } - assertEquals(16440, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -300,37 +277,29 @@ public class IoTDBMultiSeriesIT { public void seriesGlobalTimeFilterTest() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); + boolean hasResultSet; - Statement statement; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement();) { hasResultSet = statement.execute("select s0 from root.vehicle.d0 where time > 22987"); assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - String ans = - resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0); - // System.out.println(ans); - cnt++; - } + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + String ans = + resultSet.getString(Constant.TIMESTAMP_STR) + "," + resultSet.getString(Constant.d0s0); + // System.out.println(ans); + cnt++; + } - assertEquals(3012, cnt); - statement.close(); + assertEquals(3012, cnt); + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } @@ -338,39 +307,31 @@ public class IoTDBMultiSeriesIT { @Test public void crossSeriesReadUpdateTest() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); + boolean hasResultSet; - Statement statement; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { hasResultSet = statement.execute("select s1 from root.vehicle.d0 where s0 < 111"); assertTrue(hasResultSet); - ResultSet resultSet = statement.getResultSet(); - int cnt = 0; - while (resultSet.next()) { - long time = Long.valueOf(resultSet.getString(Constant.TIMESTAMP_STR)); - String value = resultSet.getString(Constant.d0s1); - if (time > 200900) { - assertEquals("7777", value); + try (ResultSet resultSet = statement.getResultSet()) { + int cnt = 0; + while (resultSet.next()) { + long time = Long.valueOf(resultSet.getString(Constant.TIMESTAMP_STR)); + String value = resultSet.getString(Constant.d0s1); + if (time > 200900) { + assertEquals("7777", value); + } + // String ans = resultSet.getString(d0s1); + cnt++; } - // String ans = resultSet.getString(d0s1); - cnt++; + assertEquals(22800, cnt); } - assertEquals(22800, cnt); - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBPreparedStmtIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBPreparedStmtIT.java index f99111a62f417354098e405f896f985ce6d4f6c5..f32ce32ef15227236892f7a541032fd5db0f3b46 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBPreparedStmtIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBPreparedStmtIT.java @@ -57,11 +57,9 @@ public class IoTDBPreparedStmtIT { } private static void prepareSeries() throws SQLException { - Connection connection; - connection = DriverManager + try (Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - try { - Statement statement = connection.createStatement(); + Statement statement = connection.createStatement()) { statement.execute("SET STORAGE GROUP TO root.device1"); statement.execute("SET STORAGE GROUP TO root.device2"); @@ -82,20 +80,18 @@ public class IoTDBPreparedStmtIT { + "sensor4) VALUES (1,1,1,1,1,1)"); statement.execute("INSERT INTO root.device3(timestamp,sensor0,sensor1,sensor2,sensor3," + "sensor4) VALUES (1,1,1,1,1,1)"); - statement.close(); - } finally { - connection.close(); } } @Test public void testPreparedInsertion() throws SQLException { - Connection connection; - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - IoTDBPreparedInsertionStatement statement = (IoTDBPreparedInsertionStatement) connection.prepareStatement("INSERT"); - try { + + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + IoTDBPreparedInsertionStatement statement = + (IoTDBPreparedInsertionStatement) connection.prepareStatement("INSERT"); + Statement queryStmt = connection.createStatement()){ statement.setDeviceId("root.device1"); String[] measurements = new String[5]; for (int i = 0; i < 5; i++) { @@ -111,9 +107,8 @@ public class IoTDBPreparedStmtIT { statement.setValues(Arrays.asList(values)); statement.execute(); } - statement.close(); - Statement queryStmt = connection.createStatement(); + ResultSet resultSet = queryStmt.executeQuery("SELECT * FROM root.device1"); int cnt = 0; while (resultSet.next()) { @@ -125,25 +120,22 @@ public class IoTDBPreparedStmtIT { } assertEquals(100, cnt); resultSet.close(); - queryStmt.close(); - } finally { - connection.close(); } } @Ignore @Test public void testPreparedInsertionPerf() throws SQLException { - Connection connection; - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); long preparedConsumption; long preparedIdealConsumption; long normalConsumption; - try { + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + IoTDBPreparedInsertionStatement statement = (IoTDBPreparedInsertionStatement) connection + .prepareStatement("INSERT")) { // normal usage, all parameters are updated - IoTDBPreparedInsertionStatement statement = (IoTDBPreparedInsertionStatement) connection.prepareStatement("INSERT"); + long startTime = System.currentTimeMillis(); String[] measurements = new String[5]; for (int i = 0; i < 5; i++) { @@ -154,24 +146,20 @@ public class IoTDBPreparedStmtIT { statement.setDeviceId("root.device1"); statement.setMeasurements(Arrays.asList(measurements)); statement.setTimestamp(i); - for (int j = 0; j < 5; j ++) { + for (int j = 0; j < 5; j++) { values[j] = String.valueOf(j); } statement.setValues(Arrays.asList(values)); statement.execute(); } - statement.close(); preparedConsumption = System.currentTimeMillis() - startTime; - } finally { - connection.close(); } - connection = DriverManager + try (Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - try { + IoTDBPreparedInsertionStatement statement = (IoTDBPreparedInsertionStatement) connection.prepareStatement("INSERT")) { // ideal usage, only value and time are updated - IoTDBPreparedInsertionStatement statement = (IoTDBPreparedInsertionStatement) connection.prepareStatement("INSERT"); long startTime = System.currentTimeMillis(); statement.setDeviceId("root.device2"); String[] measurements = new String[5]; @@ -188,17 +176,12 @@ public class IoTDBPreparedStmtIT { statement.setValues(Arrays.asList(values)); statement.execute(); } - statement.close(); preparedIdealConsumption = System.currentTimeMillis() - startTime; - } finally { - connection.close(); } - - connection = DriverManager + try (Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - try { - Statement statement = connection.createStatement(); + Statement statement = connection.createStatement()) { long startTime = System.currentTimeMillis(); String insertionTemplate = "INSERT INTO root.device3(timestamp,sensor0,sensor1,sensor2," + "sensor3,sensor4) VALUES (%d,%d,%d,%d,%d,%d)"; @@ -211,10 +194,7 @@ public class IoTDBPreparedStmtIT { } statement.execute(String.format(insertionTemplate, args)); } - statement.close(); normalConsumption = System.currentTimeMillis() - startTime; - } finally { - connection.close(); } System.out.printf("Prepared statement costs %dms, ideal prepared statement costs %dms, normal " + "statement costs %dms \n", diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java index e10ca5e69f12f03cc15df32f234dcec9cfaf4ecd..e5b1e2574db30231e19d9f69ef8b42d22aea3121 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java @@ -102,11 +102,9 @@ public class IoTDBSequenceDataQueryIT { private static void insertData() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { // create storage group and measurement for (String sql : Constant.create_sql) { @@ -157,14 +155,9 @@ public class IoTDBSequenceDataQueryIT { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java index b1b255fb1d742d2e7345ae56131c5a0c17b1c161..e83865de197738dc7290df6bb620803f7db49322 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java @@ -86,7 +86,6 @@ public class IoTDBSeriesReaderIT { daemon.active(); EnvironmentUtils.envSetUp(); - Thread.sleep(1000); insertData(); connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); @@ -108,11 +107,10 @@ public class IoTDBSeriesReaderIT { private static void insertData() throws ClassNotFoundException, SQLException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement() ) { + for (String sql : Constant.create_sql) { statement.execute(sql); @@ -156,9 +154,7 @@ public class IoTDBSeriesReaderIT { statement.execute(sql); } -// statement.execute("merge"); - - Thread.sleep(5000); + statement.execute("merge"); // buffwrite data, unsealed file for (int time = 100000; time < 101000; time++) { @@ -227,14 +223,9 @@ public class IoTDBSeriesReaderIT { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTimeZoneIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTimeZoneIT.java index c3a8338db5dbfde5f257342d368557243de1ebc2..1c60ba0747c05e1f053ca82324c903db0346ba22 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTimeZoneIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBTimeZoneIT.java @@ -84,12 +84,11 @@ public class IoTDBTimeZoneIT { @Test public void timezoneTest() throws ClassNotFoundException, SQLException, TException { Class.forName(Config.JDBC_DRIVER_NAME); - IoTDBConnection connection = null; - try { - connection = (IoTDBConnection) DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", - "root", "root"); - Statement statement = connection.createStatement(); + try (IoTDBConnection connection = (IoTDBConnection) DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", + "root", "root"); + Statement statement = connection.createStatement()) { + String insertSQLTemplate = "insert into root.timezone(timestamp,tz1) values(%s,%s)"; connection.setTimeZone("+08:00"); // 1514779200000 = 2018-1-1T12:00:00+08:00 @@ -123,30 +122,21 @@ public class IoTDBTimeZoneIT { cnt++; } Assert.assertEquals(13, cnt); - statement.close(); - } finally { - connection.close(); } } - public void createTimeseries() throws ClassNotFoundException, SQLException { + private void createTimeseries() throws ClassNotFoundException { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + for (String sql : insertSqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); - } finally { - if (connection != null) { - connection.close(); - } } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBVersionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBVersionIT.java index 7f1f8acf571ca5f713bae3148f59aa640ff46663..4b53d9f514f24f66f569532e3d576a9e27807a17 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBVersionIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBVersionIT.java @@ -52,9 +52,8 @@ public class IoTDBVersionIT { Class.forName(Config.JDBC_DRIVER_NAME); try(Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", - "root", "root")){ - Statement statement = connection.createStatement(); - + "root", "root"); + Statement statement = connection.createStatement()){ statement.execute("SET STORAGE GROUP TO root.versionTest1"); statement.execute("SET STORAGE GROUP TO root.versionTest2"); statement.execute("CREATE TIMESERIES root.versionTest1.s0" @@ -64,20 +63,18 @@ public class IoTDBVersionIT { // insert and flush enough times to make the version file persist for (int i = 0; i < 2 * SimpleFileVersionController.getSaveInterval(); i ++) { - for (int j = 1; j <= 100; j ++) { + for (int j = 1; j <= 10; j ++) { statement.execute(String .format("INSERT INTO root.versionTest1(timestamp, s0) VALUES (%d, %d)", i*100+j, j)); } statement.execute("FLUSH"); - for (int j = 1; j <= 100; j ++) { + for (int j = 1; j <= 10; j ++) { statement.execute(String .format("INSERT INTO root.versionTest2(timestamp, s0) VALUES (%d, %d)", i*100+j, j)); } statement.execute("FLUSH"); -// statement.execute("MERGE"); + statement.execute("MERGE"); } - - statement.close(); } } } diff --git a/server/src/test/java/org/apache/iotdb/db/qp/utils/MemIntQpExecutor.java b/server/src/test/java/org/apache/iotdb/db/qp/utils/MemIntQpExecutor.java index 13591b7256ecb3114e4a1b4b73a06b8875090635..01aa7d2d8396549048f0e437d053c12f12461caa 100644 --- a/server/src/test/java/org/apache/iotdb/db/qp/utils/MemIntQpExecutor.java +++ b/server/src/test/java/org/apache/iotdb/db/qp/utils/MemIntQpExecutor.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.qp.constant.SQLConstant; import org.apache.iotdb.db.qp.executor.AbstractQueryProcessExecutor; import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.qp.physical.crud.UpdatePlan; @@ -193,6 +194,11 @@ public class MemIntQpExecutor extends AbstractQueryProcessExecutor { return true; } + @Override + public Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws ProcessorException { + return null; + } + @Override protected QueryDataSet processAuthorQuery(AuthorPlan plan, QueryContext context) { return null; diff --git a/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java b/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java index 970489d74ed5c595fced8d6c7a7e0500b86741b1..683eb674b9d5bbce8b96835af91dcf22a19471b3 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/query/control/FileReaderManagerTest.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.query.control; -import static org.apache.iotdb.db.utils.EnvironmentUtils.TEST_QUERY_JOB_ID; import static org.junit.Assert.fail; import java.io.File; @@ -26,6 +25,7 @@ import java.io.IOException; import java.nio.file.Files; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -57,9 +57,12 @@ public class FileReaderManagerTest { FileReaderManager manager = FileReaderManager.getInstance(); JobFileManager testManager = new JobFileManager(); + TsFileResource[] tsFileResources = new TsFileResource[MAX_FILE_SIZE + 1]; + for (int i = 1; i <= MAX_FILE_SIZE; i++) { File file = new File(filePath + i); file.createNewFile(); + tsFileResources[i] = new TsFileResource(file); } Thread t1 = new Thread(() -> { @@ -67,10 +70,15 @@ public class FileReaderManagerTest { testManager.addJobId(1L); for (int i = 1; i <= 6; i++) { - testManager.addFilePathToMap(1L, filePath + i, + TsFileResource tsFile = tsFileResources[i]; + testManager.addFilePathToMap(1L, tsFile, false); - manager.get(filePath + i, false); - Assert.assertTrue(manager.contains(filePath + i, false)); + manager.get(tsFile, false); + Assert.assertTrue(manager.contains(tsFile, false)); + } + for (int i = 1; i <= 6; i++) { + TsFileResource tsFile = tsFileResources[i]; + manager.decreaseFileReaderReference(tsFile, false); } } catch (IOException e) { @@ -85,10 +93,15 @@ public class FileReaderManagerTest { testManager.addJobId(2L); for (int i = 4; i <= MAX_FILE_SIZE; i++) { - testManager.addFilePathToMap(2L, filePath + i, + TsFileResource tsFile = tsFileResources[i]; + testManager.addFilePathToMap(2L, tsFile, false); - manager.get(filePath + i, false); - Assert.assertTrue(manager.contains(filePath + i, false)); + manager.get(tsFile, false); + Assert.assertTrue(manager.contains(tsFile, false)); + } + for (int i = 4; i <= MAX_FILE_SIZE; i++) { + TsFileResource tsFile = tsFileResources[i]; + manager.decreaseFileReaderReference(tsFile, false); } } catch (IOException e) { @@ -102,12 +115,10 @@ public class FileReaderManagerTest { t2.join(); for (int i = 1; i <= MAX_FILE_SIZE; i++) { - Assert.assertTrue(manager.contains(filePath + i, false)); + TsFileResource tsFile = new TsFileResource(new File(filePath + i)); + Assert.assertTrue(manager.contains(tsFile, false)); } - for (int i = 1; i <= MAX_FILE_SIZE; i++) { - manager.decreaseFileReaderReference(filePath + i, true); - } // the code below is not valid because the cacheFileReaderClearPeriod config in this class is not valid diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderTest.java b/server/src/test/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderTest.java index 1191e09f31fa53289359cb913cf8388446af2170..148ae50bb68635321e3676498843ec103c5c7d42 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderTest.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/fileRelated/UnSealedTsFileReaderTest.java @@ -35,7 +35,8 @@ public class UnSealedTsFileReaderTest extends ReaderTestHelper { @Test public void testUnSealedTsFileIterateReader() throws IOException { - QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context); + QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context, + null); TsFileResource resource = queryDataSource.getSeqResources().get(0); Assert.assertEquals(false, resource.isClosed()); UnSealedTsFileIterateReader reader = new UnSealedTsFileIterateReader(resource, null, false); @@ -53,7 +54,8 @@ public class UnSealedTsFileReaderTest extends ReaderTestHelper { @Test public void testUnSealedTsFileReaderByTimestamp() throws IOException { - QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context); + QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context, + null); TsFileResource resource = queryDataSource.getSeqResources().get(0); Assert.assertEquals(false, resource.isClosed()); UnSealedTsFileReaderByTimestamp reader = new UnSealedTsFileReaderByTimestamp( diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderTest.java b/server/src/test/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderTest.java index 2c2b85c3571adc339c67338ad34cbc09bb5801bd..2d33d14631175d2e71d61dc293f7bcd726046979 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderTest.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/resourceRelated/SeqResourceReaderTest.java @@ -34,7 +34,8 @@ public class SeqResourceReaderTest extends ReaderTestHelper { @Test public void testSeqResourceIterateReader() throws IOException { - QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context); + QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context, + null); Path path = new Path(deviceId, measurementId); SeqResourceIterateReader reader = new SeqResourceIterateReader(path, queryDataSource.getSeqResources(), null, EnvironmentUtils.TEST_QUERY_CONTEXT); @@ -52,7 +53,8 @@ public class SeqResourceReaderTest extends ReaderTestHelper { @Test public void testSeqResourceReaderByTimestamp() throws IOException { - QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context); + QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context, + null); Path path = new Path(deviceId, measurementId); SeqResourceReaderByTimestamp reader = new SeqResourceReaderByTimestamp(path, queryDataSource.getSeqResources(), EnvironmentUtils.TEST_QUERY_CONTEXT); diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderTest.java b/server/src/test/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderTest.java index 18c0139806d2e0ed126b1db55d386f2dec6978b6..adbdb273d2cb6131286ad7c7ae04ee13b74e1485 100644 --- a/server/src/test/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderTest.java +++ b/server/src/test/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceReaderTest.java @@ -68,7 +68,8 @@ public class UnseqResourceReaderTest extends ReaderTestHelper { @Test public void testUnseqResourceMergeReaderWithGlobalTimeFilter() throws IOException { Path path = new Path(deviceId, measurementId); - QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context); + QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context, + null); IPointReader reader = new UnseqResourceMergeReader(path, queryDataSource.getUnseqResources(), EnvironmentUtils.TEST_QUERY_CONTEXT, TimeFilter.lt(4)); @@ -90,7 +91,7 @@ public class UnseqResourceReaderTest extends ReaderTestHelper { public void testUnseqResourceMergeReaderWithoutFilter() throws IOException { Path path = new Path(deviceId, measurementId); QueryDataSource queryDataSource = storageGroupProcessor - .query(deviceId, measurementId, context); + .query(deviceId, measurementId, context, null); IPointReader reader = new UnseqResourceMergeReader(path, queryDataSource.getUnseqResources(), EnvironmentUtils.TEST_QUERY_CONTEXT, null); @@ -112,7 +113,8 @@ public class UnseqResourceReaderTest extends ReaderTestHelper { @Test public void testUnseqResourceReaderByTimestamp() throws IOException, StorageEngineException { Path path = new Path(deviceId, measurementId); - QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context); + QueryDataSource queryDataSource = storageGroupProcessor.query(deviceId, measurementId, context, + null); IReaderByTimestamp reader = new UnseqResourceReaderByTimestamp(path, queryDataSource.getUnseqResources(), EnvironmentUtils.TEST_QUERY_CONTEXT); diff --git a/server/src/test/java/org/apache/iotdb/db/sync/sender/SingleClientSyncTest.java b/server/src/test/java/org/apache/iotdb/db/sync/sender/SingleClientSyncTest.java index 86b82f66e27867423bad05d5586cf4e94603b488..23525f305c17562a9af89a10feee980f5f6ca3ec 100644 --- a/server/src/test/java/org/apache/iotdb/db/sync/sender/SingleClientSyncTest.java +++ b/server/src/test/java/org/apache/iotdb/db/sync/sender/SingleClientSyncTest.java @@ -209,12 +209,11 @@ public class SingleClientSyncTest { Class.forName(Config.JDBC_DRIVER_NAME); try (Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root")) { - Statement statement = connection.createStatement(); + "root"); + Statement statement = connection.createStatement()) { for (String sql : sqls1) { statement.execute(sql); } - statement.close(); } } catch (SQLException | ClassNotFoundException e) { fail(e.getMessage()); @@ -228,8 +227,8 @@ public class SingleClientSyncTest { Class.forName(Config.JDBC_DRIVER_NAME); try (Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root")) { - Statement statement = connection.createStatement(); + "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select * from root.vehicle"); if (hasResultSet) { ResultSet res = statement.getResultSet(); @@ -252,7 +251,6 @@ public class SingleClientSyncTest { + res.getString("root.test.d0.s1") + res.getString("root.test.d1.g0.s0")); } } - statement.close(); } catch (Exception e) { logger.error("", e); } @@ -264,12 +262,11 @@ public class SingleClientSyncTest { dataReceiver.clear(); try { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(String.format("jdbc:iotdb://%s:6667/", serverIpTest), "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(String.format("jdbc:iotdb://%s:6667/", serverIpTest), "root", + "root"); + Statement statement = connection.createStatement()) { + boolean hasResultSet = statement.execute("select * from root.vehicle"); if (hasResultSet) { ResultSet res = statement.getResultSet(); @@ -293,15 +290,10 @@ public class SingleClientSyncTest { + res.getString("root.test.d0.s1") + res.getString("root.test.d1.g0.s0")); } } - statement.close(); } catch (Exception e) { logger.error("", e); - } finally { - if (connection != null) { - connection.close(); - } } - } catch (ClassNotFoundException | SQLException e) { + } catch (ClassNotFoundException e) { fail(e.getMessage()); Thread.currentThread().interrupt(); } @@ -318,24 +310,17 @@ public class SingleClientSyncTest { logger.debug("It's the second time to sync!"); try { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { for (String sql : sqls2) { statement.execute(sql); } - statement.close(); } catch (Exception e) { logger.error("", e); - } finally { - if (connection != null) { - connection.close(); - } } - } catch (ClassNotFoundException | SQLException e) { + } catch (ClassNotFoundException e) { fail(e.getMessage()); Thread.currentThread().interrupt(); } @@ -346,12 +331,10 @@ public class SingleClientSyncTest { dataSender.clear(); try { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root"); - Statement statement = connection.createStatement(); + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", + "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select * from root.vehicle"); if (hasResultSet) { ResultSet res = statement.getResultSet(); @@ -374,15 +357,10 @@ public class SingleClientSyncTest { + res.getString("root.test.d0.s1") + res.getString("root.test.d1.g0.s0")); } } - statement.close(); } catch (Exception e) { logger.error("", e); - } finally { - if (connection != null) { - connection.close(); - } } - } catch (ClassNotFoundException | SQLException e) { + } catch (ClassNotFoundException e) { fail(e.getMessage()); Thread.currentThread().interrupt(); } @@ -390,11 +368,9 @@ public class SingleClientSyncTest { dataReceiver.clear(); { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection(String.format("jdbc:iotdb://%s:6667/", serverIpTest), "root", - "root"); + try (Connection connection = DriverManager + .getConnection(String.format("jdbc:iotdb://%s:6667/", serverIpTest), "root", + "root")) { Statement statement = connection.createStatement(); boolean hasResultSet = statement.execute("select * from root.vehicle"); if (hasResultSet) { @@ -421,10 +397,6 @@ public class SingleClientSyncTest { statement.close(); } catch (Exception e) { logger.error("", e); - } finally { - if (connection != null) { - connection.close(); - } } } logger.debug(String.valueOf(dataSender.size())); @@ -442,12 +414,11 @@ public class SingleClientSyncTest { Class.forName(Config.JDBC_DRIVER_NAME); try (Connection connection = DriverManager .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", - "root")) { - Statement statement = connection.createStatement(); + "root"); + Statement statement = connection.createStatement()) { for (String sql : sqls3) { statement.execute(sql); } - statement.close(); } catch (Exception e) { logger.error("", e); } @@ -464,8 +435,8 @@ public class SingleClientSyncTest { Class.forName(Config.JDBC_DRIVER_NAME); try (Connection connection = DriverManager .getConnection(String.format("jdbc:iotdb://%s:6667/", serverIpTest), "root", - "root")) { - Statement statement = connection.createStatement(); + "root"); + Statement statement = connection.createStatement()) { boolean hasResultSet = statement.execute("select * from root.vehicle"); if (hasResultSet) { ResultSet res = statement.getResultSet(); @@ -510,7 +481,6 @@ public class SingleClientSyncTest { + res.getString("root.iotdb.d1.s3")); } } - statement.close(); } catch (Exception e) { logger.error("", e); } @@ -522,10 +492,8 @@ public class SingleClientSyncTest { dataReceiver.clear(); try { Class.forName(Config.JDBC_DRIVER_NAME); - Connection connection = null; - try { - connection = DriverManager - .getConnection("jdbc:iotdb://192.168.130.8:6667/", "root", "root"); + try (Connection connection = DriverManager + .getConnection("jdbc:iotdb://192.168.130.8:6667/", "root", "root")) { Statement statement = connection.createStatement(); boolean hasResultSet = statement.execute("select * from root.vehicle"); if (hasResultSet) { @@ -574,12 +542,8 @@ public class SingleClientSyncTest { statement.close(); } catch (Exception e) { logger.error("", e); - } finally { - if (connection != null) { - connection.close(); - } } - } catch (ClassNotFoundException | SQLException e) { + } catch (ClassNotFoundException e) { fail(e.getMessage()); Thread.currentThread().interrupt(); } diff --git a/server/src/test/java/org/apache/iotdb/db/sync/test/SyncTestClient1.java b/server/src/test/java/org/apache/iotdb/db/sync/test/SyncTestClient1.java index e7e7b9ed307ea1ea2a22831d73a353933f7f2ffc..5e9aab74ef515d22b375213747a0830434cbe67c 100644 --- a/server/src/test/java/org/apache/iotdb/db/sync/test/SyncTestClient1.java +++ b/server/src/test/java/org/apache/iotdb/db/sync/test/SyncTestClient1.java @@ -219,8 +219,6 @@ public class SyncTestClient1 { */ public static void main(String[] args) throws Exception { - Statement statement = null; - String path = new File(System.getProperty(IoTDBConstant.IOTDB_HOME, null)).getParent() + File.separator + "src" @@ -235,9 +233,10 @@ public class SyncTestClient1 { storageGroupList.add("root.range_event"); try (Connection connection = DriverManager - .getConnection("jdbc:iotdb://localhost:6667/", "root", "root")) { + .getConnection("jdbc:iotdb://localhost:6667/", "root", "root"); + Statement statement = connection.createStatement();) { Class.forName("org.apache.iotdb.jdbc.IoTDBDriver"); - statement = connection.createStatement(); + setStorageGroup(statement, storageGroupList); logger.debug("Finish set storage group."); @@ -249,10 +248,6 @@ public class SyncTestClient1 { } catch (Exception e) { logger.error("", e); - } finally { - if (statement != null) { - statement.close(); - } } } } \ No newline at end of file diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index 3a6a255b0ad69dc00b2f5953be9d1ec81dd425a0..2533b0313f90f32d77ce4c0adb704f7716ec8496 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@ -102,7 +102,7 @@ public class EnvironmentUtils { IoTDBConfigDynamicAdapter.getInstance().reset(); } - private static void cleanAllDir() throws IOException { + public static void cleanAllDir() throws IOException { // delete sequential files for (String path : directoryManager.getAllSequenceFileFolders()) { cleanDir(path); diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/IoTDBLogFileSizeTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/IoTDBLogFileSizeTest.java index 391a2b60db744a7c08e0f3c1331a2f8bfc3406fe..76e999b0963ac718986990b443b2f4d9deca7da1 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/IoTDBLogFileSizeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/IoTDBLogFileSizeTest.java @@ -22,13 +22,11 @@ package org.apache.iotdb.db.writelog; import java.io.File; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.SQLException; import java.sql.Statement; import org.apache.iotdb.db.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.utils.EnvironmentUtils; -import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager; import org.apache.iotdb.db.writelog.node.ExclusiveWriteLogNode; import org.apache.iotdb.db.writelog.node.WriteLogNode; @@ -80,7 +78,6 @@ public class IoTDBLogFileSizeTest { IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(groupSize); executeSQL(tearDownSqls); daemon.stop(); - Thread.sleep(5000); EnvironmentUtils.cleanEnv(); } @@ -98,16 +95,12 @@ public class IoTDBLogFileSizeTest { e.printStackTrace(); return; } - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); - while (true) { - if (Thread.interrupted()) { - //System.out.println("Exit after " + cnt + " insertion"); - break; - } + + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + //System.out.println("Exit after " + cnt + " insertion"); + while (!Thread.interrupted()) { String sql = String.format( "INSERT INTO root.logFileTest.seq(timestamp,val) VALUES (%d, %d)", ++cnt, cnt); @@ -120,17 +113,8 @@ public class IoTDBLogFileSizeTest { maxLength[0] = bufferWriteWALFile.length(); } } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - try { - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } } }); writeThread.start(); @@ -155,16 +139,11 @@ public class IoTDBLogFileSizeTest { e.printStackTrace(); return; } - Connection connection = null; - try { - connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); - Statement statement = connection.createStatement(); - while (true) { - if (Thread.interrupted()) { - //System.out.println("Exit after " + cnt + " insertion"); - break; - } + try (Connection connection = DriverManager + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { + //System.out.println("Exit after " + cnt + " insertion"); + while (!Thread.interrupted()) { String sql = String .format("INSERT INTO root.logFileTest.unsequence(timestamp,val) VALUES (%d, %d)", ++cnt, cnt); @@ -177,17 +156,8 @@ public class IoTDBLogFileSizeTest { maxLength[0] = WALFile.length(); } } - statement.close(); } catch (Exception e) { e.printStackTrace(); - } finally { - if (connection != null) { - try { - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } } }); writeThread.start(); @@ -198,15 +168,14 @@ public class IoTDBLogFileSizeTest { } } - private void executeSQL(String[] sqls) throws ClassNotFoundException, SQLException { + private void executeSQL(String[] sqls) throws ClassNotFoundException { Class.forName(Config.JDBC_DRIVER_NAME); try (Connection connection = DriverManager - .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root")) { - Statement statement = connection.createStatement(); + .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); + Statement statement = connection.createStatement()) { for (String sql : sqls) { statement.execute(sql); } - statement.close(); } catch (Exception e) { e.printStackTrace(); } diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java index 6fede6685823285b70dcc317504708a91805fd6c..b8c0613dcc5040ff822e4ae7d142d6d22db35934 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java @@ -27,13 +27,13 @@ import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.Iterator; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.memtable.IMemTable; import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable; import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.version.VersionController; import org.apache.iotdb.db.exception.ProcessorException; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; @@ -44,7 +44,7 @@ import org.apache.iotdb.db.writelog.node.WriteLogNode; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.Test; @@ -69,11 +69,12 @@ public class LogReplayerTest { }; TsFileResource tsFileResource = new TsFileResource(tsFile); IMemTable memTable = new PrimitiveMemTable(); - FileSchema schema = new FileSchema(); + Schema schema = new Schema(); try { for (int i = 0; i < 5; i++) { - schema.registerMeasurement(new MeasurementSchema("sensor" + i, TSDataType.INT64, TSEncoding.PLAIN)); + schema.registerMeasurement( + new MeasurementSchema("sensor" + i, TSDataType.INT64, TSEncoding.PLAIN)); } LogReplayer replayer = new LogReplayer(logNodePrefix, tsFile.getPath(), modFile, @@ -81,10 +82,12 @@ public class LogReplayerTest { WriteLogNode node = MultiFileLogNodeManager.getInstance().getNode(logNodePrefix + tsFile.getName()); - for (int i = 0; i < 5; i++) { + node.write(new InsertPlan("device0", 100, "sensor0", String.valueOf(0))); + node.write(new InsertPlan("device0", 2, "sensor1", String.valueOf(0))); + for (int i = 1; i < 5; i++) { node.write(new InsertPlan("device" + i, i, "sensor" + i, String.valueOf(i))); } - DeletePlan deletePlan = new DeletePlan(3, new Path("device0", "sensor0")); + DeletePlan deletePlan = new DeletePlan(200, new Path("device0", "sensor0")); node.write(deletePlan); node.close(); @@ -107,11 +110,13 @@ public class LogReplayerTest { Modification[] mods = modFile.getModifications().toArray(new Modification[0]); assertEquals(1, mods.length); - assertEquals(new Deletion(new Path("device0", "sensor0"), 5, 3), mods[0]); + assertEquals(new Deletion(new Path("device0", "sensor0"), 5, 200), mods[0]); - for (int i = 0; i < 5; i++) { - assertEquals(i, (long)tsFileResource.getStartTimeMap().get("device" + i)); - assertEquals(i, (long)tsFileResource.getEndTimeMap().get("device" + i)); + assertEquals(2, (long) tsFileResource.getStartTimeMap().get("device0")); + assertEquals(100, (long) tsFileResource.getEndTimeMap().get("device0")); + for (int i = 1; i < 5; i++) { + assertEquals(i, (long) tsFileResource.getStartTimeMap().get("device" + i)); + assertEquals(i, (long) tsFileResource.getEndTimeMap().get("device" + i)); } } finally { modFile.close(); diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java index d47214a9d3a8f501042efeb5e9f19e72d2df8ab7..908337ad5388345907687cc28e52f9e28224b638 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/SeqTsFileRecoverTest.java @@ -45,21 +45,24 @@ import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.iotdb.tsfile.write.TsFileWriter; import org.apache.iotdb.tsfile.write.record.TSRecord; import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; public class SeqTsFileRecoverTest { + private File tsF; private TsFileWriter writer; private WriteLogNode node; private String logNodePrefix = "testNode"; - private FileSchema schema; + private Schema schema; private TsFileResource resource; private VersionController versionController = new VersionController() { private int i; + @Override public long nextVersion() { return ++i; @@ -76,16 +79,23 @@ public class SeqTsFileRecoverTest { tsF = new File("temp", "test.ts"); tsF.getParentFile().mkdirs(); - schema = new FileSchema(); + schema = new Schema(); for (int i = 0; i < 10; i++) { schema.registerMeasurement(new MeasurementSchema("sensor" + i, TSDataType.INT64, TSEncoding.PLAIN)); } writer = new TsFileWriter(tsF, schema); + TSRecord tsRecord = new TSRecord(100, "device99"); + tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor4", String.valueOf(0))); + writer.write(tsRecord); + tsRecord = new TSRecord(2, "device99"); + tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor1", String.valueOf(0))); + writer.write(tsRecord); + for (int i = 0; i < 10; i++) { for (int j = 0; j < 10; j++) { - TSRecord tsRecord = new TSRecord(i, "device" + j); + tsRecord = new TSRecord(i, "device" + j); for (int k = 0; k < 10; k++) { tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor" + k, String.valueOf(k))); @@ -126,6 +136,13 @@ public class SeqTsFileRecoverTest { versionController, resource, true); performer.recover(); + assertEquals(2, (long) resource.getStartTimeMap().get("device99")); + assertEquals(100, (long) resource.getEndTimeMap().get("device99")); + for (int i = 0; i < 10; i++) { + assertEquals(0, (long) resource.getStartTimeMap().get("device" + i)); + assertEquals(19, (long) resource.getEndTimeMap().get("device" + i)); + } + ReadOnlyTsFile readOnlyTsFile = new ReadOnlyTsFile(new TsFileSequenceReader(tsF.getPath())); List pathList = new ArrayList<>(); for (int j = 0; j < 10; j++) { @@ -144,6 +161,19 @@ public class SeqTsFileRecoverTest { assertEquals(j % 10, fields.get(j).getLongV()); } } + + pathList = new ArrayList<>(); + pathList.add(new Path("device99", "sensor1")); + pathList.add(new Path("device99", "sensor4")); + queryExpression = QueryExpression.create(pathList, null); + dataSet = readOnlyTsFile.query(queryExpression); + Assert.assertTrue(dataSet.hasNext()); + RowRecord record = dataSet.next(); + Assert.assertEquals("2\t0\tnull", record.toString()); + Assert.assertTrue(dataSet.hasNext()); + record = dataSet.next(); + Assert.assertEquals("100\tnull\t0", record.toString()); + readOnlyTsFile.close(); } } diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java index 38c373d08f115bb317cbe34014bcb0cdf3fc55fd..8d1933117f5e5643c6a22110b1be85535b9070ac 100644 --- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverTest.java @@ -40,30 +40,32 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; -import org.apache.iotdb.tsfile.read.controller.MetadataQuerier; +import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter; import org.apache.iotdb.tsfile.write.TsFileWriter; import org.apache.iotdb.tsfile.write.record.TSRecord; import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; import org.junit.Test; public class UnseqTsFileRecoverTest { + private File tsF; private TsFileWriter writer; private WriteLogNode node; private String logNodePrefix = "testNode"; - private FileSchema schema; + private Schema schema; private TsFileResource resource; private VersionController versionController = new VersionController() { private int i; + @Override public long nextVersion() { return ++i; @@ -80,16 +82,23 @@ public class UnseqTsFileRecoverTest { tsF = new File("temp", "test.ts"); tsF.getParentFile().mkdirs(); - schema = new FileSchema(); + schema = new Schema(); for (int i = 0; i < 10; i++) { schema.registerMeasurement(new MeasurementSchema("sensor" + i, TSDataType.INT64, TSEncoding.PLAIN)); } writer = new TsFileWriter(tsF, schema); + TSRecord tsRecord = new TSRecord(100, "device99"); + tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor4", String.valueOf(0))); + writer.write(tsRecord); + tsRecord = new TSRecord(2, "device99"); + tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor1", String.valueOf(0))); + writer.write(tsRecord); + for (int i = 0; i < 10; i++) { for (int j = 0; j < 10; j++) { - TSRecord tsRecord = new TSRecord(i, "device" + j); + tsRecord = new TSRecord(i, "device" + j); for (int k = 0; k < 10; k++) { tsRecord.addTuple(DataPoint.getDataPoint(TSDataType.INT64, "sensor" + k, String.valueOf(k))); @@ -97,6 +106,7 @@ public class UnseqTsFileRecoverTest { writer.write(tsRecord); } } + writer.flushForTest(); writer.getIOWriter().close(); @@ -114,6 +124,12 @@ public class UnseqTsFileRecoverTest { } node.notifyStartFlush(); } + InsertPlan insertPlan = new InsertPlan("device99", 1, "sensor4", "4"); + node.write(insertPlan); + insertPlan = new InsertPlan("device99", 300, "sensor2", "2"); + node.write(insertPlan); + node.close(); + resource = new TsFileResource(tsF); } @@ -129,9 +145,16 @@ public class UnseqTsFileRecoverTest { versionController, resource, true); performer.recover(); + assertEquals(1, (long) resource.getStartTimeMap().get("device99")); + assertEquals(300, (long) resource.getEndTimeMap().get("device99")); + for (int i = 0; i < 10; i++) { + assertEquals(0, (long) resource.getStartTimeMap().get("device" + i)); + assertEquals(9, (long) resource.getEndTimeMap().get("device" + i)); + } + TsFileSequenceReader fileReader = new TsFileSequenceReader(tsF.getPath(), true); - MetadataQuerier metadataQuerier = new MetadataQuerierByFileImpl(fileReader); - ChunkLoader chunkLoader = new ChunkLoaderImpl(fileReader); + IMetadataQuerier metadataQuerier = new MetadataQuerierByFileImpl(fileReader); + IChunkLoader chunkLoader = new ChunkLoaderImpl(fileReader); Path path = new Path("device1", "sensor1"); @@ -150,7 +173,6 @@ public class UnseqTsFileRecoverTest { assertEquals(i, timeValuePair.getTimestamp()); assertEquals(11, timeValuePair.getValue().getLong()); unSeqMergeReader.next(); - } unSeqMergeReader.close(); fileReader.close(); diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRPCException.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRPCException.java new file mode 100644 index 0000000000000000000000000000000000000000..6f8620db4ca2c4266057694e37bad30aff0c6267 --- /dev/null +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRPCException.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.rpc; + +public class IoTDBRPCException extends Exception{ + + private static final long serialVersionUID = -1268775292265203036L; + + public IoTDBRPCException(String reason) { + super(reason); + } + +} diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..74df2bc5d509fd2ebc132ddec8d85c2095317b7f --- /dev/null +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.rpc; + +import java.lang.reflect.Proxy; +import org.apache.iotdb.service.rpc.thrift.TSIService; +import org.apache.iotdb.service.rpc.thrift.TS_Status; + +public class RpcUtils { + + public static TSIService.Iface newSynchronizedClient(TSIService.Iface client) { + return (TSIService.Iface) Proxy.newProxyInstance(RpcUtils.class.getClassLoader(), + new Class[]{TSIService.Iface.class}, new SynchronizedHandler(client)); + } + + /** + * verify success. + * + * @param status -status + */ + public static void verifySuccess(TS_Status status) throws IoTDBRPCException { + if (status.getStatusType().getCode() != TSStatusType.SUCCESS_STATUS.getStatusCode()) { + throw new IoTDBRPCException(status.getStatusType().getMessage()); + } + } + +} diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/SynchronizedHandler.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/SynchronizedHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..cd3599eefa14132f26e789a86f1b8c11fb4dee8c --- /dev/null +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/SynchronizedHandler.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.rpc; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import org.apache.iotdb.service.rpc.thrift.TSIService; +import org.apache.thrift.TException; + +public class SynchronizedHandler implements InvocationHandler { + + private final TSIService.Iface client; + + public SynchronizedHandler(TSIService.Iface client) { + this.client = client; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + try { + synchronized (client) { + return method.invoke(client, args); + } + } catch (InvocationTargetException e) { + // all IFace APIs throw TException + if (e.getTargetException() instanceof TException) { + throw e.getTargetException(); + } else { + // should not happen + throw new TException("Error in calling method " + method.getName(), + e.getTargetException()); + } + } catch (Exception e) { + throw new TException("Error in calling method " + method.getName(), e); + } + } +} diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusType.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusType.java new file mode 100644 index 0000000000000000000000000000000000000000..5a4acf6b4636b80d1b5b78222fa9ad8b86c6c592 --- /dev/null +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusType.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc; + +public enum TSStatusType { + SUCCESS_STATUS(200, ""), + STILL_EXECUTING_STATUS(201, ""), + INVALID_HANDLE_STATUS(202, ""), + TIMESERIES_NOT_EXIST_ERROR(301, "Timeseries does not exist"), + UNSUPPORTED_FETCH_METADATA_OPERATION_ERROR(302, "Unsupported fetch metadata operation"), + FETCH_METADATA_ERROR(303, "Failed to fetch metadata"), + CHECK_FILE_LEVEL_ERROR(304, "Meet error while checking file level"), + EXECUTE_STATEMENT_ERROR(400, "Execute statement error"), + SQL_PARSE_ERROR(401, "Meet error while parsing SQL"), + GENERATE_TIME_ZONE_ERROR(402, "Meet error while generating time zone"), + SET_TIME_ZONE_ERROR(403, "Meet error while setting time zone"), + INTERNAL_SERVER_ERROR(500, "Internal server error"), + WRONG_LOGIN_PASSWORD_ERROR(600, "Username or password is wrong"), + NOT_LOGIN_ERROR(601, "Has not logged in"), + NO_PERMISSION_ERROR(602, "No permissions for this operation"), + UNINITIALIZED_AUTH_ERROR(603, "Uninitialized authorizer"); + + private int statusCode; + private String statusMessage; + + private TSStatusType(int statusCode, String statusMessage) { + this.statusCode = statusCode; + this.statusMessage = statusMessage; + } + + public int getStatusCode() { + return statusCode; + } + + public String getStatusMessage() { + return statusMessage; + } +} diff --git a/service-rpc/src/main/thrift/rpc.thrift b/service-rpc/src/main/thrift/rpc.thrift index 43d405e90fcec720b34ab56eab51f6ead3f95122..6eee9c1bfac97a49884fd8c634f0d599feb35fc4 100644 --- a/service-rpc/src/main/thrift/rpc.thrift +++ b/service-rpc/src/main/thrift/rpc.thrift @@ -18,28 +18,17 @@ */ namespace java org.apache.iotdb.service.rpc.thrift - -// The return status code contained in each response. -enum TS_StatusCode { - SUCCESS_STATUS, - SUCCESS_WITH_INFO_STATUS, - STILL_EXECUTING_STATUS, - ERROR_STATUS, - INVALID_HANDLE_STATUS +// The return status code and message in each response. +struct TS_StatusType { + 1: required i32 code + 2: required string message } // The return status of a remote request struct TS_Status { - 1: required TS_StatusCode statusCode - - // If status is SUCCESS_WITH_INFO, info_msgs may be populated with - // additional diagnostic information. + 1: required TS_StatusType statusType 2: optional list infoMessages - - // If status is ERROR, then the following fields may be set - 3: optional string sqlState // as defined in the ISO/IEF CLI specification - 4: optional i32 errorCode // internal error code - 5: optional string errorMessage + 3: optional string sqlState // as defined in the ISO/IEF CLIENT specification } struct TSHandleIdentifier { @@ -79,7 +68,7 @@ struct TSExecuteStatementResp { } enum TSProtocolVersion { - TSFILE_SERVICE_PROTOCOL_V1, + IOTDB_SERVICE_PROTOCOL_V1, } // Client-side handle to persistent session information on the server-side. @@ -93,7 +82,7 @@ struct TSOpenSessionResp { 1: required TS_Status status // The protocol version that the server is using. - 2: required TSProtocolVersion serverProtocolVersion = TSProtocolVersion.TSFILE_SERVICE_PROTOCOL_V1 + 2: required TSProtocolVersion serverProtocolVersion = TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1 // Session Handle 3: optional TS_SessionHandle sessionHandle @@ -105,16 +94,12 @@ struct TSOpenSessionResp { // OpenSession() // Open a session (connection) on the server against which operations may be executed. struct TSOpenSessionReq { - 1: required TSProtocolVersion client_protocol = TSProtocolVersion.TSFILE_SERVICE_PROTOCOL_V1 + 1: required TSProtocolVersion client_protocol = TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1 2: optional string username 3: optional string password 4: optional map configuration } -struct TSCloseSessionResp { - 1: required TS_Status status -} - // CloseSession() // Closes the specified session and frees any resources currently allocated to that session. // Any open operations in that session will be canceled. @@ -156,10 +141,6 @@ struct TSGetOperationStatusReq { 1: required TSOperationHandle operationHandle } -struct TSGetOperationStatusResp { - 1: required TS_Status status -} - // CancelOperation() // // Cancels processing on the specified operation handle and frees any resources which were allocated. @@ -168,11 +149,6 @@ struct TSCancelOperationReq { 1: required TSOperationHandle operationHandle } -struct TSCancelOperationResp { - 1: required TS_Status status -} - - // CloseOperation() struct TSCloseOperationReq { 1: required TSOperationHandle operationHandle @@ -180,10 +156,6 @@ struct TSCloseOperationReq { 3: optional i64 stmtId } -struct TSCloseOperationResp { - 1: required TS_Status status -} - struct TSDataValue{ 1: required bool is_empty 2: optional bool bool_val @@ -192,7 +164,7 @@ struct TSDataValue{ 5: optional double float_val 6: optional double double_val 7: optional binary binary_val - 8: optional string type; + 8: optional string type } struct TSRowRecord{ @@ -247,10 +219,6 @@ struct TSSetTimeZoneReq { 1: required string timeZone } -struct TSSetTimeZoneResp { - 1: required TS_Status status -} - struct TSInsertionReq { 1: optional string deviceId 2: optional list measurements @@ -259,6 +227,29 @@ struct TSInsertionReq { 5: required i64 stmtId } +struct TSBatchInsertionReq { + 1: required string deviceId + 2: required list measurements + 3: required binary values + 4: required binary timestamps + 5: required list types + 6: required i32 size +} + +struct TSSetStorageGroupReq { + 1: required string storageGroupId +} + +struct TSRPCResp { + 1: required TS_Status status +} + +struct TSCreateTimeseriesReq { + 1: required string path + 2: required i32 dataType + 3: required i32 encoding +} + struct ServerProperties { 1: required string version; 2: required list supportedTimeAggregationOperations; @@ -268,7 +259,7 @@ struct ServerProperties { service TSIService { TSOpenSessionResp openSession(1:TSOpenSessionReq req); - TSCloseSessionResp closeSession(1:TSCloseSessionReq req); + TSRPCResp closeSession(1:TSCloseSessionReq req); TSExecuteStatementResp executeStatement(1:TSExecuteStatementReq req); @@ -282,17 +273,23 @@ service TSIService { TSFetchMetadataResp fetchMetadata(1:TSFetchMetadataReq req) - TSCancelOperationResp cancelOperation(1:TSCancelOperationReq req); + TSRPCResp cancelOperation(1:TSCancelOperationReq req); - TSCloseOperationResp closeOperation(1:TSCloseOperationReq req); + TSRPCResp closeOperation(1:TSCloseOperationReq req); TSGetTimeZoneResp getTimeZone(); - TSSetTimeZoneResp setTimeZone(1:TSSetTimeZoneReq req); + TSRPCResp setTimeZone(1:TSSetTimeZoneReq req); ServerProperties getProperties(); - TSExecuteStatementResp executeInsertion(1:TSInsertionReq req); + TSExecuteStatementResp insert(1:TSInsertionReq req); + + TSExecuteBatchStatementResp insertBatch(1:TSBatchInsertionReq req); + + TSRPCResp setStorageGroup(1:TSSetStorageGroupReq req); + + TSRPCResp createTimeseries(1:TSCreateTimeseriesReq req); i64 requestStatementId(); } diff --git a/session/pom.xml b/session/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..c1fa8bfa39063c87e5b7facaf0787c490ff78508 --- /dev/null +++ b/session/pom.xml @@ -0,0 +1,60 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.9.0-SNAPSHOT + + 4.0.0 + iotdb-session + IoTDB Session + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + UTF-8 + 1.7 + 1.7 + + + + org.apache.iotdb + service-rpc + 0.9.0-SNAPSHOT + + + org.apache.iotdb + tsfile + 0.9.0-SNAPSHOT + + + diff --git a/session/src/main/java/org/apache/iotdb/session/Config.java b/session/src/main/java/org/apache/iotdb/session/Config.java new file mode 100644 index 0000000000000000000000000000000000000000..501abe6857f1fa69ad7e70e7d5149f32d4889036 --- /dev/null +++ b/session/src/main/java/org/apache/iotdb/session/Config.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.session; + +public class Config { + + public static final String DEFAULT_USER = "user"; + public static final String DEFALUT_PASSWORD = "password"; + +} diff --git a/session/src/main/java/org/apache/iotdb/session/IoTDBSessionException.java b/session/src/main/java/org/apache/iotdb/session/IoTDBSessionException.java new file mode 100644 index 0000000000000000000000000000000000000000..440e955741f863ac321691904aa8b4de97c5c5c3 --- /dev/null +++ b/session/src/main/java/org/apache/iotdb/session/IoTDBSessionException.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.session; + +public class IoTDBSessionException extends Exception { + + private static final long serialVersionUID = 2405104784097667293L; + + public IoTDBSessionException(String msg) { + super(msg); + } + + public IoTDBSessionException(String message, Throwable cause) { + super(message, cause); + } + + public IoTDBSessionException(Throwable cause) { + super(cause); + } +} diff --git a/session/src/main/java/org/apache/iotdb/session/Session.java b/session/src/main/java/org/apache/iotdb/session/Session.java new file mode 100644 index 0000000000000000000000000000000000000000..0b13c586942ef95d91c6b3bf81e05a98a43ac58f --- /dev/null +++ b/session/src/main/java/org/apache/iotdb/session/Session.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.session; + +import org.apache.iotdb.rpc.IoTDBRPCException; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.service.rpc.thrift.*; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.record.RowBatch; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.ZoneId; + +public class Session { + + private String host; + private int port; + private String username; + private String password; + private final TSProtocolVersion protocolVersion = TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1; + public TSIService.Iface client = null; + private TS_SessionHandle sessionHandle = null; + private TSocket transport; + private boolean isClosed = true; + private ZoneId zoneId; + + public Session(String host, int port) { + this(host, port, Config.DEFAULT_USER, Config.DEFALUT_PASSWORD); + } + + public Session(String host, String port, String username, String password) { + this(host, Integer.parseInt(port), username, password); + } + + public Session(String host, int port, String username, String password) { + this.host = host; + this.port = port; + this.username = username; + this.password = password; + } + + public void open() throws IoTDBSessionException { + open(false, 0); + } + + public void open(boolean enableRPCCompression, int connectionTimeoutInMs) + throws IoTDBSessionException { + transport = new TSocket(host, port, connectionTimeoutInMs); + if (!transport.isOpen()) { + try { + transport.open(); + } catch (TTransportException e) { + throw new IoTDBSessionException(e); + } + } + + if(enableRPCCompression) { + client = new TSIService.Client(new TCompactProtocol(transport)); + } + else { + client = new TSIService.Client(new TBinaryProtocol(transport)); + } + + TSOpenSessionReq openReq = new TSOpenSessionReq(TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1); + openReq.setUsername(username); + openReq.setPassword(password); + + try { + TSOpenSessionResp openResp = client.openSession(openReq); + + // validate connectiontry { + RpcUtils.verifySuccess(openResp.getStatus()); + + if (protocolVersion.getValue() != openResp.getServerProtocolVersion().getValue()) { + throw new TException(String + .format("Protocol not supported, Client version is {}, but Server version is {}", + protocolVersion.getValue(), openResp.getServerProtocolVersion().getValue())); + } + + sessionHandle = openResp.getSessionHandle(); + + if (zoneId != null) { + setTimeZone(zoneId.toString()); + } else { + zoneId = ZoneId.of(getTimeZone()); + } + + } catch (TException | IoTDBRPCException e) { + transport.close(); + throw new IoTDBSessionException(String.format("Can not open session to %s:%s with user: %s.", + host, port, username), e); + } + isClosed = false; + + client = RpcUtils.newSynchronizedClient(client); + + } + + public void close() throws IoTDBSessionException { + if (isClosed) { + return; + } + TSCloseSessionReq req = new TSCloseSessionReq(sessionHandle); + try { + client.closeSession(req); + } catch (TException e) { + throw new IoTDBSessionException("Error occurs when closing session at server. Maybe server is down.", e); + } finally { + isClosed = true; + if (transport != null) { + transport.close(); + } + } + } + + public TSExecuteBatchStatementResp insertBatch(RowBatch rowBatch) throws IoTDBSessionException { + TSBatchInsertionReq request = new TSBatchInsertionReq(); + request.deviceId = rowBatch.deviceId; + for (MeasurementSchema measurementSchema: rowBatch.measurements) { + request.addToMeasurements(measurementSchema.getMeasurementId()); + request.addToTypes(measurementSchema.getType().ordinal()); + } + request.setTimestamps(SessionUtils.getTimeBuffer(rowBatch)); + request.setValues(SessionUtils.getValueBuffer(rowBatch)); + request.setSize(rowBatch.batchSize); + + try { + return client.insertBatch(request); + } catch (TException e) { + throw new IoTDBSessionException(e); + } + } + + public TSRPCResp setStorageGroup(String storageGroupId) throws IoTDBSessionException { + TSSetStorageGroupReq request = new TSSetStorageGroupReq(); + request.setStorageGroupId(storageGroupId); + + try { + return client.setStorageGroup(request); + } catch (TException e) { + throw new IoTDBSessionException(e); + } + } + + public TSRPCResp createTimeseries(String path, TSDataType dataType, TSEncoding encoding) throws IoTDBSessionException { + TSCreateTimeseriesReq request = new TSCreateTimeseriesReq(); + request.setPath(path); + request.setDataType(dataType.ordinal()); + request.setEncoding(encoding.ordinal()); + + try { + return client.createTimeseries(request); + } catch (TException e) { + throw new IoTDBSessionException(e); + } + } + + public String getTimeZone() throws TException, IoTDBRPCException { + if (zoneId != null) { + return zoneId.toString(); + } + + TSGetTimeZoneResp resp = client.getTimeZone(); + RpcUtils.verifySuccess(resp.getStatus()); + return resp.getTimeZone(); + } + + public void setTimeZone(String zoneId) throws TException, IoTDBRPCException { + TSSetTimeZoneReq req = new TSSetTimeZoneReq(zoneId); + TSRPCResp resp = client.setTimeZone(req); + RpcUtils.verifySuccess(resp.getStatus()); + this.zoneId = ZoneId.of(zoneId); + } + +} diff --git a/session/src/main/java/org/apache/iotdb/session/SessionUtils.java b/session/src/main/java/org/apache/iotdb/session/SessionUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..77fd93baf1574b90f9da53d7110c5d7b7ea735b0 --- /dev/null +++ b/session/src/main/java/org/apache/iotdb/session/SessionUtils.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.session; + +import java.nio.ByteBuffer; +import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.BytesUtils; +import org.apache.iotdb.tsfile.write.record.RowBatch; + +public class SessionUtils { + + public static ByteBuffer getTimeBuffer(RowBatch rowBatch) { + ByteBuffer timeBuffer = ByteBuffer.allocate(rowBatch.getTimeBytesSize()); + for (int i = 0; i < rowBatch.batchSize; i++) { + timeBuffer.putLong(rowBatch.timestamps[i]); + } + timeBuffer.flip(); + return timeBuffer; + } + + public static ByteBuffer getValueBuffer(RowBatch rowBatch) { + ByteBuffer valueBuffer = ByteBuffer.allocate(rowBatch.getValueBytesSize()); + for (int i = 0; i < rowBatch.measurements.size(); i++) { + TSDataType dataType = rowBatch.measurements.get(i).getType(); + switch (dataType) { + case INT32: + int[] intValues = (int[]) rowBatch.values[i]; + for (int index = 0; index < rowBatch.batchSize; index++) { + valueBuffer.putInt(intValues[index]); + } + break; + case INT64: + long[] longValues = (long[]) rowBatch.values[i]; + for (int index = 0; index < rowBatch.batchSize; index++) { + valueBuffer.putLong(longValues[index]); + } + break; + case FLOAT: + float[] floatValues = (float[]) rowBatch.values[i]; + for (int index = 0; index < rowBatch.batchSize; index++) { + valueBuffer.putFloat(floatValues[index]); + } + break; + case DOUBLE: + double[] doubleValues = (double[]) rowBatch.values[i]; + for (int index = 0; index < rowBatch.batchSize; index++) { + valueBuffer.putDouble(doubleValues[index]); + } + break; + case BOOLEAN: + boolean[] boolValues = (boolean[]) rowBatch.values[i]; + for (int index = 0; index < rowBatch.batchSize; index++) { + valueBuffer.put(BytesUtils.boolToByte(boolValues[index])); + } + break; + case TEXT: + Binary[] binaryValues = (Binary[]) rowBatch.values[i]; + for (int index = 0; index < rowBatch.batchSize; index++) { + valueBuffer.putInt(binaryValues[index].getLength()); + valueBuffer.put(binaryValues[index].getValues()); + } + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", dataType)); + } + } + valueBuffer.flip(); + return valueBuffer; + } + +} diff --git a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileOutputFormat.java b/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileOutputFormat.java index c925561dc98042ea1ea5a3bc3b712d1e9760bd89..528b2398fddc683725dea8cb0846b699e777f129 100644 --- a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileOutputFormat.java +++ b/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileOutputFormat.java @@ -25,21 +25,21 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; public class TsFileOutputFormat extends FileOutputFormat { - private FileSchema fileSchema; + private Schema schema; - public TsFileOutputFormat(FileSchema fileSchema) { - this.fileSchema = fileSchema; + public TsFileOutputFormat(Schema schema) { + this.schema = schema; } @Override public RecordWriter getRecordWriter(TaskAttemptContext job) throws IOException { Path path = getDefaultWorkFile(job, ""); - return new TsFileRecordWriter(job, path, fileSchema); + return new TsFileRecordWriter(job, path, schema); } } diff --git a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileRecordWriter.java b/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileRecordWriter.java index 7e579191b44c1d097869af5a69f060703d31b91f..b9806c23b40eccf48750292467e9f5b3dee1bd46 100644 --- a/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileRecordWriter.java +++ b/spark-tsfile/src/main/java/org/apache/iotdb/tsfile/io/TsFileRecordWriter.java @@ -26,18 +26,18 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.write.TsFileWriter; import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; public class TsFileRecordWriter extends RecordWriter { private TsFileWriter tsFileWriter = null; - public TsFileRecordWriter(TaskAttemptContext job, Path file, FileSchema fileSchema) + public TsFileRecordWriter(TaskAttemptContext job, Path file, Schema schema) throws IOException { HDFSOutput hdfsOutput = new HDFSOutput(file.toString(), job.getConfiguration(), false); //NOTE overwrite false here - tsFileWriter = new TsFileWriter(hdfsOutput, fileSchema); + tsFileWriter = new TsFileWriter(hdfsOutput, schema); } @Override diff --git a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/Converter.scala b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/Converter.scala index 4ad3c663d3c21ee9a8d43ba2c0538ee150a3acfa..d5b73734fa6837556d4ae5127a63faffd5d5033a 100755 --- a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/Converter.scala +++ b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/Converter.scala @@ -35,7 +35,7 @@ import org.apache.iotdb.tsfile.read.filter.{TimeFilter, ValueFilter} import org.apache.iotdb.tsfile.utils.Binary import org.apache.iotdb.tsfile.write.record.TSRecord import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint -import org.apache.iotdb.tsfile.write.schema.{FileSchema, MeasurementSchema, SchemaBuilder} +import org.apache.iotdb.tsfile.write.schema.{Schema, MeasurementSchema, SchemaBuilder} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ @@ -280,7 +280,7 @@ object Converter { * @param structType given sql schema * @return TsFile schema */ - def toTsFileSchema(structType: StructType, options: Map[String, String]): FileSchema = { + def toTsFileSchema(structType: StructType, options: Map[String, String]): Schema = { val schemaBuilder = new SchemaBuilder() structType.fields.filter(f => { !QueryConstant.RESERVED_TIME.equals(f.name) diff --git a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/NewConverter.scala b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/NewConverter.scala index 77f0b5800db563e4139d20978a70b3447209e13b..472b74fcda9f8f16433de80f2964adf636e403b1 100644 --- a/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/NewConverter.scala +++ b/spark-tsfile/src/main/scala/org/apache/iotdb/tsfile/NewConverter.scala @@ -37,7 +37,7 @@ import org.apache.iotdb.tsfile.read.filter.{TimeFilter, ValueFilter} import org.apache.iotdb.tsfile.utils.Binary import org.apache.iotdb.tsfile.write.record.TSRecord import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint -import org.apache.iotdb.tsfile.write.schema.{FileSchema, MeasurementSchema, SchemaBuilder} +import org.apache.iotdb.tsfile.write.schema.{MeasurementSchema, SchemaBuilder} import org.apache.parquet.filter2.predicate.Operators.NotEq import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.sources._ @@ -237,24 +237,6 @@ object NewConverter { new MeasurementSchema(measurement, dataType, encoding) } - /** - * Given a SparkSQL struct type, generate the TsFile schema. - * Note: Measurements of the same name should have the same schema. - * - * @param structType given sql schema - * @return TsFile schema - */ - def toTsFileSchema(structType: StructType, options: Map[String, String]): FileSchema = { - val schemaBuilder = new SchemaBuilder() - structType.fields.filter(f => { - !QueryConstant.RESERVED_TIME.equals(f.name) - }).foreach(f => { - val seriesSchema = getSeriesSchema(f, options) - schemaBuilder.addSeries(seriesSchema) - }) - schemaBuilder.build() - } - /** * Convert a row in the spark table to a list of TSRecord. * diff --git a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java index 5f6da88b1abaea516fcd948ad5597e3c0acdc830..9aab7f87c19187c6d155e36981abb7acf8f37c21 100644 --- a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java +++ b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java @@ -64,14 +64,14 @@ public class HDFSInputTest { @Test public void test_read1() throws IOException { - int size = 2000; + int size = 1000; ByteBuffer buffer = ByteBuffer.allocate(size); Assert.assertEquals(size, in.read(buffer)); } @Test public void test_read2() throws IOException { - int size = 2000; + int size = 1000; long pos = 20L; ByteBuffer buffer = ByteBuffer.allocate(size); Assert.assertEquals(size, in.read(buffer, pos)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java index 25c0bc8dd6bebdcf1b1af2ed0897edc96f0e7d10..647b610303dca6b9dc6032249b2be947c7f01ecf 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java @@ -135,9 +135,9 @@ public class TSFileConfig { */ public static int pageCheckSizeThreshold = 100; /** - * Default endian value is LITTLE_ENDIAN. + * Default endian value is BIG_ENDIAN. */ - public static String endian = "LITTLE_ENDIAN"; + public static String endian = "BIG_ENDIAN"; /** * only can be used by TsFileDescriptor. diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java index eadb5d385bf14b4c70b3dcc36adbb684c95664ca..5c01f24f5badd365f62915fd3b7f35c275aa6d71 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tsfile.encoding.common; /** - * In current verison, we only support LITTLE_ENDIAN mode. + * In current version, we only support BIG_ENDIAN mode. * * @author xuyi */ diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java index 180bc9db2d9ae2f54b29b61706a43ddc9b3d93fe..c6e3032541fee03c9624b1359185a14125956e7a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java @@ -54,14 +54,14 @@ public abstract class Decoder { public static Decoder getDecoderByType(TSEncoding type, TSDataType dataType) { // PLA and DFT encoding are not supported in current version if (type == TSEncoding.PLAIN) { - return new PlainDecoder(EndianType.LITTLE_ENDIAN); + return new PlainDecoder(EndianType.BIG_ENDIAN); } else if (type == TSEncoding.RLE) { switch (dataType) { case BOOLEAN: case INT32: - return new IntRleDecoder(EndianType.LITTLE_ENDIAN); + return new IntRleDecoder(EndianType.BIG_ENDIAN); case INT64: - return new LongRleDecoder(EndianType.LITTLE_ENDIAN); + return new LongRleDecoder(EndianType.BIG_ENDIAN); case FLOAT: case DOUBLE: return new FloatDecoder(TSEncoding.valueOf(type.toString()), dataType); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java index b4ed46f390099b4fa4be182ae8f2d14e39dda6b8..d88d8f38321e5e3fbbb96a9075e1d61e71e64dc2 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java @@ -228,6 +228,7 @@ public abstract class DeltaBinaryDecoder extends Decoder { @Override public long readLong(ByteBuffer buffer) { + return readT(buffer); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java index 7f9efbb3417983a2ba5b849e1b2387b748b4f32e..17b498f2f3714071e818f4195ee0c0f5151cac2d 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java @@ -54,10 +54,10 @@ public class FloatDecoder extends Decoder { super(encodingType); if (encodingType == TSEncoding.RLE) { if (dataType == TSDataType.FLOAT) { - decoder = new IntRleDecoder(EndianType.LITTLE_ENDIAN); + decoder = new IntRleDecoder(EndianType.BIG_ENDIAN); logger.debug("tsfile-encoding FloatDecoder: init decoder using int-rle and float"); } else if (dataType == TSDataType.DOUBLE) { - decoder = new LongRleDecoder(EndianType.LITTLE_ENDIAN); + decoder = new LongRleDecoder(EndianType.BIG_ENDIAN); logger.debug("tsfile-encoding FloatDecoder: init decoder using long-rle and double"); } else { throw new TsFileDecodingException( diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java index 67a0e4abc68a390d4ce2b60ab05552b9c1585d35..4985955de9bc65bea56ad66e5985135da51b8825 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java @@ -26,7 +26,6 @@ import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Binary; -import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,60 +52,32 @@ public class PlainDecoder extends Decoder { @Override public boolean readBoolean(ByteBuffer buffer) { - int ch1 = ReadWriteIOUtils.read(buffer); - return ch1 != 0; + return buffer.get() != 0; } @Override public short readShort(ByteBuffer buffer) { - int ch1 = ReadWriteIOUtils.read(buffer); - int ch2 = ReadWriteIOUtils.read(buffer); - if (this.endianType == EndianType.LITTLE_ENDIAN) { - return (short) ((ch2 << 8) + ch1); - } else { - logger.error( - "tsfile-encoding PlainEncoder: current version does not support short value decoding"); - } - return -1; + return buffer.getShort(); } @Override public int readInt(ByteBuffer buffer) { - int ch1 = ReadWriteIOUtils.read(buffer); - int ch2 = ReadWriteIOUtils.read(buffer); - int ch3 = ReadWriteIOUtils.read(buffer); - int ch4 = ReadWriteIOUtils.read(buffer); - if (this.endianType == EndianType.LITTLE_ENDIAN) { - return ch1 + (ch2 << 8) + (ch3 << 16) + (ch4 << 24); - } else { - logger.error( - "tsfile-encoding PlainEncoder: current version does not support int value encoding"); - } - return -1; + return buffer.getInt(); } @Override public long readLong(ByteBuffer buffer) { - int[] buf = new int[8]; - for (int i = 0; i < 8; i++) { - buf[i] = ReadWriteIOUtils.read(buffer); - } - - Long res = 0L; - for (int i = 0; i < 8; i++) { - res += ((long) buf[i] << (i * 8)); - } - return res; + return buffer.getLong(); } @Override public float readFloat(ByteBuffer buffer) { - return Float.intBitsToFloat(readInt(buffer)); + return buffer.getFloat(); } @Override public double readDouble(ByteBuffer buffer) { - return Double.longBitsToDouble(readLong(buffer)); + return buffer.getDouble(); } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java index bdf4d13fc19b99530727e8d4a53476ae992fbdb0..01b8e2771d610c59d283579fa2cb9846561f38f3 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/DeltaBinaryEncoder.java @@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.BytesUtils; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,8 +86,8 @@ public abstract class DeltaBinaryEncoder extends Encoder { } private void writeHeaderToBytes() throws IOException { - out.write(BytesUtils.intToBytes(writeIndex)); - out.write(BytesUtils.intToBytes(writeWidth)); + ReadWriteIOUtils.write(writeIndex, out); + ReadWriteIOUtils.write(writeWidth, out); writeHeader(); } @@ -208,8 +209,8 @@ public abstract class DeltaBinaryEncoder extends Encoder { @Override protected void writeHeader() throws IOException { - out.write(BytesUtils.intToBytes(minDeltaBase)); - out.write(BytesUtils.intToBytes(firstValue)); + ReadWriteIOUtils.write(minDeltaBase, out); + ReadWriteIOUtils.write(firstValue, out); } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java index 3998f3d746cb39b91a645e5752532906ac65fdc5..7568a09c534c57ecbedbb6d4a2c485a2fb821455 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java @@ -64,9 +64,9 @@ public class FloatEncoder extends Encoder { isMaxPointNumberSaved = false; if (encodingType == TSEncoding.RLE) { if (dataType == TSDataType.FLOAT) { - encoder = new IntRleEncoder(EndianType.LITTLE_ENDIAN); + encoder = new IntRleEncoder(EndianType.BIG_ENDIAN); } else if (dataType == TSDataType.DOUBLE) { - encoder = new LongRleEncoder(EndianType.LITTLE_ENDIAN); + encoder = new LongRleEncoder(EndianType.BIG_ENDIAN); } else { throw new TsFileEncodingException( String.format("data type %s is not supported by FloatEncoder", dataType)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java index dae5f83bf90f304630532cf887828edb84ec609d..93761a3e2c1d6336ba0e07231f3ef365dca74a95 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java @@ -68,10 +68,8 @@ public class PlainEncoder extends Encoder { out.write(value & 0xFF); out.write((value >> 8) & 0xFF); } else if (this.endianType == EndianType.BIG_ENDIAN) { - logger.error( - "tsfile-encoding PlainEncoder: current version does not support short value encoding"); - throw new TsFileEncodingException( - "tsfile-encoding PlainEncoder: current version does not support short value encoding"); + out.write((value >> 8) & 0xFF); + out.write(value & 0xFF); } } @@ -83,34 +81,23 @@ public class PlainEncoder extends Encoder { out.write((value >> 16) & 0xFF); out.write((value >> 24) & 0xFF); } else if (this.endianType == EndianType.BIG_ENDIAN) { - logger.error( - "tsfile-encoding PlainEncoder: current version does not support int value encoding"); - throw new TsFileEncodingException( - "tsfile-encoding PlainEncoder: current version does not support int value encoding"); + out.write((value >> 24) & 0xFF); + out.write((value >> 16) & 0xFF); + out.write((value >> 8) & 0xFF); + out.write(value & 0xFF); } } @Override public void encode(long value, ByteArrayOutputStream out) { - byte[] bufferBig = new byte[8]; - byte[] bufferLittle = new byte[8]; - - for (int i = 0; i < 8; i++) { - bufferLittle[i] = (byte) (((value) >> (i * 8)) & 0xFF); - bufferBig[8 - i - 1] = (byte) (((value) >> (i * 8)) & 0xFF); - } - try { - if (this.endianType == EndianType.LITTLE_ENDIAN) { - out.write(bufferLittle); - } else if (this.endianType == EndianType.BIG_ENDIAN) { - logger.error( - "tsfile-encoding PlainEncoder: current version does not support long value encoding"); - throw new TsFileEncodingException( - "tsfile-encoding PlainEncoder: current version does not support long value encoding"); + if (this.endianType == EndianType.LITTLE_ENDIAN) { + for (int i = 0; i < 8; i++) { + out.write((byte) (((value) >> (i * 8)) & 0xFF)); + } + } else if (this.endianType == EndianType.BIG_ENDIAN) { + for (int i = 7; i >= 0; i--) { + out.write((byte) (((value) >> (i * 8)) & 0xFF)); } - } catch (IOException e) { - logger - .error("tsfile-encoding PlainEncoder: error occurs when encode long value {}", value, e); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java index ba1e32ef4bccfed2ecb54d3988f39956e6b0bbb6..5f4acba801042b287122c15aa52428a2e7574cf6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -101,7 +101,7 @@ public abstract class TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - return new PlainEncoder(EndianType.LITTLE_ENDIAN, type, maxStringLength); + return new PlainEncoder(EndianType.BIG_ENDIAN, type, maxStringLength); } @Override @@ -133,9 +133,9 @@ public abstract class TSEncodingBuilder { switch (type) { case INT32: case BOOLEAN: - return new IntRleEncoder(EndianType.LITTLE_ENDIAN); + return new IntRleEncoder(EndianType.BIG_ENDIAN); case INT64: - return new LongRleEncoder(EndianType.LITTLE_ENDIAN); + return new LongRleEncoder(EndianType.BIG_ENDIAN); case FLOAT: case DOUBLE: return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/write/TsFileNotCompleteException.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/write/TsFileNotCompleteException.java new file mode 100644 index 0000000000000000000000000000000000000000..aa38fd901fb1242aeab2e94c352829fd6b859b6c --- /dev/null +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/write/TsFileNotCompleteException.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tsfile.exception.write; + +import java.io.IOException; + +public class TsFileNotCompleteException extends IOException { + + public TsFileNotCompleteException() { + } + + public TsFileNotCompleteException(String message) { + super(message); + } + + public TsFileNotCompleteException(String message, Throwable cause) { + super(message, cause); + } + + public TsFileNotCompleteException(Throwable cause) { + super(cause); + } +} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java index 01cc0abacd4726399c2dd10f31188baf2576ba55..dc17641c2a0a1525ed3acb8dda81a9883582ae23 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaData.java @@ -173,7 +173,7 @@ public class ChunkGroupMetaData { } public List getChunkMetaDataList() { - return chunkMetaDataList == null ? null : Collections.unmodifiableList(chunkMetaDataList); + return chunkMetaDataList; } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java index 7c89cf2b690c855e8dff87130c1d337048f15ff3..1fe974ee7aa27d059b5e7b33052242b98c50aa36 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaData.java @@ -56,7 +56,7 @@ public class ChunkMetaData { /** * All data with timestamp <= deletedAt are considered deleted. */ - private long deletedAt = -1; + private long deletedAt = Long.MIN_VALUE; private TsDigest valuesStatistics; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDigest.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDigest.java index 1b27efaf527a45fafc908c7d127f6c8268b8b524..0fa252bee69d3e17e0819df8c95cbdfe7e2769fc 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDigest.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDigest.java @@ -23,10 +23,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; +import java.util.Arrays; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; /** @@ -34,11 +31,15 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class TsDigest { - private Map statistics; + private ByteBuffer[] statistics; - private int serializedSize = Integer.BYTES; + /** + * size of valid values in statistics. Note that some values in statistics can be null and thus + * invalid. + */ + private int validSizeOfArray = 0; - private int sizeOfList; + private int serializedSize = Integer.BYTES; // initialize for number of statistics public TsDigest() { // allowed to clair an empty TsDigest whose fields will be assigned later. @@ -66,17 +67,18 @@ public class TsDigest { TsDigest digest = new TsDigest(); int size = ReadWriteIOUtils.readInt(inputStream); + digest.validSizeOfArray = size; + digest.serializedSize = Integer.BYTES; if (size > 0) { - Map statistics = new HashMap<>(); - String key; + digest.statistics = new ByteBuffer[StatisticType.getTotalTypeNum()]; ByteBuffer value; for (int i = 0; i < size; i++) { - key = ReadWriteIOUtils.readString(inputStream); + short n = ReadWriteIOUtils.readShort(inputStream); value = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(inputStream); - statistics.put(key, value); + digest.statistics[n] = value; + digest.serializedSize += Short.BYTES + Integer.BYTES + value.remaining(); } - digest.statistics = statistics; - } + } // else left digest.statistics as null return digest; } @@ -91,69 +93,56 @@ public class TsDigest { TsDigest digest = new TsDigest(); int size = ReadWriteIOUtils.readInt(buffer); + digest.validSizeOfArray = size; + digest.serializedSize = Integer.BYTES; if (size > 0) { - Map statistics = new HashMap<>(); - String key; + digest.statistics = new ByteBuffer[StatisticType.getTotalTypeNum()]; ByteBuffer value; for (int i = 0; i < size; i++) { - key = ReadWriteIOUtils.readString(buffer); + short n = ReadWriteIOUtils.readShort(buffer); value = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(buffer); - statistics.put(key, value); + digest.statistics[n] = value; + digest.serializedSize += Short.BYTES + Integer.BYTES + value.remaining(); } - digest.statistics = statistics; - } + } // else left digest.statistics as null return digest; } - private void reCalculateSerializedSize() { + private void reCalculate() { + validSizeOfArray = 0; serializedSize = Integer.BYTES; if (statistics != null) { - for (Map.Entry entry : statistics.entrySet()) { - serializedSize += Integer.BYTES + entry.getKey().length() + Integer.BYTES - + entry.getValue().remaining(); + for (ByteBuffer value : statistics) { + if (value != null) { + // StatisticType serialized value, byteBuffer.capacity and byteBuffer.array + serializedSize += Short.BYTES + Integer.BYTES + value.remaining(); + validSizeOfArray++; + } } - sizeOfList = statistics.size(); - } else { - sizeOfList = 0; } } /** * get statistics of the current object. - * - * @return -unmodifiableMap of the current object's statistics */ - public Map getStatistics() { - if (statistics == null) { - return null; - } - return Collections.unmodifiableMap(this.statistics); + public ByteBuffer[] getStatistics() { + return statistics; //TODO unmodifiable } - public void setStatistics(Map statistics) { - this.statistics = statistics; - reCalculateSerializedSize(); - } - - /** - * add statistics using given param. - * - * @param key -key of the entry - * @param value -value of the entry - */ - public void addStatistics(String key, ByteBuffer value) { - if (statistics == null) { - statistics = new HashMap<>(); + public void setStatistics(ByteBuffer[] statistics) throws IOException { + if (statistics != null && statistics.length != StatisticType.getTotalTypeNum()) { + throw new IOException(String.format( + "The length of array of statistics doesn't equal StatisticType.getTotalTypeNum() %d", + StatisticType.getTotalTypeNum())); } - statistics.put(key, value); - serializedSize += Integer.BYTES + key.length() + Integer.BYTES + value.remaining(); - sizeOfList++; + this.statistics = statistics; + reCalculate(); // DO NOT REMOVE THIS } @Override public String toString() { - return statistics != null ? statistics.toString() : ""; + return statistics != null ? Arrays.toString(statistics) : ""; } /** @@ -163,20 +152,16 @@ public class TsDigest { * @return -byte length */ public int serializeTo(OutputStream outputStream) throws IOException { - if ((statistics != null && sizeOfList != statistics.size()) || (statistics == null - && sizeOfList != 0)) { - reCalculateSerializedSize(); - } int byteLen = 0; - if (statistics == null || statistics.size() == 0) { + if (validSizeOfArray == 0) { byteLen += ReadWriteIOUtils.write(0, outputStream); } else { - byteLen += ReadWriteIOUtils.write(statistics.size(), outputStream); - for (Map.Entry entry : statistics.entrySet()) { - byteLen += ReadWriteIOUtils - .write(entry.getKey(), outputStream); - byteLen += ReadWriteIOUtils - .write(entry.getValue(), outputStream); + byteLen += ReadWriteIOUtils.write(validSizeOfArray, outputStream); + for (int i = 0; i < statistics.length; i++) { + if (statistics[i] != null) { + byteLen += ReadWriteIOUtils.write((short) i, outputStream); + byteLen += ReadWriteIOUtils.write(statistics[i], outputStream); + } } } return byteLen; @@ -189,20 +174,16 @@ public class TsDigest { * @return -byte length */ public int serializeTo(ByteBuffer buffer) { - if ((statistics != null && sizeOfList != statistics.size()) || (statistics == null - && sizeOfList != 0)) { - reCalculateSerializedSize(); - } int byteLen = 0; - - if (statistics == null || statistics.size() == 0) { + if (validSizeOfArray == 0) { byteLen += ReadWriteIOUtils.write(0, buffer); } else { - byteLen += ReadWriteIOUtils.write(statistics.size(), buffer); - for (Map.Entry entry : statistics.entrySet()) { - byteLen += ReadWriteIOUtils.write(entry.getKey(), buffer); - byteLen += ReadWriteIOUtils - .write(entry.getValue(), buffer); + byteLen += ReadWriteIOUtils.write(validSizeOfArray, buffer); + for (int i = 0; i < statistics.length; i++) { + if (statistics[i] != null) { + byteLen += ReadWriteIOUtils.write((short) i, buffer); + byteLen += ReadWriteIOUtils.write(statistics[i], buffer); + } } } return byteLen; @@ -214,9 +195,6 @@ public class TsDigest { * @return -serializedSize */ public int getSerializedSize() { - if (statistics == null || (sizeOfList != statistics.size())) { - reCalculateSerializedSize(); - } return serializedSize; } @@ -229,17 +207,44 @@ public class TsDigest { return false; } TsDigest digest = (TsDigest) o; - if (serializedSize != digest.serializedSize || sizeOfList != digest.sizeOfList - || statistics.size() != digest.statistics.size()) { + if (serializedSize != digest.serializedSize || validSizeOfArray != digest.validSizeOfArray + || ((statistics == null) ^ (digest.statistics == null))) { return false; } - for (Entry entry : statistics.entrySet()) { - String key = entry.getKey(); - ByteBuffer value = entry.getValue(); - if (!digest.statistics.containsKey(key) || !value.equals(digest.statistics.get(key))) { - return false; + + if (statistics != null) { + for (int i = 0; i < statistics.length; i++) { + if ((statistics[i] == null) ^ (digest.statistics[i] == null)) { + // one is null and the other is not null + return false; + } + if (statistics[i] != null) { + if (!statistics[i].equals(digest.statistics[i])) { + return false; + } + } } } return true; } + + public enum StatisticType { + min_value, max_value, first_value, last_value, sum_value; + + public static int getTotalTypeNum() { + return StatisticType.values().length; + } + + public static StatisticType deserialize(short i) { + return StatisticType.values()[i]; + } + + public static int getSerializedSize() { + return Short.BYTES; + } + + public short serialize() { + return (short) this.ordinal(); + } + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetaData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetaData.java index c7a57e3766f14fceb2a51f3487afa90145ac4b53..5fa8450b7ccb88de38bbb23ceed7737141f78725 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetaData.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetaData.java @@ -29,8 +29,8 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; -import java.util.Map; import java.util.List; +import java.util.Map; /** * TSFileMetaData collects all metadata info and saves in its data structure. @@ -38,20 +38,29 @@ import java.util.List; public class TsFileMetaData { private Map deviceIndexMap = new HashMap<>(); + /** * TSFile schema for this file. This schema contains metadata for all the measurements. */ private Map measurementSchema = new HashMap<>(); + /** * Version of this file. */ private int currentVersion; + /** * String for application that wrote this file. This should be in the format [Application] version * [App Version](build [App Build Hash]). e.g. impala version 1.0 (build SHA-1_hash_code) */ private String createdBy; + // fields below are IoTDB extensions and they does not affect TsFile's stand-alone functionality + private int totalChunkNum; + // invalid means a chunk has been rewritten by merge and the chunk's data is in + // another new chunk + private int invalidChunkNum; + public TsFileMetaData() { //do nothing } @@ -108,6 +117,8 @@ public class TsFileMetaData { if (ReadWriteIOUtils.readIsNull(inputStream)) { fileMetaData.createdBy = ReadWriteIOUtils.readString(inputStream); } + fileMetaData.totalChunkNum = ReadWriteIOUtils.readInt(inputStream); + fileMetaData.invalidChunkNum = ReadWriteIOUtils.readInt(inputStream); return fileMetaData; } @@ -151,6 +162,8 @@ public class TsFileMetaData { if (ReadWriteIOUtils.readIsNull(buffer)) { fileMetaData.createdBy = ReadWriteIOUtils.readString(buffer); } + fileMetaData.totalChunkNum = ReadWriteIOUtils.readInt(buffer); + fileMetaData.invalidChunkNum = ReadWriteIOUtils.readInt(buffer); return fileMetaData; } @@ -253,6 +266,9 @@ public class TsFileMetaData { byteLen += ReadWriteIOUtils.write(createdBy, outputStream); } + byteLen += ReadWriteIOUtils.write(totalChunkNum, outputStream); + byteLen += ReadWriteIOUtils.write(invalidChunkNum, outputStream); + return byteLen; } @@ -284,10 +300,29 @@ public class TsFileMetaData { byteLen += ReadWriteIOUtils.write(createdBy, buffer); } + byteLen += ReadWriteIOUtils.write(totalChunkNum, buffer); + byteLen += ReadWriteIOUtils.write(invalidChunkNum, buffer); + return byteLen; } - public List getMeasurementSchemaList(){ + public int getTotalChunkNum() { + return totalChunkNum; + } + + public void setTotalChunkNum(int totalChunkNum) { + this.totalChunkNum = totalChunkNum; + } + + public int getInvalidChunkNum() { + return invalidChunkNum; + } + + public void setInvalidChunkNum(int invalidChunkNum) { + this.invalidChunkNum = invalidChunkNum; + } + + public List getMeasurementSchemaList() { return new ArrayList(measurementSchema.values()); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java index 2a57fc256b272d7560cbae67cbf13c66151f71c6..54fd32cb14b7f16ebc40d03cbbd73bb192fabd4f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java @@ -33,16 +33,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class BinaryStatistics extends Statistics { - private Binary max = new Binary(""); private Binary min = new Binary(""); + private Binary max = new Binary(""); private Binary first = new Binary(""); - private double sum;// FIXME sum is meaningless private Binary last = new Binary(""); + private double sum;// FIXME sum is meaningless @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = new Binary(maxBytes); min = new Binary(minBytes); + max = new Binary(maxBytes); } @Override @@ -61,13 +61,13 @@ public class BinaryStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Binary getLast() { + return last; } @Override - public Binary getLast() { - return last; + public double getSum() { + return sum; } /** @@ -76,15 +76,15 @@ public class BinaryStatistics extends Statistics { * @param min minimum value * @param max maximum value * @param first the first value - * @param sum sum * @param last the last value + * @param sum sum */ - public void initializeStats(Binary min, Binary max, Binary first, double sum, Binary last) { + private void initializeStats(Binary min, Binary max, Binary first, Binary last, double sum) { this.min = min; this.max = max; this.first = first; - this.sum = sum; this.last = last; + this.sum = sum; } @Override @@ -92,23 +92,21 @@ public class BinaryStatistics extends Statistics { BinaryStatistics stringStats = (BinaryStatistics) stats; if (isEmpty) { initializeStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(), - stringStats.getSum(), - stringStats.getLast()); + stringStats.getLast(), stringStats.getSum()); isEmpty = false; } else { updateStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(), - stringStats.getSum(), - stringStats.getLast()); + stringStats.getLast(), stringStats.getSum()); } } @Override public void updateStats(Binary value) { if (isEmpty) { - initializeStats(value, value, value, 0, value); + initializeStats(value, value, value, value, 0); isEmpty = false; } else { - updateStats(value, value, value, 0, value); + updateStats(value, value, value, value, 0); isEmpty = false; } } @@ -117,17 +115,17 @@ public class BinaryStatistics extends Statistics { public void updateStats(Binary[] values) { for (Binary value : values) { if (isEmpty) { - initializeStats(value, value, value, 0, value); + initializeStats(value, value, value, value, 0); isEmpty = false; } else { - updateStats(value, value, value, 0, value); + updateStats(value, value, value, value, 0); isEmpty = false; } } } - private void updateStats(Binary minValue, Binary maxValue, Binary firstValue, double sum, - Binary lastValue) { + private void updateStats(Binary minValue, Binary maxValue, Binary firstValue, Binary lastValue, + double sum) { if (minValue.compareTo(min) < 0) { min = minValue; } @@ -137,24 +135,19 @@ public class BinaryStatistics extends Statistics { this.last = lastValue; } - @Override - public byte[] getMaxBytes() { - return max.getValues(); - } - @Override public byte[] getMinBytes() { return min.getValues(); } @Override - public byte[] getFirstBytes() { - return first.getValues(); + public byte[] getMaxBytes() { + return max.getValues(); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return first.getValues(); } @Override @@ -163,8 +156,8 @@ public class BinaryStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ByteBuffer.wrap(max.getValues()); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -173,13 +166,13 @@ public class BinaryStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ByteBuffer.wrap(first.getValues()); + public ByteBuffer getMaxBytebuffer() { + return ByteBuffer.wrap(max.getValues()); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ByteBuffer.wrap(first.getValues()); } @Override @@ -187,6 +180,11 @@ public class BinaryStatistics extends Statistics { return ByteBuffer.wrap(last.getValues()); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override public int sizeOfDatum() { return -1; @@ -194,7 +192,7 @@ public class BinaryStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + "]"; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java index ff04992bfbc9d442d6cc161c614f58dea9cc1b7d..7c2a7e8ce4ff28c01a9f293ae12e6a217b7b29a7 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java @@ -31,25 +31,25 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class BooleanStatistics extends Statistics { - private boolean max; private boolean min; + private boolean max; private boolean first; - private double sum; private boolean last; + private double sum; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToBool(maxBytes); min = BytesUtils.bytesToBool(minBytes); + max = BytesUtils.bytesToBool(maxBytes); } @Override public void updateStats(boolean value) { if (isEmpty) { - initializeStats(value, value, value, 0, value); + initializeStats(value, value, value, value, 0); isEmpty = false; } else { - updateStats(value, value, value, 0, value); + updateStats(value, value, value, value, 0); isEmpty = false; } } @@ -58,17 +58,17 @@ public class BooleanStatistics extends Statistics { public void updateStats(boolean[] values) { for (boolean value : values) { if (isEmpty) { - initializeStats(value, value, value, 0, value); + initializeStats(value, value, value, value, 0); isEmpty = false; } else { - updateStats(value, value, value, 0, value); + updateStats(value, value, value, value, 0); isEmpty = false; } } } - private void updateStats(boolean minValue, boolean maxValue, boolean firstValue, double sumValue, - boolean lastValue) { + private void updateStats(boolean minValue, boolean maxValue, boolean firstValue, + boolean lastValue, double sumValue) { if (!minValue && min) { min = false; } @@ -78,24 +78,19 @@ public class BooleanStatistics extends Statistics { this.last = lastValue; } - @Override - public Boolean getMax() { - return max; - } - @Override public Boolean getMin() { return min; } @Override - public Boolean getFirst() { - return first; + public Boolean getMax() { + return max; } @Override - public double getSum() { - return sum; + public Boolean getFirst() { + return first; } @Override @@ -104,8 +99,8 @@ public class BooleanStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + public double getSum() { + return sum; } @Override @@ -114,13 +109,13 @@ public class BooleanStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -128,17 +123,21 @@ public class BooleanStatistics extends Statistics { return ReadWriteIOUtils.getByteBuffer(last); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override protected void mergeStatisticsValue(Statistics stats) { BooleanStatistics boolStats = (BooleanStatistics) stats; if (isEmpty) { initializeStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), - boolStats.getSum(), - boolStats.getLast()); + boolStats.getLast(), boolStats.getSum()); isEmpty = false; } else { - updateStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), boolStats.getSum(), - boolStats.getLast()); + updateStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), + boolStats.getLast(), boolStats.getSum()); } } @@ -148,11 +147,11 @@ public class BooleanStatistics extends Statistics { * @param min min boolean * @param max max boolean * @param firstValue first boolean value - * @param sumValue sum value (double type) * @param lastValue last boolean value + * @param sumValue sum value (double type) */ - public void initializeStats(boolean min, boolean max, boolean firstValue, double sumValue, - boolean lastValue) { + private void initializeStats(boolean min, boolean max, boolean firstValue, boolean lastValue, + double sumValue) { this.min = min; this.max = max; this.first = firstValue; @@ -160,13 +159,13 @@ public class BooleanStatistics extends Statistics { } @Override - public byte[] getMaxBytes() { - return BytesUtils.boolToBytes(max); + public byte[] getMinBytes() { + return BytesUtils.boolToBytes(min); } @Override - public byte[] getMinBytes() { - return BytesUtils.boolToBytes(min); + public byte[] getMaxBytes() { + return BytesUtils.boolToBytes(max); } @Override @@ -175,13 +174,13 @@ public class BooleanStatistics extends Statistics { } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getLastBytes() { + return BytesUtils.boolToBytes(last); } @Override - public byte[] getLastBytes() { - return BytesUtils.boolToBytes(last); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -191,7 +190,7 @@ public class BooleanStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + "]"; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java index 57d500e1d54624e94e946bc979faf3fe8d6e095a..e0e2998342a9788668750a196a30cb988facf4e5 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java @@ -31,16 +31,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class DoubleStatistics extends Statistics { - private double max; private double min; + private double max; private double first; - private double sum; private double last; + private double sum; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToDouble(maxBytes); min = BytesUtils.bytesToDouble(minBytes); + max = BytesUtils.bytesToDouble(maxBytes); } @Override @@ -65,8 +65,8 @@ public class DoubleStatistics extends Statistics { } } - private void updateStats(double minValue, double maxValue, double firstValue, double sumValue, - double lastValue) { + private void updateStats(double minValue, double maxValue, double firstValue, double lastValue, + double sumValue) { if (minValue < min) { min = minValue; } @@ -78,13 +78,13 @@ public class DoubleStatistics extends Statistics { } @Override - public Double getMax() { - return max; + public Double getMin() { + return min; } @Override - public Double getMin() { - return min; + public Double getMax() { + return max; } @Override @@ -93,13 +93,13 @@ public class DoubleStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Double getLast() { + return last; } @Override - public Double getLast() { - return last; + public double getSum() { + return sum; } @Override @@ -107,13 +107,11 @@ public class DoubleStatistics extends Statistics { DoubleStatistics doubleStats = (DoubleStatistics) stats; if (this.isEmpty) { initializeStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(), - doubleStats.getSum(), - doubleStats.getLast()); + doubleStats.getLast(), doubleStats.getSum()); isEmpty = false; } else { updateStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(), - doubleStats.getSum(), - doubleStats.getLast()); + doubleStats.getLast(), doubleStats.getSum()); } } @@ -124,20 +122,15 @@ public class DoubleStatistics extends Statistics { * @param min min value * @param max max value * @param first the first value - * @param sum sum value * @param last the last value + * @param sum sum value */ - public void initializeStats(double min, double max, double first, double sum, double last) { + private void initializeStats(double min, double max, double first, double last, double sum) { this.min = min; this.max = max; this.first = first; - this.sum = sum; this.last = last; - } - - @Override - public byte[] getMaxBytes() { - return BytesUtils.doubleToBytes(max); + this.sum = sum; } @Override @@ -146,13 +139,13 @@ public class DoubleStatistics extends Statistics { } @Override - public byte[] getFirstBytes() { - return BytesUtils.doubleToBytes(first); + public byte[] getMaxBytes() { + return BytesUtils.doubleToBytes(max); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return BytesUtils.doubleToBytes(first); } @Override @@ -161,8 +154,8 @@ public class DoubleStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -171,13 +164,13 @@ public class DoubleStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -185,6 +178,11 @@ public class DoubleStatistics extends Statistics { return ReadWriteIOUtils.getByteBuffer(last); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override public int sizeOfDatum() { return 8; @@ -192,7 +190,7 @@ public class DoubleStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + "]"; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java index 2fa13f6ebc292b766ba013fa6ea7cfd698a972e4..b995d6d6c5780b557f7a11a8514c3b00bdf3cf3d 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java @@ -31,16 +31,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class FloatStatistics extends Statistics { - private float max; private float min; + private float max; private float first; private double sum; private float last; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToFloat(maxBytes); min = BytesUtils.bytesToFloat(minBytes); + max = BytesUtils.bytesToFloat(maxBytes); } @Override @@ -65,8 +65,8 @@ public class FloatStatistics extends Statistics { } } - private void updateStats(float minValue, float maxValue, float firstValue, - double sumValue, float last) { + private void updateStats(float minValue, float maxValue, float firstValue, float last, + double sumValue) { if (minValue < min) { min = minValue; } @@ -78,13 +78,13 @@ public class FloatStatistics extends Statistics { } @Override - public Float getMax() { - return max; + public Float getMin() { + return min; } @Override - public Float getMin() { - return min; + public Float getMax() { + return max; } @Override @@ -93,13 +93,13 @@ public class FloatStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Float getLast() { + return last; } @Override - public Float getLast() { - return last; + public double getSum() { + return sum; } @Override @@ -107,28 +107,21 @@ public class FloatStatistics extends Statistics { FloatStatistics floatStats = (FloatStatistics) stats; if (isEmpty) { initializeStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(), - floatStats.getSum(), - floatStats.getLast()); + floatStats.getLast(), floatStats.getSum()); isEmpty = false; } else { updateStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(), - floatStats.getSum(), - floatStats.getLast()); + floatStats.getLast(), floatStats.getSum()); } } - public void initializeStats(float min, float max, float first, double sum, float last) { + private void initializeStats(float min, float max, float first, float last, double sum) { this.min = min; this.max = max; this.first = first; - this.sum = sum; this.last = last; - } - - @Override - public byte[] getMaxBytes() { - return BytesUtils.floatToBytes(max); + this.sum = sum; } @Override @@ -137,13 +130,13 @@ public class FloatStatistics extends Statistics { } @Override - public byte[] getFirstBytes() { - return BytesUtils.floatToBytes(first); + public byte[] getMaxBytes() { + return BytesUtils.floatToBytes(max); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return BytesUtils.floatToBytes(first); } @Override @@ -152,8 +145,8 @@ public class FloatStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -162,13 +155,13 @@ public class FloatStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -176,6 +169,11 @@ public class FloatStatistics extends Statistics { return ReadWriteIOUtils.getByteBuffer(last); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override public int sizeOfDatum() { return 4; @@ -183,8 +181,8 @@ public class FloatStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" - + first + ",sum:" + sum + ",last:" + last + "]"; + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + + "]"; } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java index 9726e3c44a75f8dc59e07d8d4d2aa5a9085439cc..bb81e677b13b0375171288b9641007cdc572479a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java @@ -31,16 +31,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class IntegerStatistics extends Statistics { - private int max; private int min; + private int max; private int first; - private double sum; private int last; + private double sum; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToInt(maxBytes); min = BytesUtils.bytesToInt(minBytes); + max = BytesUtils.bytesToInt(maxBytes); } @Override @@ -67,8 +67,8 @@ public class IntegerStatistics extends Statistics { } } - private void updateStats(int minValue, int maxValue, - int firstValue, double sumValue, int lastValue) { + private void updateStats(int minValue, int maxValue, int firstValue, int lastValue, + double sumValue) { // TODO: unused parameter if (minValue < min) { min = minValue; @@ -81,13 +81,13 @@ public class IntegerStatistics extends Statistics { } @Override - public Integer getMax() { - return max; + public Integer getMin() { + return min; } @Override - public Integer getMin() { - return min; + public Integer getMax() { + return max; } @Override @@ -96,40 +96,35 @@ public class IntegerStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Integer getLast() { + return last; } @Override - public Integer getLast() { - return last; + public double getSum() { + return sum; } @Override protected void mergeStatisticsValue(Statistics stats) { IntegerStatistics intStats = (IntegerStatistics) stats; if (isEmpty) { - initializeStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getSum(), - intStats.getLast()); + initializeStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getLast(), + intStats.getSum()); isEmpty = false; } else { - updateStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getSum(), - intStats.getLast()); + updateStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getLast(), + intStats.getSum()); } } - void initializeStats(int min, int max, int first, double sum, int last) { + private void initializeStats(int min, int max, int first, int last, double sum) { this.min = min; this.max = max; this.first = first; - this.sum = sum; this.last = last; - } - - @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + this.sum = sum; } @Override @@ -138,13 +133,13 @@ public class IntegerStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -153,8 +148,8 @@ public class IntegerStatistics extends Statistics { } @Override - public byte[] getMaxBytes() { - return BytesUtils.intToBytes(max); + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); } @Override @@ -163,13 +158,13 @@ public class IntegerStatistics extends Statistics { } @Override - public byte[] getFirstBytes() { - return BytesUtils.intToBytes(first); + public byte[] getMaxBytes() { + return BytesUtils.intToBytes(max); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return BytesUtils.intToBytes(first); } @Override @@ -177,6 +172,11 @@ public class IntegerStatistics extends Statistics { return BytesUtils.intToBytes(last); } + @Override + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); + } + @Override public int sizeOfDatum() { return 4; @@ -184,8 +184,8 @@ public class IntegerStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" - + first + ",sum:" + sum + ",last:" + last + "]"; + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + + "]"; } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java index 1c9ba6474a027173cc66fd434c8c26e051ebc126..d8e401f78e9e217eb33d1d03e168303fd39f950a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java @@ -31,16 +31,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class LongStatistics extends Statistics { - private long max; private long min; + private long max; private long first; - private double sum; private long last; + private double sum; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToLong(maxBytes); min = BytesUtils.bytesToLong(minBytes); + max = BytesUtils.bytesToLong(maxBytes); } @Override @@ -59,13 +59,13 @@ public class LongStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Long getLast() { + return last; } @Override - public Long getLast() { - return last; + public double getSum() { + return sum; } @Override @@ -90,8 +90,8 @@ public class LongStatistics extends Statistics { } } - private void updateStats(long minValue, long maxValue, long firstValue, double sumValue, - long lastValue) { + private void updateStats(long minValue, long maxValue, long firstValue, long lastValue, + double sumValue) { if (minValue < min) { min = minValue; } @@ -117,27 +117,21 @@ public class LongStatistics extends Statistics { LongStatistics longStats = (LongStatistics) stats; if (isEmpty) { initializeStats(longStats.getMin(), longStats.getMax(), longStats.getFirst(), - longStats.getSum(), - longStats.getLast()); + longStats.getLast(), longStats.getSum()); isEmpty = false; } else { - updateStats(longStats.getMin(), longStats.getMax(), longStats.getFirst(), longStats.getSum(), - longStats.getLast()); + updateStats(longStats.getMin(), longStats.getMax(), longStats.getFirst(), longStats.getLast(), + longStats.getSum()); } } - void initializeStats(long min, long max, long firstValue, double sum, long last) { + private void initializeStats(long min, long max, long firstValue, long last, double sum) { this.min = min; this.max = max; this.first = firstValue; - this.sum += sum; this.last = last; - } - - @Override - public byte[] getMaxBytes() { - return BytesUtils.longToBytes(max); + this.sum += sum; } @Override @@ -146,13 +140,13 @@ public class LongStatistics extends Statistics { } @Override - public byte[] getFirstBytes() { - return BytesUtils.longToBytes(first); + public byte[] getMaxBytes() { + return BytesUtils.longToBytes(max); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return BytesUtils.longToBytes(first); } @Override @@ -161,8 +155,8 @@ public class LongStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -171,13 +165,13 @@ public class LongStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -185,9 +179,14 @@ public class LongStatistics extends Statistics { return ReadWriteIOUtils.getByteBuffer(last); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + "]"; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java index 353fbf7dfa18cd22caa4f6df48969f1f191a8995..fb2d3e8cbca861eb784465f14264b17293bb07a1 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java @@ -32,7 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * This class is used for recording statistic information of each measurement in a delta file.While + * This class is used for recording statistic information of each measurement in a delta file. While * writing processing, the processor records the digest information. Statistics includes maximum, * minimum and null value count up to version 0.0.1.
Each data type extends this Statistic as * super class.
@@ -105,30 +105,30 @@ public abstract class Statistics { public abstract T getFirst(); - public abstract double getSum(); - public abstract T getLast(); - public abstract byte[] getMaxBytes(); + public abstract double getSum(); public abstract byte[] getMinBytes(); - public abstract byte[] getFirstBytes(); + public abstract byte[] getMaxBytes(); - public abstract byte[] getSumBytes(); + public abstract byte[] getFirstBytes(); public abstract byte[] getLastBytes(); - public abstract ByteBuffer getMaxBytebuffer(); + public abstract byte[] getSumBytes(); public abstract ByteBuffer getMinBytebuffer(); - public abstract ByteBuffer getFirstBytebuffer(); + public abstract ByteBuffer getMaxBytebuffer(); - public abstract ByteBuffer getSumBytebuffer(); + public abstract ByteBuffer getFirstBytebuffer(); public abstract ByteBuffer getLastBytebuffer(); + public abstract ByteBuffer getSumBytebuffer(); + /** * merge parameter to this statistic. Including * @@ -262,7 +262,7 @@ public abstract class Statistics { } else if (sizeOfDatum() != -1) { return sizeOfDatum() * 4 + 8; } else { - return 4 * Integer.BYTES + getMaxBytes().length + getMinBytes().length + return 4 * Integer.BYTES + getMinBytes().length + getMaxBytes().length + getFirstBytes().length + getLastBytes().length + getSumBytes().length; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java index 4f6617bf151d31f572180898d36e94e1a72a154a..624ea49fa7b3e1493cc8e721994fe30c767f6a7e 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/ReadOnlyTsFile.java @@ -19,9 +19,9 @@ package org.apache.iotdb.tsfile.read; import java.io.IOException; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; -import org.apache.iotdb.tsfile.read.controller.MetadataQuerier; +import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.expression.QueryExpression; import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; @@ -30,8 +30,8 @@ import org.apache.iotdb.tsfile.read.query.executor.TsFileExecutor; public class ReadOnlyTsFile implements AutoCloseable { private TsFileSequenceReader fileReader; - private MetadataQuerier metadataQuerier; - private ChunkLoader chunkLoader; + private IMetadataQuerier metadataQuerier; + private IChunkLoader chunkLoader; private TsFileExecutor tsFileExecutor; /** diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index 8359d8be4340b6fdc6a0119e134d1bbda0e61bb3..819c7b61c145d5c97ba4a7efa3f2ac9c83312764 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -19,17 +19,18 @@ package org.apache.iotdb.tsfile.read; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.compress.IUnCompressor; import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.header.PageHeader; import org.apache.iotdb.tsfile.file.metadata.*; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.reader.DefaultTsFileInput; import org.apache.iotdb.tsfile.read.reader.TsFileInput; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -40,7 +41,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -54,6 +54,11 @@ public class TsFileSequenceReader implements AutoCloseable { private long fileMetadataPos; private int fileMetadataSize; private ByteBuffer markerBuffer = ByteBuffer.allocate(Byte.BYTES); + private int totalChunkNum; + private TsFileMetaData tsFileMetaData; + + private boolean cacheDeviceMetadata = false; + private Map deviceMetadataMap; /** * Create a file reader of the given file. The reader will read the tail of the file to get the @@ -75,7 +80,7 @@ public class TsFileSequenceReader implements AutoCloseable { */ public TsFileSequenceReader(String file, boolean loadMetadataSize) throws IOException { this.file = file; - final Path path = Paths.get(file); + final java.nio.file.Path path = Paths.get(file); tsFileInput = new DefaultTsFileInput(path); try { if (loadMetadataSize) { @@ -87,6 +92,14 @@ public class TsFileSequenceReader implements AutoCloseable { } } + public TsFileSequenceReader(String file, boolean loadMetadata, boolean cacheDeviceMetadata) throws IOException { + this(file, loadMetadata); + this.cacheDeviceMetadata = cacheDeviceMetadata; + if (cacheDeviceMetadata) { + deviceMetadataMap = new HashMap<>(); + } + } + /** * Create a file reader of the given file. The reader will read the tail of the file to get the * file metadata size.Then the reader will skip the first TSFileConfig.MAGIC_STRING.length() bytes @@ -202,7 +215,10 @@ public class TsFileSequenceReader implements AutoCloseable { * this function does not modify the position of the file reader. */ public TsFileMetaData readFileMetadata() throws IOException { - return TsFileMetaData.deserializeFrom(readData(fileMetadataPos, fileMetadataSize)); + if (tsFileMetaData == null) { + tsFileMetaData = TsFileMetaData.deserializeFrom(readData(fileMetadataPos, fileMetadataSize)); + } + return tsFileMetaData; } /** @@ -225,7 +241,18 @@ public class TsFileSequenceReader implements AutoCloseable { * this function does not modify the position of the file reader. */ public TsDeviceMetadata readTsDeviceMetaData(TsDeviceMetadataIndex index) throws IOException { - return TsDeviceMetadata.deserializeFrom(readData(index.getOffset(), index.getLen())); + TsDeviceMetadata deviceMetadata = null; + if (cacheDeviceMetadata) { + deviceMetadata = deviceMetadataMap.get(index); + } + if (deviceMetadata == null) { + deviceMetadata = TsDeviceMetadata.deserializeFrom(readData(index.getOffset() + , index.getLen())); + if (cacheDeviceMetadata) { + deviceMetadataMap.put(index, deviceMetadata); + } + } + return deviceMetadata; } /** @@ -326,7 +353,7 @@ public class TsFileSequenceReader implements AutoCloseable { ChunkHeader header = readChunkHeader(metaData.getOffsetOfChunkHeader(), false); ByteBuffer buffer = readChunk(metaData.getOffsetOfChunkHeader() + header.getSerializedSize(), header.getDataSize()); - return new Chunk(header, buffer); + return new Chunk(header, buffer, metaData.getDeletedAt()); } /** @@ -409,6 +436,7 @@ public class TsFileSequenceReader implements AutoCloseable { public void close() throws IOException { this.tsFileInput.close(); + deviceMetadataMap = null; } public String getFileName() { @@ -510,6 +538,7 @@ public class TsFileSequenceReader implements AutoCloseable { long truncatedPosition = magicStringBytes.length; boolean goon = true; byte marker; + int chunkCnt = 0; try { while (goon && (marker = this.readMarker()) != MetaMarker.SEPARATOR) { switch (marker) { @@ -558,17 +587,23 @@ public class TsFileSequenceReader implements AutoCloseable { currentChunk = new ChunkMetaData(measurementID, dataType, fileOffsetOfChunk, startTimeOfChunk, endTimeOfChunk); currentChunk.setNumOfPoints(numOfPoints); - Map statisticsMap = new HashMap<>(); - statisticsMap.put(StatisticConstant.MAX_VALUE, ByteBuffer.wrap(chunkStatistics.getMaxBytes())); - statisticsMap.put(StatisticConstant.MIN_VALUE, ByteBuffer.wrap(chunkStatistics.getMinBytes())); - statisticsMap.put(StatisticConstant.FIRST, ByteBuffer.wrap(chunkStatistics.getFirstBytes())); - statisticsMap.put(StatisticConstant.SUM, ByteBuffer.wrap(chunkStatistics.getSumBytes())); - statisticsMap.put(StatisticConstant.LAST, ByteBuffer.wrap(chunkStatistics.getLastBytes())); + ByteBuffer[] statisticsArray = new ByteBuffer[StatisticType.getTotalTypeNum()]; + statisticsArray[StatisticType.min_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getMinBytes()); + statisticsArray[StatisticType.max_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getMaxBytes()); + statisticsArray[StatisticType.first_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getFirstBytes()); + statisticsArray[StatisticType.last_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getLastBytes()); + statisticsArray[StatisticType.sum_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getSumBytes()); TsDigest tsDigest = new TsDigest(); - tsDigest.setStatistics(statisticsMap); + tsDigest.setStatistics(statisticsArray); currentChunk.setDigest(tsDigest); chunks.add(currentChunk); numOfPoints = 0; + chunkCnt++; break; case MetaMarker.CHUNK_GROUP_FOOTER: //this is a chunk group @@ -583,6 +618,9 @@ public class TsFileSequenceReader implements AutoCloseable { newMetaData.add(currentChunkGroup); newChunkGroup = true; truncatedPosition = this.position(); + + totalChunkNum += chunkCnt; + chunkCnt = 0; break; default: // the disk file is corrupted, using this file may be dangerous @@ -603,33 +641,68 @@ public class TsFileSequenceReader implements AutoCloseable { return truncatedPosition; } + public int getTotalChunkNum() { + return totalChunkNum; + } + + public List getChunkMetadataList(Path path) throws IOException { + if (tsFileMetaData == null) { + readFileMetadata(); + } + if (!tsFileMetaData.containsDevice(path.getDevice())) { + return new ArrayList<>(); + } + + // get the index information of TsDeviceMetadata + TsDeviceMetadataIndex index = tsFileMetaData.getDeviceMetadataIndex(path.getDevice()); + + // read TsDeviceMetadata from file + TsDeviceMetadata tsDeviceMetadata = readTsDeviceMetaData(index); + + // get all ChunkMetaData of this path included in all ChunkGroups of this device + List chunkMetaDataList = new ArrayList<>(); + for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) { + List chunkMetaDataListInOneChunkGroup = chunkGroupMetaData + .getChunkMetaDataList(); + for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) { + if (path.getMeasurement().equals(chunkMetaData.getMeasurementUid())) { + chunkMetaData.setVersion(chunkGroupMetaData.getVersion()); + chunkMetaDataList.add(chunkMetaData); + } + } + } + chunkMetaDataList.sort(Comparator.comparingLong(ChunkMetaData::getStartTime)); + return chunkMetaDataList; + } + /** - * get device name in range - * @param start start of the range - * @param end end of the range - * @return device name in range + * get device names in range + * @param start start of the file + * @param end end of the file + * @return device names in range */ - public List getDeviceNameInRange(long start, long end){ + public List getDeviceNameInRange(long start, long end) { List res = new ArrayList<>(); + try { TsFileMetaData tsFileMetaData = readFileMetadata(); - for (Map.Entry deviceIndex : tsFileMetaData.getDeviceMap().entrySet()) { - TsDeviceMetadata tsDeviceMetadata = readTsDeviceMetaData(deviceIndex.getValue()); + for (Map.Entry entry: tsFileMetaData.getDeviceMap().entrySet()) { + TsDeviceMetadata tsDeviceMetadata = readTsDeviceMetaData(entry.getValue()); for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata .getChunkGroupMetaDataList()) { LocateStatus mode = checkLocateStatus(chunkGroupMetaData, start, end); if (mode == LocateStatus.in) { - res.add(deviceIndex.getKey()); + res.add(entry.getKey()); break; } } } - } - catch (IOException e){ + } catch (IOException e) { e.printStackTrace(); } + return res; } @@ -641,8 +714,8 @@ public class TsFileSequenceReader implements AutoCloseable { * @param spacePartitionEndPos the end position of the space partition * @return LocateStatus */ - - private LocateStatus checkLocateStatus(ChunkGroupMetaData chunkGroupMetaData, long spacePartitionStartPos, long spacePartitionEndPos) { + private LocateStatus checkLocateStatus(ChunkGroupMetaData chunkGroupMetaData, + long spacePartitionStartPos, long spacePartitionEndPos) { long startOffsetOfChunkGroup = chunkGroupMetaData.getStartOffsetOfChunkGroup(); long endOffsetOfChunkGroup = chunkGroupMetaData.getEndOffsetOfChunkGroup(); long middleOffsetOfChunkGroup = (startOffsetOfChunkGroup + endOffsetOfChunkGroup) / 2; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java index 962192ff4b32ae9b81d29598013da80b2e86aa58..d9d0641985b5eabeab0c604614771468ea8d2522 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java @@ -97,6 +97,7 @@ public class BatchData implements Serializable { init(type, recordTime, hasEmptyTime); } + // FIXME: this means hasCurrent actually public boolean hasNext() { return curIdx < timeLength; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java index f4ad12533b29d049169a4c337c3fc7bec995c6ea..289fd46bb1a87c42cccfaacbdabf15a1b4dee2b0 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Chunk.java @@ -28,11 +28,15 @@ public class Chunk { private ChunkHeader chunkHeader; private ByteBuffer chunkData; - private long deletedAt = -1; + /** + * All data with timestamp <= deletedAt are considered deleted. + */ + private long deletedAt; - public Chunk(ChunkHeader header, ByteBuffer buffer) { + public Chunk(ChunkHeader header, ByteBuffer buffer, long deletedAt) { this.chunkHeader = header; this.chunkData = buffer; + this.deletedAt = deletedAt; } public ChunkHeader getHeader() { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Path.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Path.java index 9f6c52899254d39f67c37109f31f851bcd3a35fc..0600be2b9ed0e085e738876629f0503adf0874c4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Path.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/Path.java @@ -185,7 +185,7 @@ public class Path implements Serializable { @Override public boolean equals(Object obj) { - return obj != null && obj instanceof Path && this.fullPath.equals(((Path) obj).fullPath); + return obj instanceof Path && this.fullPath.equals(((Path) obj).fullPath); } public boolean equals(String obj) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java index 4907634e5481214e4a3bf8096816791e592ac389..50ea1c5785b68c4e4dbd0c7d181a54e1bb589e2b 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoaderImpl.java @@ -27,7 +27,7 @@ import org.apache.iotdb.tsfile.read.common.Chunk; /** * Read one Chunk and cache it into a LRUCache. */ -public class ChunkLoaderImpl implements ChunkLoader { +public class ChunkLoaderImpl implements IChunkLoader { private static final int DEFAULT_CHUNK_CACHE_SIZE = 100000; private TsFileSequenceReader reader; @@ -59,9 +59,7 @@ public class ChunkLoaderImpl implements ChunkLoader { @Override public Chunk getChunk(ChunkMetaData chunkMetaData) throws IOException { Chunk chunk = chunkCache.get(chunkMetaData); - Chunk chunkRet = new Chunk(chunk.getHeader(), chunk.getData().duplicate()); - chunkRet.setDeletedAt(chunkMetaData.getDeletedAt()); - return chunkRet; + return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeletedAt()); } @Override @@ -69,4 +67,8 @@ public class ChunkLoaderImpl implements ChunkLoader { reader.close(); } + @Override + public void clear() { + chunkCache.clear(); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkLoader.java similarity index 92% rename from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoader.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkLoader.java index 78e561e86e2eec6520c1dfcbb24a9e6fcc1a3821..bcbb0832fbc058fde90b810c8df915ca7b4529e4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/ChunkLoader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkLoader.java @@ -22,7 +22,7 @@ import java.io.IOException; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.common.Chunk; -public interface ChunkLoader { +public interface IChunkLoader { /** * read all content of any chunk. @@ -34,4 +34,8 @@ public interface ChunkLoader { */ void close() throws IOException; + /** + * clear Chunk cache if used. + */ + void clear(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerier.java similarity index 95% rename from tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerier.java index 2202bb65042055e1154232fff8ad257f6673f699..22405f8e8e8c947c44cbae377deac419af40c4ed 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerier.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerier.java @@ -28,7 +28,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.common.TimeRange; -public interface MetadataQuerier { +public interface IMetadataQuerier { List getChunkMetaDataList(Path path) throws IOException; @@ -62,4 +62,9 @@ public interface MetadataQuerier { */ List convertSpace2TimePartition(List paths, long spacePartitionStartPos, long spacePartitionEndPos) throws IOException; + + /** + * clear caches (if used) to release memory. + */ + void clear(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImpl.java index 93294d638c27b43bbb44cef8828f93b9858b6c73..ff86bc2bf062f11870256bc14a0976a248bd3550 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImpl.java @@ -18,29 +18,19 @@ */ package org.apache.iotdb.tsfile.read.controller; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; import org.apache.iotdb.tsfile.common.cache.LRUCache; import org.apache.iotdb.tsfile.exception.write.NoMeasurementException; -import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData; -import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; -import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; -import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; -import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; +import org.apache.iotdb.tsfile.file.metadata.*; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; -public class MetadataQuerierByFileImpl implements MetadataQuerier { +import java.io.IOException; +import java.util.*; + +public class MetadataQuerierByFileImpl implements IMetadataQuerier { private static final int CHUNK_METADATA_CACHE_SIZE = 100000; @@ -172,30 +162,7 @@ public class MetadataQuerierByFileImpl implements MetadataQuerier { } private List loadChunkMetadata(Path path) throws IOException { - - if (!fileMetaData.containsDevice(path.getDevice())) { - return new ArrayList<>(); - } - - // get the index information of TsDeviceMetadata - TsDeviceMetadataIndex index = fileMetaData.getDeviceMetadataIndex(path.getDevice()); - - // read TsDeviceMetadata from file - TsDeviceMetadata tsDeviceMetadata = tsFileReader.readTsDeviceMetaData(index); - - // get all ChunkMetaData of this path included in all ChunkGroups of this device - List chunkMetaDataList = new ArrayList<>(); - for (ChunkGroupMetaData chunkGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) { - List chunkMetaDataListInOneChunkGroup = chunkGroupMetaData - .getChunkMetaDataList(); - for (ChunkMetaData chunkMetaData : chunkMetaDataListInOneChunkGroup) { - if (path.getMeasurement().equals(chunkMetaData.getMeasurementUid())) { - chunkMetaData.setVersion(chunkGroupMetaData.getVersion()); - chunkMetaDataList.add(chunkMetaData); - } - } - } - return chunkMetaDataList; + return tsFileReader.getChunkMetadataList(path); } @Override @@ -302,4 +269,11 @@ public class MetadataQuerierByFileImpl implements MetadataQuerier { private enum LocateStatus { in, before, after } + + @Override + public void clear() { + chunkMetaDataCache.clear(); + } + + } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java index e3784afdc23156df464ab36f264e94d1592d8cae..9f8806393535c685e7f108e129bdd708d8db8203 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java @@ -100,10 +100,24 @@ public class DigestForFilter { return maxTime; } + public boolean isMinValueNull() { + return minValue == null; + } + + public boolean isMaxValueNull() { + return maxValue == null; + } + + /** + * Note check isMinValueNull before its usage + */ public > T getMinValue() { return getValue(minValue); } + /** + * Note check isMaxValueNull before its usage + */ public > T getMaxValue() { return getValue(maxValue); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java index 6c3b587fd641cdc8ee6297c8e601151e0cd653ff..7af77061d6e2acecb80eaf09c0d6b6de20823d45 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java @@ -41,6 +41,9 @@ public class Eq> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) >= digest.getMinTime() && ((Long) value) <= digest.getMaxTime(); } else { + if (digest.isMinValueNull() || digest.isMaxValueNull()) { + return true; + } return value.compareTo(digest.getMinValue()) >= 0 && value.compareTo(digest.getMaxValue()) <= 0; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java index 0b12713e66ba53fbd1c0f08849341b09eb5f4277..eadc8ffc8b659015cc5308b1b0477ce20fbebf5c 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java @@ -41,6 +41,9 @@ public class Gt> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) < digest.getMaxTime(); } else { + if (digest.isMaxValueNull()) { + return true; + } return value.compareTo(digest.getMaxValue()) < 0; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java index 5d1cb6d42f7729cfd1725366a8d173314e5e6dc1..20f3e19b3f0fd2801d776a9717f07f6353ef394b 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java @@ -41,6 +41,9 @@ public class GtEq> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) <= digest.getMaxTime(); } else { + if (digest.isMaxValueNull()) { + return true; + } return value.compareTo(digest.getMaxValue()) <= 0; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java index 3385241e7b14583aa9368f0996ce26bbbc8404a3..e7449c7223bf99f4efab4ed1b836ca5a078543aa 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java @@ -41,6 +41,9 @@ public class Lt> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) > digest.getMinTime(); } else { + if (digest.isMinValueNull()) { + return true; + } return value.compareTo(digest.getMinValue()) > 0; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java index b1cef193cf2a1b8f47ee80ac1cbbf7bbd8a4922d..1076b63531d3da54b15f21c7903082be701613c6 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java @@ -41,6 +41,9 @@ public class LtEq> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) >= digest.getMinTime(); } else { + if (digest.isMinValueNull()) { + return true; + } return value.compareTo(digest.getMinValue()) >= 0; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java index 0b92639f9cdc9b2a2846611280f3571adc7f7640..634388e03a7e24d0e3159185b3961f5d8b12ff24 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java @@ -41,6 +41,9 @@ public class NotEq> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return !(((Long) value) == digest.getMinTime() && (Long) value == digest.getMaxTime()); } else { + if (digest.isMinValueNull() || digest.isMaxValueNull()) { + return true; + } return !(value.compareTo(digest.getMinValue()) == 0 && value.compareTo(digest.getMaxValue()) == 0); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/ExecutorWithTimeGenerator.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/ExecutorWithTimeGenerator.java index 5bd689298ef134723dc59c570522819b56626b53..16e574ed948f43746cd55bff733b3c49720e3a28 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/ExecutorWithTimeGenerator.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/ExecutorWithTimeGenerator.java @@ -26,8 +26,8 @@ import java.util.List; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; -import org.apache.iotdb.tsfile.read.controller.MetadataQuerier; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier; import org.apache.iotdb.tsfile.read.expression.IExpression; import org.apache.iotdb.tsfile.read.expression.QueryExpression; import org.apache.iotdb.tsfile.read.expression.impl.BinaryExpression; @@ -39,10 +39,10 @@ import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderByTimestamp; public class ExecutorWithTimeGenerator implements QueryExecutor { - private MetadataQuerier metadataQuerier; - private ChunkLoader chunkLoader; + private IMetadataQuerier metadataQuerier; + private IChunkLoader chunkLoader; - public ExecutorWithTimeGenerator(MetadataQuerier metadataQuerier, ChunkLoader chunkLoader) { + public ExecutorWithTimeGenerator(IMetadataQuerier metadataQuerier, IChunkLoader chunkLoader) { this.metadataQuerier = metadataQuerier; this.chunkLoader = chunkLoader; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java index 13aa278a907ae9cb33b1f34fdb37760b53d901eb..72657546d889ee4e77b73caad4c83f26320af535 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java @@ -28,8 +28,8 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.common.TimeRange; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; -import org.apache.iotdb.tsfile.read.controller.MetadataQuerier; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier; import org.apache.iotdb.tsfile.read.expression.IExpression; import org.apache.iotdb.tsfile.read.expression.QueryExpression; import org.apache.iotdb.tsfile.read.expression.impl.BinaryExpression; @@ -44,10 +44,10 @@ import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithoutFilter; public class TsFileExecutor implements QueryExecutor { - private MetadataQuerier metadataQuerier; - private ChunkLoader chunkLoader; + private IMetadataQuerier metadataQuerier; + private IChunkLoader chunkLoader; - public TsFileExecutor(MetadataQuerier metadataQuerier, ChunkLoader chunkLoader) { + public TsFileExecutor(IMetadataQuerier metadataQuerier, IChunkLoader chunkLoader) { this.metadataQuerier = metadataQuerier; this.chunkLoader = chunkLoader; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorImpl.java index c17600110fad6152846d290a6aa60df15f11191d..906c09f9fbb1d4782b5010f2f1296d219a112469 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorImpl.java @@ -25,8 +25,8 @@ import java.util.List; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; -import org.apache.iotdb.tsfile.read.controller.MetadataQuerier; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IMetadataQuerier; import org.apache.iotdb.tsfile.read.expression.ExpressionType; import org.apache.iotdb.tsfile.read.expression.IBinaryExpression; import org.apache.iotdb.tsfile.read.expression.IExpression; @@ -40,8 +40,8 @@ import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithFilter; public class TimeGeneratorImpl implements TimeGenerator { - private ChunkLoader chunkLoader; - private MetadataQuerier metadataQuerier; + private IChunkLoader chunkLoader; + private IMetadataQuerier metadataQuerier; private Node operatorNode; private HashMap> leafCache; @@ -53,8 +53,8 @@ public class TimeGeneratorImpl implements TimeGenerator { * @param chunkLoader -construct param * @param metadataQuerier -construct param */ - public TimeGeneratorImpl(IExpression iexpression, ChunkLoader chunkLoader, - MetadataQuerier metadataQuerier) + public TimeGeneratorImpl(IExpression iexpression, IChunkLoader chunkLoader, + IMetadataQuerier metadataQuerier) throws IOException { this.chunkLoader = chunkLoader; this.metadataQuerier = metadataQuerier; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java index 4b2d4a4d4058db02675716df2d350d7e3a8ac0d3..f7727280c43169e213c64a6442bdf86937ffc8ca 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java @@ -161,4 +161,7 @@ public abstract class ChunkReader { public void close() { } + public ChunkHeader getChunkHeader() { + return chunkHeader; + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java index a686d997ab92d6cf609f81459d189a324e2496b1..024133fa4fa01a2338493d5bbc101c492e357f45 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java @@ -20,6 +20,7 @@ package org.apache.iotdb.tsfile.read.reader.page; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import org.apache.iotdb.tsfile.encoding.decoder.Decoder; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -48,7 +49,7 @@ public class PageReader { private Filter filter = null; - private long deletedAt = -1; + private long deletedAt = Long.MIN_VALUE; public PageReader(ByteBuffer pageData, TSDataType dataType, Decoder valueDecoder, Decoder timeDecoder, diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java index 110fec32c7ba4984e1a6b14bb4c989a1892807ee..dcbd8f960b2560e761db7d840412b27ba2be0c25 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java @@ -24,7 +24,7 @@ import java.util.List; import org.apache.iotdb.tsfile.file.header.PageHeader; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.common.BatchData; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; /** @@ -32,7 +32,7 @@ import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; */ public abstract class FileSeriesReader { - protected ChunkLoader chunkLoader; + protected IChunkLoader chunkLoader; protected List chunkMetaDataList; protected ChunkReader chunkReader; private int chunkToRead; @@ -42,7 +42,7 @@ public abstract class FileSeriesReader { /** * constructor of FileSeriesReader. */ - public FileSeriesReader(ChunkLoader chunkLoader, List chunkMetaDataList) { + public FileSeriesReader(IChunkLoader chunkLoader, List chunkMetaDataList) { this.chunkLoader = chunkLoader; this.chunkMetaDataList = chunkMetaDataList; this.chunkToRead = 0; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java index 90e31a9a2ed220c3e9cd3aa9b412207c6d41f4e1..0eb6e40f18b45fc878b5b3111760b45b9dec4165 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderByTimestamp.java @@ -24,7 +24,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.BatchData; import org.apache.iotdb.tsfile.read.common.Chunk; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderByTimestamp; @@ -36,7 +36,7 @@ import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderByTimestamp; */ public class FileSeriesReaderByTimestamp { - protected ChunkLoader chunkLoader; + protected IChunkLoader chunkLoader; protected List chunkMetaDataList; private int currentChunkIndex = 0; @@ -47,7 +47,7 @@ public class FileSeriesReaderByTimestamp { /** * init with chunkLoader and chunkMetaDataList. */ - public FileSeriesReaderByTimestamp(ChunkLoader chunkLoader, List chunkMetaDataList) { + public FileSeriesReaderByTimestamp(IChunkLoader chunkLoader, List chunkMetaDataList) { this.chunkLoader = chunkLoader; this.chunkMetaDataList = chunkMetaDataList; currentTimestamp = Long.MIN_VALUE; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java index c874e1788065be8ad9d1f03e05b1d356a9baba93..588e1f509827598d7a2f305472a7fdb618218b90 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java @@ -19,11 +19,12 @@ package org.apache.iotdb.tsfile.read.reader.series; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; -import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; import org.apache.iotdb.tsfile.read.common.Chunk; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.filter.DigestForFilter; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithFilter; @@ -36,7 +37,7 @@ public class FileSeriesReaderWithFilter extends FileSeriesReader { private Filter filter; - public FileSeriesReaderWithFilter(ChunkLoader chunkLoader, + public FileSeriesReaderWithFilter(IChunkLoader chunkLoader, List chunkMetaDataList, Filter filter) { super(chunkLoader, chunkMetaDataList); this.filter = filter; @@ -50,11 +51,16 @@ public class FileSeriesReaderWithFilter extends FileSeriesReader { @Override protected boolean chunkSatisfied(ChunkMetaData chunkMetaData) { + ByteBuffer minValue = null; + ByteBuffer maxValue = null; + ByteBuffer[] statistics = chunkMetaData.getDigest().getStatistics(); + if (statistics != null) { + minValue = statistics[StatisticType.min_value.ordinal()]; // note still CAN be null + maxValue = statistics[StatisticType.max_value.ordinal()]; // note still CAN be null + } + DigestForFilter digest = new DigestForFilter(chunkMetaData.getStartTime(), - chunkMetaData.getEndTime(), - chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MIN_VALUE), - chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MAX_VALUE), - chunkMetaData.getTsDataType()); + chunkMetaData.getEndTime(), minValue, maxValue, chunkMetaData.getTsDataType()); return filter.satisfy(digest); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithoutFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithoutFilter.java index c4efcb8a450c50d2f01d4d0b457638f599d0fdbc..9992f3d3401ac170f26b4929603a16a66d459d7b 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithoutFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithoutFilter.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.util.List; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.read.common.Chunk; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter; /** @@ -30,7 +30,7 @@ import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter; */ public class FileSeriesReaderWithoutFilter extends FileSeriesReader { - public FileSeriesReaderWithoutFilter(ChunkLoader chunkLoader, + public FileSeriesReaderWithoutFilter(IChunkLoader chunkLoader, List chunkMetaDataList) { super(chunkLoader, chunkMetaDataList); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java index 78c635001330f41c3cd1de12984e5d9a90d385ff..990dba90f6b17d59a444941750d748d5547c9804 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/BytesUtils.java @@ -31,8 +31,6 @@ import org.slf4j.LoggerFactory; * integer, long, float, boolean, double and string.
It also provide other usable function as * follow:
reading function which receives InputStream.
concat function to join a list of * byte array to one.
get and set one bit in a byte array. - * - * @author kangrong */ public class BytesUtils { @@ -366,6 +364,18 @@ public class BytesUtils { return b; } + public static byte boolToByte(boolean x) { + if (x) { + return 1; + } else { + return 0; + } + } + + public static boolean byteToBool(byte b) { + return b == 1; + } + /** * boolean convert to byte array, then write four bytes to parameter desc start from * index:offset. diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java index cb47c20977570e53124fb5b13f255c42cfad9567..0129eabe3ee41ead571a3156aacfc5480393808f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java @@ -33,7 +33,7 @@ import org.apache.iotdb.tsfile.write.chunk.IChunkGroupWriter; import org.apache.iotdb.tsfile.write.record.RowBatch; import org.apache.iotdb.tsfile.write.record.TSRecord; import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; import org.apache.iotdb.tsfile.write.writer.TsFileOutput; @@ -53,7 +53,7 @@ public class TsFileWriter implements AutoCloseable{ /** * schema of this TsFile. **/ - protected final FileSchema schema; + protected final Schema schema; /** * IO writer of this TsFile. **/ @@ -78,7 +78,7 @@ public class TsFileWriter implements AutoCloseable{ * @param file the File to be written by this TsFileWriter */ public TsFileWriter(File file) throws IOException { - this(new TsFileIOWriter(file), new FileSchema(), TSFileDescriptor.getInstance().getConfig()); + this(new TsFileIOWriter(file), new Schema(), TSFileDescriptor.getInstance().getConfig()); } /** @@ -87,7 +87,7 @@ public class TsFileWriter implements AutoCloseable{ * @param fileWriter the io writer of this TsFile */ public TsFileWriter(TsFileIOWriter fileWriter) throws IOException { - this(fileWriter, new FileSchema(), TSFileDescriptor.getInstance().getConfig()); + this(fileWriter, new Schema(), TSFileDescriptor.getInstance().getConfig()); } /** @@ -96,7 +96,7 @@ public class TsFileWriter implements AutoCloseable{ * @param file the File to be written by this TsFileWriter * @param schema the schema of this TsFile */ - public TsFileWriter(File file, FileSchema schema) throws IOException { + public TsFileWriter(File file, Schema schema) throws IOException { this(new TsFileIOWriter(file), schema, TSFileDescriptor.getInstance().getConfig()); } @@ -107,7 +107,7 @@ public class TsFileWriter implements AutoCloseable{ * @param schema the schema of this TsFile * @throws IOException */ - public TsFileWriter(TsFileOutput output, FileSchema schema) throws IOException { + public TsFileWriter(TsFileOutput output, Schema schema) throws IOException { this(new TsFileIOWriter(output), schema, TSFileDescriptor.getInstance().getConfig()); } @@ -118,7 +118,7 @@ public class TsFileWriter implements AutoCloseable{ * @param schema the schema of this TsFile * @param conf the configuration of this TsFile */ - public TsFileWriter(File file, FileSchema schema, TSFileConfig conf) throws IOException { + public TsFileWriter(File file, Schema schema, TSFileConfig conf) throws IOException { this(new TsFileIOWriter(file), schema, conf); } @@ -129,7 +129,7 @@ public class TsFileWriter implements AutoCloseable{ * @param schema the schema of this TsFile * @param conf the configuration of this TsFile */ - protected TsFileWriter(TsFileIOWriter fileWriter, FileSchema schema, TSFileConfig conf) + protected TsFileWriter(TsFileIOWriter fileWriter, Schema schema, TSFileConfig conf) throws IOException { if (!fileWriter.canWrite()) { throw new IOException( diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkBuffer.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkBuffer.java index 290c516d60679bcf0bc7458a79aa603d7acd6819..fb07f677a441911dc3e03a3f65199f46d2fff402 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkBuffer.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkBuffer.java @@ -183,7 +183,7 @@ public class ChunkBuffer { schema.getEncodingType(), statistics, maxTimestamp, minTimestamp, pageBuffer.size(), numOfPages); - long totalByteSize = writer.getPos(); + long dataOffset = writer.getPos(); LOG.debug("start writing pages of {} into file, position {}", schema.getMeasurementId(), writer.getPos()); @@ -192,11 +192,14 @@ public class ChunkBuffer { LOG.debug("finish writing pages of {} into file, position {}", schema.getMeasurementId(), writer.getPos()); - long size = writer.getPos() - totalByteSize; - assert size == pageBuffer.size(); + long dataSize = writer.getPos() - dataOffset; + if (dataSize != pageBuffer.size()) { + throw new IOException("Bytes written is inconsistent with the size of data: " + dataSize +" !=" + + " " + pageBuffer.size()); + } writer.endChunk(totalValueCount); - return headerSize + size; + return headerSize + dataSize; } /** @@ -245,4 +248,8 @@ public class ChunkBuffer { public void setSchema(MeasurementSchema schema) { this.schema = schema; } + + public MeasurementSchema getSchema() { + return schema; + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkGroupWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkGroupWriterImpl.java index 9efcff12559553c7c62b97bdd26d9a8ee6a7d226..c1b1e2bc8898af1a67164f2a129480c5e39459d6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkGroupWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkGroupWriterImpl.java @@ -58,7 +58,7 @@ public class ChunkGroupWriterImpl implements IChunkGroupWriter { public void tryToAddSeriesWriter(MeasurementSchema schema, int pageSizeThreshold) { if (!chunkWriters.containsKey(schema.getMeasurementId())) { ChunkBuffer chunkBuffer = new ChunkBuffer(schema); - IChunkWriter seriesWriter = new ChunkWriterImpl(schema, chunkBuffer, pageSizeThreshold); + IChunkWriter seriesWriter = new ChunkWriterImpl(chunkBuffer, pageSizeThreshold); this.chunkWriters.put(schema.getMeasurementId(), seriesWriter); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java index 3ca4d9f583e9cbfd0046b5f4918eabe3c2251da4..6b571bd68a7a389e5e233fd66e3e6fed3cc4b056 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java @@ -83,13 +83,11 @@ public class ChunkWriterImpl implements IChunkWriter { /** * constructor of ChunkWriterImpl. * - * @param measurementSchema measurement schema * @param chunkBuffer chunk in buffer * @param pageSizeThreshold page size threshold */ - public ChunkWriterImpl(MeasurementSchema measurementSchema, ChunkBuffer chunkBuffer, - int pageSizeThreshold) { - this.measurementSchema = measurementSchema; + public ChunkWriterImpl(ChunkBuffer chunkBuffer, int pageSizeThreshold) { + this.measurementSchema = chunkBuffer.getSchema(); this.dataType = measurementSchema.getType(); this.chunkBuffer = chunkBuffer; this.psThres = pageSizeThreshold; @@ -377,4 +375,9 @@ public class ChunkWriterImpl implements IChunkWriter { public ChunkBuffer getChunkBuffer() { return chunkBuffer; } + + @Override + public TSDataType getDataType() { + return dataType; + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java index 616ef7720590c975cf7471420b1dc30a61f351d0..f0463071a9ffb08644d50a2e735b71b000d8f03b 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java @@ -20,6 +20,7 @@ package org.apache.iotdb.tsfile.write.chunk; import java.io.IOException; import java.math.BigDecimal; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; @@ -123,4 +124,6 @@ public interface IChunkWriter { void sealCurrentPage(); int getNumOfPages(); + + TSDataType getDataType(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/RowBatch.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/RowBatch.java index ac20ec854c167c1d0e031f01d6983c5a70d61400..7bcf2346036372534c2422948f4f85e7a92830fe 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/RowBatch.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/RowBatch.java @@ -58,6 +58,11 @@ public class RowBatch { */ private int maxBatchSize; + /** + * total byte size that values occupies + */ + private int valueOccupation = -1; + /** * Return a row batch with default specified row number. * This is the standard constructor (all RowBatch should be the same size). @@ -134,4 +139,46 @@ public class RowBatch { } } } + + public int getTimeBytesSize() { + return batchSize * 8; + } + + /** + * @return total bytes of values + */ + public int getValueBytesSize() { + if (valueOccupation != -1) { + return valueOccupation; + } + valueOccupation = 0; + for (int i = 0; i < measurements.size(); i++) { + switch (measurements.get(i).getType()) { + case BOOLEAN: + valueOccupation += batchSize; + break; + case INT32: + valueOccupation += batchSize * 4; + break; + case INT64: + valueOccupation += batchSize * 8; + break; + case FLOAT: + valueOccupation += batchSize * 4; + break; + case DOUBLE: + valueOccupation += batchSize * 8; + break; + case TEXT: + for (Binary value: (Binary[]) values[i]) { + valueOccupation += value.getLength(); + } + break; + default: + throw new UnSupportedDataTypeException( + String.format("Data type %s is not supported.", measurements.get(i).getType())); + } + } + return valueOccupation; + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/TSRecord.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/TSRecord.java index 9ce98a7661c5866f64adc9a8e52c713d8b575fb2..d7822ff9cf6ef247ddf51627662b38d3edf59252 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/TSRecord.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/record/TSRecord.java @@ -26,8 +26,6 @@ import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; /** * TSRecord is a kind of format that TsFile receives.TSRecord contains timestamp, deviceId and a * list of data points. - * - * @author kangrong */ public class TSRecord { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java index 687029a0e1613ecebe846d993ff2aca766f81691..e7d3f3b2a7f498ac719d6c2ee1c3dff05ba25cb3 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java @@ -38,7 +38,7 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.utils.StringContainer; /** - * This class describes a measurement's information registered in {@linkplain FileSchema FilSchema}, + * This class describes a measurement's information registered in {@linkplain Schema FilSchema}, * including measurement id, data type, encoding and compressor type. For each TSEncoding, * MeasurementSchema maintains respective TSEncodingBuilder; For TSDataType, only ENUM has * TSDataTypeConverter up to now. @@ -288,13 +288,12 @@ public class MeasurementSchema implements Comparable, Seriali MeasurementSchema that = (MeasurementSchema) o; return type == that.type && encoding == that.encoding && Objects .equals(measurementId, that.measurementId) - && Objects.equals(encodingConverter, that.encodingConverter) - && Objects.equals(compressor, that.compressor) && Objects.equals(props, that.props); + && Objects.equals(compressor, that.compressor); } @Override public int hashCode() { - return Objects.hash(type, encoding, measurementId, encodingConverter, compressor, props); + return Objects.hash(type, encoding, measurementId, compressor); } /** diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/FileSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/Schema.java similarity index 83% rename from tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/FileSchema.java rename to tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/Schema.java index 67643bf64c404f6777cd446c83c9765c0748d54a..37d40081d06fb94366cc8e6e23f5ef82a8717827 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/FileSchema.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/Schema.java @@ -27,14 +27,12 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.write.record.RowBatch; /** - * FileSchema stores the schema of the measurements and devices that exist in this file. All - * devices written to the same TsFile shall have the same schema. FileSchema takes the JSON schema - * file as a parameter and registers measurements in such JSON. FileSchema also records all existing + * Schema stores the schema of the measurements and devices that exist in this file. All + * devices written to the same TsFile shall have the same schema. Schema takes the JSON schema + * file as a parameter and registers measurements in such JSON. Schema also records all existing * device IDs in this file. - * - * @author kangrong */ -public class FileSchema { +public class Schema { /** * the key is the measurementId. @@ -49,35 +47,35 @@ public class FileSchema { /** * init measurementSchemaMap as an empty map and an empty list. */ - public FileSchema() { + public Schema() { this.measurementSchemaMap = new HashMap<>(); this.measurementSchemaList = new ArrayList<>(); } /** - * Construct a FileSchema using provided schema map. + * Construct a Schema using provided schema map. * @param measurements a map whose key is the measurementId and value is the schema of * the measurement. */ - public FileSchema(Map measurements) { + public Schema(Map measurements) { this(); this.registerMeasurements(measurements); } /** - * Construct a FileSchema using provided schema list. + * Construct a Schema using provided schema list. * @param measurements a list with schemas of measurements */ - public FileSchema(List measurements) { + public Schema(List measurements) { this(); this.registerMeasurements(measurements); } /** - * Construct a FileSchema using provided schema array. + * Construct a Schema using provided schema array. * @param measurements an array with schemas of measurements */ - public FileSchema(MeasurementSchema[] measurements) { + public Schema(MeasurementSchema[] measurements) { this(); this.registerMeasurements(measurements); } @@ -85,16 +83,23 @@ public class FileSchema { /** * Create a row batch to write aligned data * @param deviceId the name of the device specified to be written in - * @return */ public RowBatch createRowBatch(String deviceId) { return new RowBatch(deviceId, measurementSchemaList); } + /** + * Create a row batch to write aligned data + * @param deviceId the name of the device specified to be written in + * @param maxBatchSize max size of rows in batch + */ + public RowBatch createRowBatch(String deviceId, int maxBatchSize) { + return new RowBatch(deviceId, measurementSchemaList, maxBatchSize); + } + /** * Get the data type fo a measurement specified by measurementId. * @param measurementId the name of the measurement being queried. - * @return */ public TSDataType getMeasurementDataType(String measurementId) { MeasurementSchema measurement = this.measurementSchemaMap.get(measurementId); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/SchemaBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/SchemaBuilder.java index 2f7f05e48d8f84afac38669c0a6f1bee4bff1a35..afb4cd92d6300fd29cd53d60007d0169864ee2ea 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/SchemaBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/SchemaBuilder.java @@ -24,20 +24,20 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; /** - * This class is used to build FileSchema of tsfile. + * This class is used to build Schema of tsfile. */ public class SchemaBuilder { /** - * the FileSchema which is being built. + * the Schema which is being built. **/ - private FileSchema fileSchema; + private Schema schema; /** * init schema by default value. */ public SchemaBuilder() { - fileSchema = new FileSchema(); + schema = new Schema(); } /** @@ -53,7 +53,7 @@ public class SchemaBuilder { public SchemaBuilder addSeries(String measurementId, TSDataType dataType, TSEncoding tsEncoding, CompressionType type, Map props) { MeasurementSchema md = new MeasurementSchema(measurementId, dataType, tsEncoding, type, props); - fileSchema.registerMeasurement(md); + schema.registerMeasurement(md); return this; } @@ -67,7 +67,7 @@ public class SchemaBuilder { */ public SchemaBuilder addSeries(String measurementId, TSDataType dataType, TSEncoding tsEncoding) { MeasurementSchema md = new MeasurementSchema(measurementId, dataType, tsEncoding); - fileSchema.registerMeasurement(md); + schema.registerMeasurement(md); return this; } @@ -78,7 +78,7 @@ public class SchemaBuilder { * @return schema builder */ public SchemaBuilder addSeries(MeasurementSchema descriptor) { - fileSchema.registerMeasurement(descriptor); + schema.registerMeasurement(descriptor); return this; } @@ -87,7 +87,7 @@ public class SchemaBuilder { * * @return constructed file schema */ - public FileSchema build() { - return this.fileSchema; + public Schema build() { + return this.schema; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/DefaultTsFileOutput.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/DefaultTsFileOutput.java index 0ab5e21f8fb4cadbc7677ae3ebdaeb0fa143981d..3cf667d9ac6727870ce5c4986481e53a0c0e8089 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/DefaultTsFileOutput.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/DefaultTsFileOutput.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.write.writer; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -32,47 +33,53 @@ import java.nio.ByteBuffer; public class DefaultTsFileOutput implements TsFileOutput { private FileOutputStream outputStream; + private BufferedOutputStream bufferedStream; - public DefaultTsFileOutput(File file) throws FileNotFoundException { + DefaultTsFileOutput(File file) throws FileNotFoundException { this.outputStream = new FileOutputStream(file); + this.bufferedStream = new BufferedOutputStream(outputStream); } - public DefaultTsFileOutput(File file, boolean append) throws FileNotFoundException { + DefaultTsFileOutput(File file, boolean append) throws FileNotFoundException { this.outputStream = new FileOutputStream(file, append); + this.bufferedStream = new BufferedOutputStream(outputStream); } public DefaultTsFileOutput(FileOutputStream outputStream) { this.outputStream = outputStream; + this.bufferedStream = new BufferedOutputStream(outputStream); } @Override public void write(byte[] b) throws IOException { - outputStream.write(b); + bufferedStream.write(b); } @Override public void write(ByteBuffer b) throws IOException { - throw new UnsupportedOperationException(); + bufferedStream.write(b.array()); } @Override public long getPosition() throws IOException { + bufferedStream.flush(); return outputStream.getChannel().position(); } @Override public void close() throws IOException { + bufferedStream.close(); outputStream.close(); } @Override - public OutputStream wrapAsStream() throws IOException { - return outputStream; + public OutputStream wrapAsStream() { + return bufferedStream; } @Override public void flush() throws IOException { - this.outputStream.flush(); + this.bufferedStream.flush(); } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java new file mode 100644 index 0000000000000000000000000000000000000000..10c5f425d334893b25b48b5946fd97d8dd3e1241 --- /dev/null +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/ForceAppendTsFileWriter.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.tsfile.write.writer; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; +import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +/** + * ForceAppendTsFileWriter opens a COMPLETE TsFile, reads and truncate its metadata to support + * appending new data. + */ +public class ForceAppendTsFileWriter extends TsFileIOWriter{ + + private Map knownSchemas; + private long truncatePosition; + + public ForceAppendTsFileWriter(File file) throws IOException { + this.out = new DefaultTsFileOutput(file, true); + this.file = file; + + // file doesn't exist + if (file.length() == 0 || !file.exists()) { + throw new TsFileNotCompleteException("File " + file.getPath() + " is not a complete TsFile"); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), true)) { + + // this tsfile is not complete + if (!reader.isComplete()) { + throw new TsFileNotCompleteException("File " + file.getPath() + " is not a complete TsFile"); + } + TsFileMetaData fileMetaData = reader.readFileMetadata(); + Map deviceMap = fileMetaData.getDeviceMap(); + long firstDeviceMetaPos = Long.MAX_VALUE; + for (TsDeviceMetadataIndex deviceMetadataIndex : deviceMap.values()) { + TsDeviceMetadata tsDeviceMetadata = reader + .readTsDeviceMetaData(deviceMetadataIndex); + chunkGroupMetaDataList.addAll(tsDeviceMetadata.getChunkGroupMetaDataList()); + firstDeviceMetaPos = firstDeviceMetaPos > deviceMetadataIndex.getOffset() ? + deviceMetadataIndex.getOffset() : firstDeviceMetaPos; + } + // truncate metadata and marker + truncatePosition = firstDeviceMetaPos - 1; + knownSchemas = fileMetaData.getMeasurementSchema(); + + } + } + + public void doTruncate() throws IOException { + out.truncate(truncatePosition); + } + + public long getTruncatePosition() { + return truncatePosition; + } + + @Override + public Map getKnownSchema() { + return knownSchemas; + } +} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java index 9af6b6e1f70bc5d81bd9b2316850f010ce6926f5..9fd10ea0c35990d5d233d92104328a001f4b9de9 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java @@ -70,6 +70,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { * @throws IOException if write failed, or the file is broken but autoRepair==false. */ public RestorableTsFileIOWriter(File file) throws IOException { + this.file = file; this.out = new DefaultTsFileOutput(file, true); // file doesn't exist @@ -91,9 +92,11 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { // uncompleted file truncatedPosition = reader.selfCheck(knownSchemas, chunkGroupMetaDataList, true); + totalChunkNum = reader.getTotalChunkNum(); if (truncatedPosition == TsFileCheckStatus.INCOMPATIBLE_FILE) { out.close(); - throw new IOException(String.format("%s is not in TsFile format.", file.getAbsolutePath())); + throw new IOException( + String.format("%s is not in TsFile format.", file.getAbsolutePath())); } else if (truncatedPosition == TsFileCheckStatus.ONLY_MAGIC_HEAD) { crashed = true; out.truncate(TSFileConfig.MAGIC_STRING.length()); @@ -129,7 +132,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { // filter: if a device'sensor is defined as float type, and data has been persistent. // Then someone deletes the timeseries and recreate it with Int type. We have to ignore // all the stale data. - if (dataType.equals(chunkMetaData.getTsDataType())) { + if (dataType == null || dataType.equals(chunkMetaData.getTsDataType())) { chunkMetaDataList.add(chunkMetaData); } } @@ -177,7 +180,8 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { private List getAppendedRowGroupMetadata() { List append = new ArrayList<>(); if (lastFlushedChunkGroupIndex < chunkGroupMetaDataList.size()) { - append.addAll(chunkGroupMetaDataList.subList(lastFlushedChunkGroupIndex, chunkGroupMetaDataList.size())); + append.addAll(chunkGroupMetaDataList + .subList(lastFlushedChunkGroupIndex, chunkGroupMetaDataList.size())); lastFlushedChunkGroupIndex = chunkGroupMetaDataList.size(); } return append; @@ -219,4 +223,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { return new RestorableTsFileIOWriter(file); } + public void addSchema(MeasurementSchema schema) { + knownSchemas.put(schema.getMeasurementId(), schema); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index be5a8338c0c13f19f96f9aed45f86f7cb90de11d..ff2a24c87d7bf9dc85591978a2af5d1347a96afd 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -21,23 +21,37 @@ package org.apache.iotdb.tsfile.write.writer; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter; import org.apache.iotdb.tsfile.file.header.ChunkHeader; -import org.apache.iotdb.tsfile.file.metadata.*; +import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; +import org.apache.iotdb.tsfile.file.metadata.TsDigest; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; +import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; +import org.apache.iotdb.tsfile.read.common.Chunk; +import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.BytesUtils; import org.apache.iotdb.tsfile.utils.PublicBAOS; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.apache.iotdb.tsfile.write.schema.FileSchema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +77,11 @@ public class TsFileIOWriter { private long markedPosition; + protected int totalChunkNum = 0; + protected int invalidChunkNum; + + protected File file; + /** * empty construct function. */ @@ -85,7 +104,6 @@ public class TsFileIOWriter { * for writing a new tsfile. * * @param output be used to output written data - * @throws IOException */ public TsFileIOWriter(TsFileOutput output) throws IOException { this.out = output; @@ -131,7 +149,8 @@ public class TsFileIOWriter { */ public void startChunkGroup(String deviceId) throws IOException { LOG.debug("start chunk group:{}, file position {}", deviceId, out.getPosition()); - currentChunkGroupMetaData = new ChunkGroupMetaData(deviceId, new ArrayList<>(), out.getPosition()); + currentChunkGroupMetaData = new ChunkGroupMetaData(deviceId, new ArrayList<>(), + out.getPosition()); } /** @@ -139,7 +158,8 @@ public class TsFileIOWriter { */ public void endChunkGroup(long version) throws IOException { long dataSize = out.getPosition() - currentChunkGroupMetaData.getStartOffsetOfChunkGroup(); - ChunkGroupFooter chunkGroupFooter = new ChunkGroupFooter(currentChunkGroupMetaData.getDeviceID(), + ChunkGroupFooter chunkGroupFooter = new ChunkGroupFooter( + currentChunkGroupMetaData.getDeviceID(), dataSize, currentChunkGroupMetaData.getChunkMetaDataList().size()); chunkGroupFooter.serializeTo(out.wrapAsStream()); currentChunkGroupMetaData.setEndOffsetOfChunkGroup(out.getPosition()); @@ -169,8 +189,7 @@ public class TsFileIOWriter { LOG.debug("start series chunk:{}, file position {}", descriptor, out.getPosition()); currentChunkMetaData = new ChunkMetaData(descriptor.getMeasurementId(), tsDataType, - out.getPosition(), minTime, - maxTime); + out.getPosition(), minTime, maxTime); ChunkHeader header = new ChunkHeader(descriptor.getMeasurementId(), dataSize, tsDataType, compressionCodecName, @@ -178,23 +197,40 @@ public class TsFileIOWriter { header.serializeTo(out.wrapAsStream()); LOG.debug("finish series chunk:{} header, file position {}", header, out.getPosition()); - Map statisticsMap = new HashMap<>(); // TODO add your statistics - statisticsMap.put(StatisticConstant.MAX_VALUE, ByteBuffer.wrap(statistics.getMaxBytes())); - statisticsMap.put(StatisticConstant.MIN_VALUE, ByteBuffer.wrap(statistics.getMinBytes())); - statisticsMap.put(StatisticConstant.FIRST, ByteBuffer.wrap(statistics.getFirstBytes())); - statisticsMap.put(StatisticConstant.SUM, ByteBuffer.wrap(statistics.getSumBytes())); - statisticsMap.put(StatisticConstant.LAST, ByteBuffer.wrap(statistics.getLastBytes())); + ByteBuffer[] statisticsArray = new ByteBuffer[StatisticType.getTotalTypeNum()]; + statisticsArray[StatisticType.max_value.ordinal()] = ByteBuffer.wrap(statistics.getMaxBytes()); + statisticsArray[StatisticType.min_value.ordinal()] = ByteBuffer.wrap(statistics.getMinBytes()); + statisticsArray[StatisticType.first_value.ordinal()] = ByteBuffer + .wrap(statistics.getFirstBytes()); + statisticsArray[StatisticType.last_value.ordinal()] = ByteBuffer + .wrap(statistics.getLastBytes()); + statisticsArray[StatisticType.sum_value.ordinal()] = ByteBuffer.wrap(statistics.getSumBytes()); TsDigest tsDigest = new TsDigest(); - tsDigest.setStatistics(statisticsMap); + tsDigest.setStatistics(statisticsArray); currentChunkMetaData.setDigest(tsDigest); return header.getSerializedSize(); } + /** + * Write a whole chunk in another file into this file. Providing fast merge for IoTDB. + * @param chunk + */ + public void writeChunk(Chunk chunk, ChunkMetaData chunkMetadata) throws IOException { + ChunkHeader chunkHeader = chunk.getHeader(); + currentChunkMetaData = new ChunkMetaData(chunkHeader.getMeasurementID(), + chunkHeader.getDataType(), out.getPosition(), chunkMetadata.getStartTime(), + chunkMetadata.getEndTime()); + currentChunkMetaData.setDigest(chunkMetadata.getDigest()); + chunkHeader.serializeTo(out.wrapAsStream()); + out.write(chunk.getData()); + endChunk(chunkMetadata.getNumOfPoints()); + } + /** * end chunk and write some log. * @@ -205,15 +241,16 @@ public class TsFileIOWriter { currentChunkGroupMetaData.addTimeSeriesChunkMetaData(currentChunkMetaData); LOG.debug("end series chunk:{},totalvalue:{}", currentChunkMetaData, totalValueCount); currentChunkMetaData = null; + totalChunkNum ++; } /** * write {@linkplain TsFileMetaData TSFileMetaData} to output stream and close it. * - * @param schema FileSchema + * @param schema Schema * @throws IOException if I/O error occurs */ - public void endFile(FileSchema schema) throws IOException { + public void endFile(Schema schema) throws IOException { // serialize the SEPARATOR of MetaData and ChunkGroups ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream()); @@ -227,6 +264,8 @@ public class TsFileIOWriter { TsFileMetaData tsFileMetaData = new TsFileMetaData(tsDeviceMetadataIndexMap, schemaDescriptors, TSFileConfig.CURRENT_VERSION); + tsFileMetaData.setTotalChunkNum(totalChunkNum); + tsFileMetaData.setInvalidChunkNum(invalidChunkNum); long footerIndex = out.getPosition(); LOG.debug("start to flush the footer,file pos:{}", footerIndex); @@ -315,7 +354,6 @@ public class TsFileIOWriter { } - /** * get chunkGroupMetaDataList. * @@ -338,8 +376,12 @@ public class TsFileIOWriter { } /** +<<<<<<< HEAD * close the outputStream or file channel without writing FileMetadata. - * This is just used for Testing. +======= + * close the outputStream or file channel without writing FileMetadata. This is just used for + * Testing. +>>>>>>> master */ public void close() throws IOException { canWrite = false; @@ -349,6 +391,7 @@ public class TsFileIOWriter { void writeSeparatorMaskForTest() throws IOException { out.write(new byte[]{MetaMarker.SEPARATOR}); } + void writeChunkMaskForTest() throws IOException { out.write(new byte[]{MetaMarker.CHUNK_HEADER}); } @@ -360,4 +403,52 @@ public class TsFileIOWriter { public Map getKnownSchema() { return Collections.emptyMap(); } + + public int getTotalChunkNum() { + return totalChunkNum; + } + + public int getInvalidChunkNum() { + return invalidChunkNum; + } + + public File getFile() { + return file; + } + + /** + * Remove such ChunkMetadata that its startTime is not in chunkStartTimes + * @param chunkStartTimes + */ + public void filterChunks(Map> chunkStartTimes) { + Map startTimeIdxes = new HashMap<>(); + chunkStartTimes.forEach((p, t) -> startTimeIdxes.put(p, 0)); + + Iterator chunkGroupMetaDataIterator = chunkGroupMetaDataList.iterator(); + while (chunkGroupMetaDataIterator.hasNext()) { + ChunkGroupMetaData chunkGroupMetaData = chunkGroupMetaDataIterator.next(); + String deviceId = chunkGroupMetaData.getDeviceID(); + int chunkNum = chunkGroupMetaData.getChunkMetaDataList().size(); + Iterator chunkMetaDataIterator = + chunkGroupMetaData.getChunkMetaDataList().iterator(); + while (chunkMetaDataIterator.hasNext()) { + ChunkMetaData chunkMetaData = chunkMetaDataIterator.next(); + Path path = new Path(deviceId, chunkMetaData.getMeasurementUid()); + int startTimeIdx = startTimeIdxes.get(path); + List pathChunkStartTimes = chunkStartTimes.get(path); + boolean chunkValid = startTimeIdx < pathChunkStartTimes.size() + && pathChunkStartTimes.get(startTimeIdx) == chunkMetaData.getStartTime(); + if (!chunkValid) { + chunkMetaDataIterator.remove(); + chunkNum--; + invalidChunkNum++; + } else { + startTimeIdxes.put(path, startTimeIdx + 1); + } + } + if (chunkNum == 0) { + chunkGroupMetaDataIterator.remove(); + } + } + } } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java index 65d31f3fc698b01f46b81f4689674164ce832031..dd6ac6d44bd0a9c5a34533ae5388cf0833006872 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java @@ -177,7 +177,7 @@ public class IntRleDecoderTest { public void testBooleanLength(List list, int bitWidth, boolean isDebug, int repeatCount) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - RleEncoder encoder = new IntRleEncoder(EndianType.LITTLE_ENDIAN); + RleEncoder encoder = new IntRleEncoder(EndianType.BIG_ENDIAN); for (int i = 0; i < repeatCount; i++) { for (int value : list) { if (value % 2 == 0) { @@ -191,7 +191,7 @@ public class IntRleDecoderTest { } ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); - RleDecoder decoder = new IntRleDecoder(EndianType.LITTLE_ENDIAN); + RleDecoder decoder = new IntRleDecoder(EndianType.BIG_ENDIAN); for (int i = 0; i < repeatCount; i++) { for (int value : list) { boolean value_ = decoder.readBoolean(buffer); @@ -211,7 +211,7 @@ public class IntRleDecoderTest { public void testLength(List list, int bitWidth, boolean isDebug, int repeatCount) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - RleEncoder encoder = new IntRleEncoder(EndianType.LITTLE_ENDIAN); + RleEncoder encoder = new IntRleEncoder(EndianType.BIG_ENDIAN); for (int i = 0; i < repeatCount; i++) { for (int value : list) { encoder.encode(value, baos); @@ -220,7 +220,7 @@ public class IntRleDecoderTest { } ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); - RleDecoder decoder = new IntRleDecoder(EndianType.LITTLE_ENDIAN); + RleDecoder decoder = new IntRleDecoder(EndianType.BIG_ENDIAN); for (int i = 0; i < repeatCount; i++) { for (int value : list) { int value_ = decoder.readInt(buffer); @@ -240,7 +240,7 @@ public class IntRleDecoderTest { } ByteArrayOutputStream baos = new ByteArrayOutputStream(); int bitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(list); - RleEncoder encoder = new IntRleEncoder(EndianType.LITTLE_ENDIAN); + RleEncoder encoder = new IntRleEncoder(EndianType.BIG_ENDIAN); for (int value : list) { encoder.encode(value, baos); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java index a90293f9174d36b6f4c278954673694992fed572..caa25df2a5289a7bcc0679a09c7eb023fd1cbbdc 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java @@ -180,7 +180,7 @@ public class LongRleDecoderTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); int bitWidth = ReadWriteForEncodingUtils.getLongMaxBitWidth(list); - RleEncoder encoder = new LongRleEncoder(EndianType.LITTLE_ENDIAN); + RleEncoder encoder = new LongRleEncoder(EndianType.BIG_ENDIAN); for (long value : list) { encoder.encode(value, baos); } @@ -202,7 +202,7 @@ public class LongRleDecoderTest { public void testLength(List list, int bitWidth, boolean isDebug, int repeatCount) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - RleEncoder encoder = new LongRleEncoder(EndianType.LITTLE_ENDIAN); + RleEncoder encoder = new LongRleEncoder(EndianType.BIG_ENDIAN); for (int i = 0; i < repeatCount; i++) { for (long value : list) { encoder.encode(value, baos); @@ -211,7 +211,7 @@ public class LongRleDecoderTest { } ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); - RleDecoder decoder = new LongRleDecoder(EndianType.LITTLE_ENDIAN); + RleDecoder decoder = new LongRleDecoder(EndianType.BIG_ENDIAN); for (int i = 0; i < repeatCount; i++) { for (long value : list) { long value_ = decoder.readLong(buffer); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java index 29863fee5d1bfdc447cedd974b3100143139741a..b6910952914269b2af5648055f03ad96f95bf01b 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java @@ -18,11 +18,13 @@ */ package org.apache.iotdb.tsfile.file.metadata; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; - import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.header.ChunkHeader; @@ -87,7 +89,7 @@ public class ChunkGroupMetaDataTest { } int offsetListIndex = 0; List deviceMetadataIndexList = metaData.getDeviceMap().values().stream() - .sorted((x, y) -> (int) (x.getOffset() - y.getOffset())).collect(Collectors.toList()); + .sorted((x, y) -> (int) (x.getOffset() - y.getOffset())).collect(Collectors.toList()); for (TsDeviceMetadataIndex index : deviceMetadataIndexList) { TsDeviceMetadata deviceMetadata = reader.readTsDeviceMetaData(index); List chunkGroupMetaDataList = deviceMetadata.getChunkGroupMetaDataList(); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java index 39e85e082caea5b2b955bb97cbe403e62c10cb21..c2e9bd778b301294d69360afe27e200ebf797888 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java @@ -52,7 +52,7 @@ public class ChunkMetaDataTest { } @Test - public void testWriteIntoFile() throws IOException { + public void testWriteIntoFile() { ChunkMetaData metaData = TestHelper.createSimpleTimeSeriesChunkMetaData(); serialized(metaData); ChunkMetaData readMetaData = deSerialized(); @@ -60,6 +60,15 @@ public class ChunkMetaDataTest { serialized(readMetaData); } + @Test + public void testWriteIntoFile2() throws IOException { + ChunkMetaData metaData = TestHelper.createNotCompleteSimpleTimeSeriesChunkMetaData(); + serialized(metaData); + ChunkMetaData readMetaData = deSerialized(); + Utils.isTimeSeriesChunkMetadataEqual(metaData, readMetaData); + serialized(readMetaData); + } + private ChunkMetaData deSerialized() { FileInputStream fis = null; ChunkMetaData metaData = null; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatisticsTest.java index 949c39fcbbea3b4562e2561ce5bd6459efab5296..4fc6d27cfa3a60868c481ddaee59ec328ecd5a6a 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatisticsTest.java @@ -19,6 +19,8 @@ package org.apache.iotdb.tsfile.file.metadata.statistics; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import org.junit.Test; @@ -33,14 +35,14 @@ public class BooleanStatisticsTest { public void testUpdate() { Statistics booleanStatistics = new BooleanStatistics(); booleanStatistics.updateStats(true); - assertEquals(false, booleanStatistics.isEmpty()); + assertFalse(booleanStatistics.isEmpty()); booleanStatistics.updateStats(false); - assertEquals(false, booleanStatistics.isEmpty()); - assertEquals(true, (boolean) booleanStatistics.getMax()); - assertEquals(false, (boolean) booleanStatistics.getMin()); - assertEquals(0, (double) booleanStatistics.getSum(), maxError); - assertEquals(true, (boolean) booleanStatistics.getFirst()); - assertEquals(false, (boolean) booleanStatistics.getLast()); + assertFalse(booleanStatistics.isEmpty()); + assertTrue(booleanStatistics.getMax()); + assertFalse(booleanStatistics.getMin()); + assertEquals(0, booleanStatistics.getSum(), maxError); + assertTrue(booleanStatistics.getFirst()); + assertFalse(booleanStatistics.getLast()); } @Test @@ -55,18 +57,18 @@ public class BooleanStatisticsTest { Statistics booleanStats3 = new BooleanStatistics(); booleanStats3.mergeStatistics(booleanStats1); - assertEquals(false, booleanStats3.isEmpty()); - assertEquals(false, (boolean) booleanStats3.getMax()); - assertEquals(false, (boolean) booleanStats3.getMin()); - assertEquals(0, (double) booleanStats3.getSum(), maxError); - assertEquals(false, (boolean) booleanStats3.getFirst()); - assertEquals(false, (boolean) booleanStats3.getLast()); + assertFalse(booleanStats3.isEmpty()); + assertFalse(booleanStats3.getMax()); + assertFalse(booleanStats3.getMin()); + assertEquals(0,booleanStats3.getSum(), maxError); + assertFalse(booleanStats3.getFirst()); + assertFalse(booleanStats3.getLast()); booleanStats3.mergeStatistics(booleanStats2); - assertEquals(true, (boolean) booleanStats3.getMax()); - assertEquals(false, (boolean) booleanStats3.getMin()); - assertEquals(0, (double) booleanStats3.getSum(), maxError); - assertEquals(false, (boolean) booleanStats3.getFirst()); - assertEquals(true, (boolean) booleanStats3.getLast()); + assertTrue(booleanStats3.getMax()); + assertFalse(booleanStats3.getMin()); + assertEquals(0, booleanStats3.getSum(), maxError); + assertFalse(booleanStats3.getFirst()); + assertTrue(booleanStats3.getLast()); } } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java index b19ffa1f307dc0d61ebbe4250eade10815d6bc92..7bb34332bc065d88414092281fce838ce11c6f34 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.file.metadata.statistics; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import org.junit.Test; @@ -30,14 +31,14 @@ public class DoubleStatisticsTest { public void testUpdate() { Statistics doubleStats = new DoubleStatistics(); doubleStats.updateStats(1.34d); - assertEquals(false, doubleStats.isEmpty()); + assertFalse(doubleStats.isEmpty()); doubleStats.updateStats(2.32d); - assertEquals(false, doubleStats.isEmpty()); - assertEquals(2.32d, (double) doubleStats.getMax(), maxError); - assertEquals(1.34d, (double) doubleStats.getMin(), maxError); - assertEquals(2.32d + 1.34d, (double) doubleStats.getSum(), maxError); - assertEquals(1.34d, (double) doubleStats.getFirst(), maxError); - assertEquals(2.32d, (double) doubleStats.getLast(), maxError); + assertFalse(doubleStats.isEmpty()); + assertEquals(2.32d, doubleStats.getMax(), maxError); + assertEquals(1.34d, doubleStats.getMin(), maxError); + assertEquals(2.32d + 1.34d, doubleStats.getSum(), maxError); + assertEquals(1.34d, doubleStats.getFirst(), maxError); + assertEquals(2.32d, doubleStats.getLast(), maxError); } @Test @@ -52,19 +53,19 @@ public class DoubleStatisticsTest { Statistics doubleStats3 = new DoubleStatistics(); doubleStats3.mergeStatistics(doubleStats1); - assertEquals(false, doubleStats3.isEmpty()); - assertEquals(100.13453d, (double) doubleStats3.getMax(), maxError); - assertEquals(1.34d, (double) doubleStats3.getMin(), maxError); - assertEquals(100.13453d + 1.34d, (double) doubleStats3.getSum(), maxError); - assertEquals(1.34d, (double) doubleStats3.getFirst(), maxError); - assertEquals(100.13453d, (double) doubleStats3.getLast(), maxError); + assertFalse(doubleStats3.isEmpty()); + assertEquals(100.13453d, doubleStats3.getMax(), maxError); + assertEquals(1.34d, doubleStats3.getMin(), maxError); + assertEquals(100.13453d + 1.34d, doubleStats3.getSum(), maxError); + assertEquals(1.34d, doubleStats3.getFirst(), maxError); + assertEquals(100.13453d, doubleStats3.getLast(), maxError); doubleStats3.mergeStatistics(doubleStats2); - assertEquals(200.435d, (double) doubleStats3.getMax(), maxError); - assertEquals(1.34d, (double) doubleStats3.getMin(), maxError); - assertEquals(100.13453d + 1.34d + 200.435d, (double) doubleStats3.getSum(), maxError); - assertEquals(1.34d, (double) doubleStats3.getFirst(), maxError); - assertEquals(200.435d, (double) doubleStats3.getLast(), maxError); + assertEquals(200.435d, doubleStats3.getMax(), maxError); + assertEquals(1.34d, doubleStats3.getMin(), maxError); + assertEquals(100.13453d + 1.34d + 200.435d, doubleStats3.getSum(), maxError); + assertEquals(1.34d, doubleStats3.getFirst(), maxError); + assertEquals(200.435d, doubleStats3.getLast(), maxError); } } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java index 867be8c6717689170122dcf0438843f0ca967096..b6b9841b75241e18ad669ee7321d84d88d1aa54a 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.file.metadata.statistics; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import org.junit.Test; @@ -30,9 +31,9 @@ public class FloatStatisticsTest { public void testUpdate() { Statistics floatStats = new FloatStatistics(); floatStats.updateStats(1.34f); - assertEquals(false, floatStats.isEmpty()); + assertFalse(floatStats.isEmpty()); floatStats.updateStats(2.32f); - assertEquals(false, floatStats.isEmpty()); + assertFalse(floatStats.isEmpty()); assertEquals(2.32f, (double) floatStats.getMax(), maxError); assertEquals(1.34f, (double) floatStats.getMin(), maxError); assertEquals(2.32f + 1.34f, (double) floatStats.getSum(), maxError); @@ -52,19 +53,19 @@ public class FloatStatisticsTest { Statistics floatStats3 = new FloatStatistics(); floatStats3.mergeStatistics(floatStats1); - assertEquals(false, floatStats3.isEmpty()); - assertEquals(100.13453f, (float) floatStats3.getMax(), maxError); - assertEquals(1.34f, (float) floatStats3.getMin(), maxError); + assertFalse(floatStats3.isEmpty()); + assertEquals(100.13453f, floatStats3.getMax(), maxError); + assertEquals(1.34f, floatStats3.getMin(), maxError); assertEquals(100.13453f + 1.34f, (float) floatStats3.getSum(), maxError); - assertEquals(1.34f, (float) floatStats3.getFirst(), maxError); - assertEquals(100.13453f, (float) floatStats3.getLast(), maxError); + assertEquals(1.34f, floatStats3.getFirst(), maxError); + assertEquals(100.13453f, floatStats3.getLast(), maxError); floatStats3.mergeStatistics(floatStats2); - assertEquals(200.435d, (float) floatStats3.getMax(), maxError); - assertEquals(1.34d, (float) floatStats3.getMin(), maxError); + assertEquals(200.435d, floatStats3.getMax(), maxError); + assertEquals(1.34d, floatStats3.getMin(), maxError); assertEquals(100.13453f + 1.34f + 200.435d, (float) floatStats3.getSum(), maxError); - assertEquals(1.34f, (float) floatStats3.getFirst(), maxError); - assertEquals(200.435f, (float) floatStats3.getLast(), maxError); + assertEquals(1.34f, floatStats3.getFirst(), maxError); + assertEquals(200.435f, floatStats3.getLast(), maxError); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java index 8b6a5949d93abe2a51aa3207c901318dc961b459..f34f9cffa8e5bdd96bf751360cbb9e8c75578fc5 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.file.metadata.statistics; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import org.junit.Test; @@ -28,9 +29,9 @@ public class IntegerStatisticsTest { public void testUpdate() { Statistics intStats = new IntegerStatistics(); intStats.updateStats(1); - assertEquals(false, intStats.isEmpty()); + assertFalse(intStats.isEmpty()); intStats.updateStats(2); - assertEquals(false, intStats.isEmpty()); + assertFalse(intStats.isEmpty()); assertEquals(2, (int) intStats.getMax()); assertEquals(1, (int) intStats.getMin()); assertEquals(1, (int) intStats.getFirst()); @@ -50,7 +51,7 @@ public class IntegerStatisticsTest { Statistics intStats3 = new IntegerStatistics(); intStats3.mergeStatistics(intStats1); - assertEquals(false, intStats3.isEmpty()); + assertFalse(intStats3.isEmpty()); assertEquals(100, (int) intStats3.getMax()); assertEquals(1, (int) intStats3.getMin()); assertEquals(1, (int) intStats3.getFirst()); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java index 6be700fd648fe828f0599e7afe75824d0ac9a66a..4ddf2e145cb5ba2acd12bf28c18b2ad1fdbaf305 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java @@ -19,6 +19,8 @@ package org.apache.iotdb.tsfile.file.metadata.statistics; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.junit.Test; @@ -28,13 +30,13 @@ public class LongStatisticsTest { @Test public void testUpdate() { Statistics longStats = new LongStatistics(); - assertEquals(true, longStats.isEmpty()); - long firstValue = -120985402913209l; - long secondValue = 1251465332132513l; + assertTrue(longStats.isEmpty()); + long firstValue = -120985402913209L; + long secondValue = 1251465332132513L; longStats.updateStats(firstValue); - assertEquals(false, longStats.isEmpty()); + assertFalse(longStats.isEmpty()); longStats.updateStats(secondValue); - assertEquals(false, longStats.isEmpty()); + assertFalse(longStats.isEmpty()); assertEquals(secondValue, (long) longStats.getMax()); assertEquals(firstValue, (long) longStats.getMin()); assertEquals(firstValue, (long) longStats.getFirst()); @@ -46,17 +48,17 @@ public class LongStatisticsTest { public void testMerge() { Statistics longStats1 = new LongStatistics(); Statistics longStats2 = new LongStatistics(); - assertEquals(true, longStats1.isEmpty()); - assertEquals(true, longStats2.isEmpty()); - long max1 = 100000000000l; - long max2 = 200000000000l; - longStats1.updateStats(1l); + assertTrue(longStats1.isEmpty()); + assertTrue(longStats2.isEmpty()); + long max1 = 100000000000L; + long max2 = 200000000000L; + longStats1.updateStats(1L); longStats1.updateStats(max1); longStats2.updateStats(max2); Statistics longStats3 = new LongStatistics(); longStats3.mergeStatistics(longStats1); - assertEquals(false, longStats3.isEmpty()); + assertFalse(longStats3.isEmpty()); assertEquals(max1, (long) longStats3.getMax()); assertEquals(1, (long) longStats3.getMin()); assertEquals(max1 + 1, (long) longStats3.getSum()); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/StringStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/StringStatisticsTest.java index 7a6cfcdfe22033ecd4d31bd5d2b1b1a641633f9e..8fd12b28f2c4b3f62e65bfcf458a0f83bcb6458e 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/StringStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/StringStatisticsTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.file.metadata.statistics; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import org.apache.iotdb.tsfile.utils.Binary; import org.junit.Test; @@ -31,9 +32,9 @@ public class StringStatisticsTest { public void testUpdate() { Statistics binaryStats = new BinaryStatistics(); binaryStats.updateStats(new Binary("aaa")); - assertEquals(false, binaryStats.isEmpty()); + assertFalse(binaryStats.isEmpty()); binaryStats.updateStats(new Binary("bbb")); - assertEquals(false, binaryStats.isEmpty()); + assertFalse(binaryStats.isEmpty()); assertEquals("bbb", binaryStats.getMax().getStringValue()); assertEquals("aaa", binaryStats.getMin().getStringValue()); assertEquals(0, binaryStats.getSum(), maxError); @@ -53,11 +54,11 @@ public class StringStatisticsTest { Statistics stringStats3 = new BinaryStatistics(); stringStats3.mergeStatistics(stringStats1); - assertEquals(false, stringStats3.isEmpty()); - assertEquals("ccc", (String) stringStats3.getMax().getStringValue()); - assertEquals("aaa", (String) stringStats3.getMin().getStringValue()); + assertFalse(stringStats3.isEmpty()); + assertEquals("ccc", stringStats3.getMax().getStringValue()); + assertEquals("aaa", stringStats3.getMin().getStringValue()); assertEquals(0, stringStats3.getSum(), maxError); - assertEquals("aaa", (String) stringStats3.getFirst().getStringValue()); + assertEquals("aaa", stringStats3.getFirst().getStringValue()); assertEquals("ccc", stringStats3.getLast().getStringValue()); stringStats3.mergeStatistics(stringStats2); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java index e76f522a37bc9e6babd791f73eff5b8943f31433..33aa6a3f9442b058eb2904935ce8d40a16953c51 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.file.metadata.utils; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -34,6 +35,7 @@ import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataTest; import org.apache.iotdb.tsfile.file.metadata.TsDigest; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaDataTest; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -116,19 +118,49 @@ public class TestHelper { return metaData; } + public static ChunkMetaData createNotCompleteSimpleTimeSeriesChunkMetaData() throws IOException { + ChunkMetaData metaData = new ChunkMetaData(ChunkMetaDataTest.MEASUREMENT_UID, + ChunkMetaDataTest.DATA_TYPE, + ChunkMetaDataTest.FILE_OFFSET, ChunkMetaDataTest.START_TIME, ChunkMetaDataTest.END_TIME + ); + metaData.setNumOfPoints(ChunkMetaDataTest.NUM_OF_POINTS); + metaData.setDigest(createNotCompleteSimpleTsDigest()); + return metaData; + } + public static MeasurementSchema createSimpleMeasurementSchema() { return new MeasurementSchema(TimeSeriesMetadataTest.measurementUID, TSDataType.INT64, TSEncoding.RLE); } - public static TsDigest createSimpleTsDigest() { + public static TsDigest createSimpleTsDigest() throws IOException { + TsDigest digest = new TsDigest(); + ByteBuffer[] statisticsArray = new ByteBuffer[StatisticType.getTotalTypeNum()]; + statisticsArray[StatisticType.min_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(MIN_VALUE)); + statisticsArray[StatisticType.max_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(MAX_VALUE)); + statisticsArray[StatisticType.first_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(FIRST_VALUE)); + statisticsArray[StatisticType.last_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(LAST_VALUE)); + statisticsArray[StatisticType.sum_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(SUM_VALUE)); + digest.setStatistics(statisticsArray); + return digest; + } + + public static TsDigest createNotCompleteSimpleTsDigest() throws IOException { TsDigest digest = new TsDigest(); - digest.addStatistics("max", ByteBuffer.wrap(BytesUtils.stringToBytes(MAX_VALUE))); - digest.addStatistics("min", ByteBuffer.wrap(BytesUtils.stringToBytes(MIN_VALUE))); - digest.addStatistics("sum", ByteBuffer.wrap(BytesUtils.stringToBytes(SUM_VALUE))); - digest.addStatistics("first", ByteBuffer.wrap(BytesUtils.stringToBytes(FIRST_VALUE))); - digest.addStatistics("last", ByteBuffer.wrap(BytesUtils.stringToBytes(LAST_VALUE))); + ByteBuffer[] statisticsArray = new ByteBuffer[StatisticType.getTotalTypeNum()]; + statisticsArray[StatisticType.first_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(FIRST_VALUE)); + statisticsArray[StatisticType.last_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(LAST_VALUE)); + statisticsArray[StatisticType.sum_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(SUM_VALUE)); + digest.setStatistics(statisticsArray); return digest; } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java index a645c2e26e9f93c484b32d988ff103c439413dba..e217d8bc569ef3b46f204964c69c71bd88bf6443 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java @@ -19,10 +19,10 @@ package org.apache.iotdb.tsfile.file.metadata.utils; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import org.apache.iotdb.tsfile.file.header.PageHeader; @@ -30,14 +30,16 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; +import org.apache.iotdb.tsfile.file.metadata.TsDigest; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; public class Utils { private static final double maxError = 0.0001d; - + public static void isListEqual(List listA, List listB, String name) { if ((listA == null) ^ (listB == null)) { @@ -70,32 +72,21 @@ public class Utils { } } - public static void isMapBufferEqual(Map mapA, Map mapB, - String name) { - if ((mapA == null) ^ (mapB == null)) { + public static void isTwoTsDigestEqual(TsDigest digestA, TsDigest digestB, String name) { + if ((digestA == null) ^ (digestB == null)) { System.out.println("error"); fail(String.format("one of %s is null", name)); } - if ((mapA != null) && (mapB != null)) { - if (mapA.size() != mapB.size()) { - fail(String.format("%s size is different", name)); - } - for (String key : mapB.keySet()) { - ByteBuffer b = mapB.get(key); - ByteBuffer a = mapA.get(key); - assertTrue(b.equals(a)); - } + if (digestA != null) { + Assert.assertEquals(digestA, digestB); } } /** * when one of A and B is Null, A != B, so test case fails. * - * @param objectA - * @param objectB - * @param name - * @return false - A and B both are NULL, so we do not need to check whether their members are equal true - A and B - * both are not NULL, so we need to check their members + * @return false - A and B both are NULL, so we do not need to check whether their members are + * equal true - A and B both are not NULL, so we need to check their members */ public static boolean isTwoObjectsNotNULL(Object objectA, Object objectB, String name) { if ((objectA == null) && (objectB == null)) { @@ -128,11 +119,9 @@ public class Utils { assertTrue(metadata1.getNumOfPoints() == metadata2.getNumOfPoints()); assertTrue(metadata1.getStartTime() == metadata2.getStartTime()); assertTrue(metadata1.getEndTime() == metadata2.getEndTime()); - if (Utils.isTwoObjectsNotNULL(metadata1.getDigest(), metadata2.getDigest(), "digest")) { - Utils.isMapBufferEqual(metadata1.getDigest().getStatistics(), - metadata2.getDigest().getStatistics(), - "statistics"); - } + assertNotNull(metadata1.getDigest()); + assertNotNull(metadata2.getDigest()); + Utils.isTwoTsDigestEqual(metadata1.getDigest(), metadata2.getDigest(), "TsDigest"); } } @@ -142,9 +131,11 @@ public class Utils { assertEquals(metadata1.getStartTime(), metadata2.getStartTime()); assertEquals(metadata1.getEndTime(), metadata2.getEndTime()); - if (Utils.isTwoObjectsNotNULL(metadata1.getChunkGroupMetaDataList(), metadata2.getChunkGroupMetaDataList(), + if (Utils.isTwoObjectsNotNULL(metadata1.getChunkGroupMetaDataList(), + metadata2.getChunkGroupMetaDataList(), "Rowgroup metadata list")) { - assertEquals(metadata1.getChunkGroupMetaDataList().size(), metadata2.getChunkGroupMetaDataList().size()); + assertEquals(metadata1.getChunkGroupMetaDataList().size(), + metadata2.getChunkGroupMetaDataList().size()); for (int i = 0; i < metadata1.getChunkGroupMetaDataList().size(); i++) { Utils.isChunkGroupMetaDataEqual(metadata1.getChunkGroupMetaDataList().get(i), metadata1.getChunkGroupMetaDataList().get(i)); @@ -219,7 +210,8 @@ public class Utils { assertTrue(header1.getNumOfValues() == header2.getNumOfValues()); assertTrue(header1.getMaxTimestamp() == header2.getMaxTimestamp()); assertTrue(header1.getMinTimestamp() == header2.getMinTimestamp()); - if (Utils.isTwoObjectsNotNULL(header1.getStatistics(), header2.getStatistics(), "statistics")) { + if (Utils + .isTwoObjectsNotNULL(header1.getStatistics(), header2.getStatistics(), "statistics")) { Utils.isStatisticsEqual(header1.getStatistics(), header2.getStatistics()); } } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java index bb32923e1bafcde75ee2977a675ac3e298c4e141..865da95ba5b51c6d21a6b84f0776459d8f83ed60 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java @@ -54,7 +54,7 @@ import org.junit.Test; The test target here is the logic of converting the query partition constraint to an additional time filter. Note that the correctness of the constructed additional time filter, which is guaranteed and tested in - MetadataQuerierByFileImplTest and TimeRangeTest, is not the test focus here. + IMetadataQuerierByFileImplTest and TimeRangeTest, is not the test focus here. */ public class ReadInPartitionTest { diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImplTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerierByFileImplTest.java similarity index 99% rename from tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImplTest.java rename to tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerierByFileImplTest.java index fe1882d85d0dc9bbc2b91c8bdaa6d1746c6832e5..1b7565377ab56d0e8970d0704cd5452e93c005f8 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/MetadataQuerierByFileImplTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/controller/IMetadataQuerierByFileImplTest.java @@ -36,7 +36,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -public class MetadataQuerierByFileImplTest { +public class IMetadataQuerierByFileImplTest { private static final String FILE_PATH = TsFileGeneratorForTest.outputDataFile; private TsFileSequenceReader reader; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java index e66aa1c42359a93c3262d3d1e97be3aae4e825c4..e0686c806f641c0f28ea64b81ea2ca3fd58430af 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.read.filter; +import java.nio.ByteBuffer; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; @@ -31,16 +32,20 @@ public class DigestFilterTest { BytesUtils.intToBytes(100), TSDataType.INT32); private DigestForFilter digest2 = new DigestForFilter(101L, 200L, BytesUtils.intToBytes(101), BytesUtils.intToBytes(200), TSDataType.INT32); + private DigestForFilter digest3 = new DigestForFilter(101L, 200L, (ByteBuffer) null, null, + TSDataType.INT32); @Test public void testEq() { Filter timeEq = TimeFilter.eq(10L); Assert.assertEquals(true, timeEq.satisfy(digest1)); Assert.assertEquals(false, timeEq.satisfy(digest2)); + Assert.assertEquals(false, timeEq.satisfy(digest3)); Filter valueEq = ValueFilter.eq(100); Assert.assertEquals(true, valueEq.satisfy(digest1)); Assert.assertEquals(false, valueEq.satisfy(digest2)); + Assert.assertEquals(true, valueEq.satisfy(digest3)); } @Test @@ -48,10 +53,12 @@ public class DigestFilterTest { Filter timeGt = TimeFilter.gt(100L); Assert.assertEquals(false, timeGt.satisfy(digest1)); Assert.assertEquals(true, timeGt.satisfy(digest2)); + Assert.assertEquals(true, timeGt.satisfy(digest3)); Filter valueGt = ValueFilter.gt(100); Assert.assertEquals(false, valueGt.satisfy(digest1)); Assert.assertEquals(true, valueGt.satisfy(digest2)); + Assert.assertEquals(true, valueGt.satisfy(digest3)); } @Test @@ -59,10 +66,12 @@ public class DigestFilterTest { Filter timeGtEq = TimeFilter.gtEq(100L); Assert.assertEquals(true, timeGtEq.satisfy(digest1)); Assert.assertEquals(true, timeGtEq.satisfy(digest2)); + Assert.assertEquals(true, timeGtEq.satisfy(digest3)); Filter valueGtEq = ValueFilter.gtEq(100); Assert.assertEquals(true, valueGtEq.satisfy(digest1)); - Assert.assertEquals(true, valueGtEq.satisfy(digest2)); + Assert.assertEquals(true, valueGtEq.satisfy(digest3)); + Assert.assertEquals(true, valueGtEq.satisfy(digest3)); } @Test @@ -70,10 +79,12 @@ public class DigestFilterTest { Filter timeLt = TimeFilter.lt(101L); Assert.assertEquals(true, timeLt.satisfy(digest1)); Assert.assertEquals(false, timeLt.satisfy(digest2)); + Assert.assertEquals(false, timeLt.satisfy(digest3)); Filter valueLt = ValueFilter.lt(101); Assert.assertEquals(true, valueLt.satisfy(digest1)); Assert.assertEquals(false, valueLt.satisfy(digest2)); + Assert.assertEquals(true, valueLt.satisfy(digest3)); } @Test @@ -81,10 +92,12 @@ public class DigestFilterTest { Filter timeLtEq = TimeFilter.ltEq(101L); Assert.assertEquals(true, timeLtEq.satisfy(digest1)); Assert.assertEquals(true, timeLtEq.satisfy(digest2)); + Assert.assertEquals(true, timeLtEq.satisfy(digest3)); Filter valueLtEq = ValueFilter.ltEq(101); Assert.assertEquals(true, valueLtEq.satisfy(digest1)); Assert.assertEquals(true, valueLtEq.satisfy(digest2)); + Assert.assertEquals(true, valueLtEq.satisfy(digest3)); } @Test @@ -92,10 +105,12 @@ public class DigestFilterTest { Filter andFilter = FilterFactory.and(TimeFilter.gt(10L), ValueFilter.lt(50)); Assert.assertEquals(true, andFilter.satisfy(digest1)); Assert.assertEquals(false, andFilter.satisfy(digest2)); + Assert.assertEquals(true, andFilter.satisfy(digest3)); Filter orFilter = FilterFactory.or(andFilter, TimeFilter.eq(200L)); Assert.assertEquals(true, orFilter.satisfy(digest1)); Assert.assertEquals(true, orFilter.satisfy(digest2)); + Assert.assertEquals(true, orFilter.satisfy(digest3)); } } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java index 2b426b99f73ef415a71f0ebdeefbf5cb742a5516..72156f1ed6dffd05b62a0c206bcde59ddb0596ec 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/executor/QueryExecutorTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.read.common.RowRecord; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.expression.IExpression; @@ -49,7 +49,7 @@ public class QueryExecutorTest { private static final String FILE_PATH = TsFileGeneratorForTest.outputDataFile; private TsFileSequenceReader fileReader; private MetadataQuerierByFileImpl metadataQuerierByFile; - private ChunkLoader chunkLoader; + private IChunkLoader chunkLoader; private int rowCount = 10000; private TsFileExecutor queryExecutorWithQueryFilter; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java index 1c50875e8d56b058d3ba810ec241d905258a7889..23d6b2764a204b8fb9dcddc2a7b490d26e0db67e 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TimeGeneratorTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.controller.ChunkLoader; +import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl; import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.iotdb.tsfile.read.expression.IExpression; @@ -45,7 +45,7 @@ public class TimeGeneratorTest { private static final String FILE_PATH = TsFileGeneratorForTest.outputDataFile; private TsFileSequenceReader fileReader; private MetadataQuerierByFileImpl metadataQuerierByFile; - private ChunkLoader chunkLoader; + private IChunkLoader chunkLoader; @Before public void before() throws InterruptedException, WriteProcessException, IOException { diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TsFileGeneratorForSeriesReaderByTimestamp.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TsFileGeneratorForSeriesReaderByTimestamp.java index ec2eebb959e6c7583e6f61a7591512feeb9bb36d..d3ccd3546eb4e2adae6cf719c686af7bf5fe2408 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TsFileGeneratorForSeriesReaderByTimestamp.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/query/timegenerator/TsFileGeneratorForSeriesReaderByTimestamp.java @@ -24,11 +24,8 @@ import java.io.FileWriter; import java.io.IOException; import java.util.Scanner; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -38,7 +35,7 @@ import org.apache.iotdb.tsfile.utils.FileUtils.Unit; import org.apache.iotdb.tsfile.utils.RecordUtils; import org.apache.iotdb.tsfile.write.TsFileWriter; import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Ignore; @@ -55,7 +52,7 @@ public class TsFileGeneratorForSeriesReaderByTimestamp { public static String inputDataFile; public static String outputDataFile = "target/testTsFile.tsfile"; public static String errorOutputDataFile; - public static FileSchema schema; + public static Schema schema; private static int rowCount; private static int chunkGroupSize; private static int pageSize; @@ -176,7 +173,7 @@ public class TsFileGeneratorForSeriesReaderByTimestamp { private static void generateTestData() { TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); - schema = new FileSchema(); + schema = new Schema(); schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder))); schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder), CompressionType.UNCOMPRESSED)); schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder), CompressionType.SNAPPY)); @@ -186,7 +183,7 @@ public class TsFileGeneratorForSeriesReaderByTimestamp { schema.registerMeasurement(new MeasurementSchema("s7", TSDataType.DOUBLE, TSEncoding.RLE)); } - static public void writeToFile(FileSchema schema) + static public void writeToFile(Schema schema) throws InterruptedException, IOException, WriteProcessException { Scanner in = getDataFile(inputDataFile); long lineCount = 0; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java index 681cd5a5b215581b75c901beadb7f6036fef3a86..ad303cd0a84e759ece27ade982a77263f2f04df1 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java @@ -132,8 +132,8 @@ public class PageReaderTest { @Test public void testBinary() { LoopWriteReadTest test = new LoopWriteReadTest("Test Double", - new PlainEncoder(EndianType.LITTLE_ENDIAN, TSDataType.TEXT, 1000), - new PlainDecoder(EndianType.LITTLE_ENDIAN), TSDataType.TEXT, POINTS_COUNT_IN_ONE_PAGE) { + new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.TEXT, 1000), + new PlainDecoder(EndianType.BIG_ENDIAN), TSDataType.TEXT, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { return new Binary(new StringBuilder("TEST TEXT").append(i).toString()); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java index f0059f727a109e56a857d30f267e3824366327fb..d1ff2e91bfd9a1220fe91fa8294090a2188e2f34 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/FileGenerator.java @@ -24,17 +24,14 @@ import java.io.FileWriter; import java.io.IOException; import java.util.Scanner; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.write.TsFileWriter; import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +44,7 @@ public class FileGenerator { public static String inputDataFile; public static String outputDataFile = "target/perTestOutputData.tsfile"; public static String errorOutputDataFile; - public static FileSchema schema; + public static Schema schema; public static int oldMaxNumberOfPointsInPage; public static void generateFile(int rowCount, int maxNumberOfPointsInPage) @@ -169,7 +166,7 @@ public class FileGenerator { } private static void generateTestData() { - schema = new FileSchema(); + schema = new Schema(); TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder))); schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); @@ -180,7 +177,7 @@ public class FileGenerator { schema.registerMeasurement(new MeasurementSchema("s7", TSDataType.DOUBLE, TSEncoding.RLE)); } - static public void writeToFile(FileSchema schema) + static public void writeToFile(Schema schema) throws IOException, WriteProcessException { Scanner in = getDataFile(inputDataFile); long lineCount = 0; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtils.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtils.java index 04f7527b48cd0dff62b1a78d6893c4a7883690af..33762c009c10244c5049df38679d468c48083239 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtils.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtils.java @@ -27,7 +27,7 @@ import org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint; import org.apache.iotdb.tsfile.write.record.datapoint.IntDataPoint; import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint; import org.apache.iotdb.tsfile.write.record.datapoint.StringDataPoint; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +47,7 @@ public class RecordUtils { * - constructed file schema * @return TSRecord constructed from str */ - public static TSRecord parseSimpleTupleRecord(String str, FileSchema schema) { + public static TSRecord parseSimpleTupleRecord(String str, Schema schema) { // split items String[] items = str.split(JsonFormatConstant.TSRECORD_SEPARATOR); // get deviceId and timestamp, then create a new TSRecord diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtilsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtilsTest.java index 16f2e469a37a9c0c225b9a423af883423f898de6..27af23cbe66fb05dad2d2f382dd9181802a25aca 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtilsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/RecordUtilsTest.java @@ -22,17 +22,14 @@ import static org.junit.Assert.assertEquals; import java.util.List; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.write.record.TSRecord; import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.Before; import org.junit.Test; @@ -44,24 +41,24 @@ import org.junit.Test; */ public class RecordUtilsTest { - FileSchema schema; + Schema schema; - private static FileSchema generateTestData() { - FileSchema fileSchema = new FileSchema(); + private static Schema generateTestData() { + Schema schema = new Schema(); TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); - fileSchema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s3", TSDataType.FLOAT, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s4", TSDataType.DOUBLE, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s5", TSDataType.BOOLEAN, TSEncoding.PLAIN)); - fileSchema.registerMeasurement(new MeasurementSchema("s6", TSDataType.TEXT, TSEncoding.PLAIN)); - return fileSchema; + schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.FLOAT, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s4", TSDataType.DOUBLE, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s5", TSDataType.BOOLEAN, TSEncoding.PLAIN)); + schema.registerMeasurement(new MeasurementSchema("s6", TSDataType.TEXT, TSEncoding.PLAIN)); + return schema; } @Before public void prepare() throws WriteProcessException { - schema = new FileSchema(); + schema = new Schema(); schema = generateTestData(); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java index a99948e3699c2af0a1b34674d3ba1f5a76f3971a..a067dc0e10076a9bd6a7805ac3db6408236efb9b 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TsFileGeneratorForTest.java @@ -36,7 +36,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.write.TsFileWriter; import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.SchemaBuilder; import org.junit.Assert; import org.junit.Ignore; @@ -154,7 +154,7 @@ public class TsFileGeneratorForTest { Assert.assertTrue(errorFile.delete()); } - FileSchema schema = generateTestSchema(); + Schema schema = generateTestSchema(); TSFileDescriptor.getInstance().getConfig().groupSizeInByte = chunkGroupSize; TSFileDescriptor.getInstance().getConfig().maxNumberOfPointsInPage = pageSize; @@ -221,7 +221,7 @@ public class TsFileGeneratorForTest { return jsonSchema; } - private static FileSchema generateTestSchema() { + private static Schema generateTestSchema() { SchemaBuilder schemaBuilder = new SchemaBuilder(); schemaBuilder.addSeries("s1", TSDataType.INT32, TSEncoding.RLE); schemaBuilder.addSeries("s2", TSDataType.INT64, TSEncoding.PLAIN); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java index 372bc9da647a73b1f682e99745f53177b29fdc3d..8b7e80fb734598e6a1eb44f7cc7448170085a09d 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.util.Random; import java.util.Scanner; -import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; @@ -35,11 +34,9 @@ import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.utils.FileUtils; -import org.apache.iotdb.tsfile.utils.FileUtils.Unit; import org.apache.iotdb.tsfile.utils.RecordUtils; import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -60,7 +57,7 @@ public class PerfTest { static public String inputDataFile; static public String outputDataFile; static public String errorOutputDataFile; - static public FileSchema schema; + static public Schema schema; static public Random r = new Random(); static private void generateSampleInputDataFile() throws IOException { @@ -141,7 +138,7 @@ public class PerfTest { } } - static private void writeToFile(FileSchema schema) + static private void writeToFile(Schema schema) throws InterruptedException, IOException, WriteProcessException { Scanner in = getDataFile(inputDataFile); assert in != null; @@ -153,18 +150,18 @@ public class PerfTest { innerWriter.close(); } - private static FileSchema generateTestData() { - FileSchema fileSchema = new FileSchema(); + private static Schema generateTestData() { + Schema schema = new Schema(); TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); - fileSchema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN)); + schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN)); JSONObject s4 = new JSONObject(); s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4"); s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString()); s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString()); - return fileSchema; + return schema; } @Before diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/ReadPageInMemTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/ReadPageInMemTest.java index d0d68c7a022bdee08bb472661f070150e3bd4520..1d4f9b68d96d1c42c3f99d20f0d1afada9db909e 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/ReadPageInMemTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/ReadPageInMemTest.java @@ -23,17 +23,14 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.RecordUtils; import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -45,21 +42,21 @@ public class ReadPageInMemTest { private File file = new File(filePath); private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); private TsFileWriter innerWriter; - private FileSchema schema = null; + private Schema schema = null; private int pageSize; private int ChunkGroupSize; private int pageCheckSizeThreshold; private int defaultMaxStringLength; - private static FileSchema getFileSchema() { - FileSchema fileSchema = new FileSchema(); + private static Schema getSchema() { + Schema schema = new Schema(); TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); - fileSchema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s3", TSDataType.FLOAT, TSEncoding.valueOf(conf.valueEncoder))); - fileSchema.registerMeasurement(new MeasurementSchema("s4", TSDataType.DOUBLE, TSEncoding.valueOf(conf.valueEncoder))); - return fileSchema; + schema.registerMeasurement(new MeasurementSchema("s1", TSDataType.INT32, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s3", TSDataType.FLOAT, TSEncoding.valueOf(conf.valueEncoder))); + schema.registerMeasurement(new MeasurementSchema("s4", TSDataType.DOUBLE, TSEncoding.valueOf(conf.valueEncoder))); + return schema; } @Before @@ -73,7 +70,7 @@ public class ReadPageInMemTest { conf.pageCheckSizeThreshold = 1; defaultMaxStringLength = conf.maxStringLength; conf.maxStringLength = 2; - schema = getFileSchema(); + schema = getSchema(); innerWriter = new TsFileWriter(new File(filePath), schema, conf); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java index 911c3c00590c926c48e962e10c65d00bf9943fe5..5e6bdb395d491a5260e58424c2b5d455834bd4c0 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java @@ -29,7 +29,7 @@ import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.file.metadata.utils.TestHelper; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; import org.junit.After; @@ -48,8 +48,8 @@ public class TsFileIOWriterTest { // file schema MeasurementSchema measurementSchema = TestHelper.createSimpleMeasurementSchema(); - FileSchema fileSchema = new FileSchema(); - fileSchema.registerMeasurement(measurementSchema); + Schema schema = new Schema(); + schema.registerMeasurement(measurementSchema); // chunk statistics Statistics statistics = Statistics.getStatsByType(measurementSchema.getType()); @@ -63,7 +63,7 @@ public class TsFileIOWriterTest { writer.endChunkGroup(0); // end file - writer.endFile(fileSchema); + writer.endFile(schema); } @After diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java index 8d384c02cb593072386a69301fa5bb0153e3377e..386cd0dd5b68e846d61e02477910a74a1244f92a 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileReadWriteTest.java @@ -38,7 +38,7 @@ import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; import org.apache.iotdb.tsfile.write.record.RowBatch; import org.apache.iotdb.tsfile.write.record.TSRecord; import org.apache.iotdb.tsfile.write.record.datapoint.*; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Before; @@ -152,13 +152,13 @@ public class TsFileReadWriteTest { private void writeDataByRowBatch() throws IOException, WriteProcessException { - FileSchema fileSchema = new FileSchema(); - fileSchema.registerMeasurement( + Schema schema = new Schema(); + schema.registerMeasurement( new MeasurementSchema("sensor_1", TSDataType.INT64, TSEncoding.TS_2DIFF)); int rowNum = 1024 * 1024 * 13 + 1023; int sensorNum = 1; - TsFileWriter tsFileWriter = new TsFileWriter(f, fileSchema); - RowBatch rowBatch = fileSchema.createRowBatch("device_1"); + TsFileWriter tsFileWriter = new TsFileWriter(f, schema); + RowBatch rowBatch = schema.createRowBatch("device_1"); long[] timestamps = rowBatch.timestamps; Object[] sensors = rowBatch.values; long timestamp = 1; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java index 044d82af54b106ea21076bbfef78257b05c0aa7b..a0004fdd5b89dd908e34ba7d0b92a9efb9241714 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/WriteTest.java @@ -38,7 +38,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.utils.RecordUtils; import org.apache.iotdb.tsfile.utils.StringContainer; import org.apache.iotdb.tsfile.write.record.TSRecord; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -62,7 +62,7 @@ public class WriteTest { private String errorOutputDataFile; private Random rm = new Random(); private ArrayList measurementArray; - private FileSchema schema; + private Schema schema; private int stageSize = 4; private int stageState = -1; private int prePageSize; @@ -110,7 +110,7 @@ public class WriteTest { measurementArray.add(new MeasurementSchema("s3", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.valueOf(TSFileConfig.compressor), props)); measurementArray.add(new MeasurementSchema("s4", TSDataType.BOOLEAN, TSEncoding.PLAIN)); - schema = new FileSchema(); + schema = new Schema(); LOG.info(schema.toString()); tsFileWriter = new TsFileWriter(file, schema, conf); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/SchemaBuilderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/SchemaBuilderTest.java index 0adccd89b1ad543c53015f995d168e057878d5f3..d014f72f9330bc3415f73304f3032280c18ec30a 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/SchemaBuilderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/SchemaBuilderTest.java @@ -27,7 +27,7 @@ import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.write.schema.FileSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.apache.iotdb.tsfile.write.schema.SchemaBuilder; import org.junit.Test; @@ -42,15 +42,13 @@ public class SchemaBuilderTest { SchemaBuilder builder = new SchemaBuilder(); Map props = new HashMap<>(); - props.put("enum_values", "[\"MAN\",\"WOMAN\"]"); - props.clear(); props.put(JsonFormatConstant.MAX_POINT_NUMBER, "3"); builder.addSeries("s4", TSDataType.DOUBLE, TSEncoding.RLE, CompressionType.SNAPPY, props); builder .addSeries("s5", TSDataType.INT32, TSEncoding.TS_2DIFF, CompressionType.UNCOMPRESSED, null); - FileSchema fileSchema = builder.build(); + Schema schema = builder.build(); - Collection measurements = fileSchema.getMeasurementSchemaMap().values(); + Collection measurements = schema.getMeasurementSchemaMap().values(); String[] measureDesStrings = {"[s4,DOUBLE,RLE,{max_point_number=3},SNAPPY]", "[s5,INT32,TS_2DIFF,{},UNCOMPRESSED]"}; int i = 0; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/series/PageWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/series/PageWriterTest.java index 588932ea172a0a9c980680eec834e6a318bb2edf..10a5cb6514e6b165823e6e45b16d0afc658d001e 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/series/PageWriterTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/series/PageWriterTest.java @@ -39,8 +39,8 @@ public class PageWriterTest { @Test public void testNoFreq() { PageWriter writer = new PageWriter(); - writer.setTimeEncoder(new PlainEncoder(EndianType.LITTLE_ENDIAN, TSDataType.INT64, 0)); - writer.setValueEncoder(new PlainEncoder(EndianType.LITTLE_ENDIAN, TSDataType.INT64, 0)); + writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0)); + writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0)); short s1 = 12; boolean b1 = false; int i1 = 1; @@ -66,7 +66,7 @@ public class PageWriterTest { byte[] timeBytes = new byte[timeSize]; buffer.get(timeBytes); ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes); - PlainDecoder decoder = new PlainDecoder(EndianType.LITTLE_ENDIAN); + PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN); for (int i = 0; i < timeCount; i++) { assertEquals(i, decoder.readLong(buffer2)); }
Member DescriptionMember Type