diff --git a/client/src/main/java/org/apache/iotdb/client/Client.java b/client/src/main/java/org/apache/iotdb/client/Client.java index 9068e77d79eaa6b5936988eae34214521db7b4cc..9678abcc3b2a33d5017517d6ed9e38f04359852a 100644 --- a/client/src/main/java/org/apache/iotdb/client/Client.java +++ b/client/src/main/java/org/apache/iotdb/client/Client.java @@ -33,6 +33,9 @@ import org.apache.iotdb.jdbc.Config; import org.apache.iotdb.jdbc.IoTDBConnection; import org.apache.thrift.TException; +/** + * usage: -h 127.0.0.1 -p 6667 -u root -pw root + */ public class Client extends AbstractClient { private static CommandLine commandLine; diff --git a/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md b/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md new file mode 100644 index 0000000000000000000000000000000000000000..15f77d90e1a69c6875a2da796db013be8402e69f --- /dev/null +++ b/docs/Documentation-CHN/UserGuide/6-JDBC API/2-Status Code.md @@ -0,0 +1,64 @@ + + +# Chapter6: JDBC API + +## Status Code + +在最新版本中引入了**状态码**这一概念。例如，因为IoTDB需要在写入数据之前首先注册时间序列，一种可能的解决方案是： + +``` +try { + writeData(); +} catch (SQLException e) { + // the most case is that the time series does not exist + if (e.getMessage().contains("exist")) { + //However, using the content of the error message is not so efficient + registerTimeSeries(); + //write data once again + writeData(); + } +} + +``` + +利用状态码，我们就可以不必写诸如`if (e.getErrorMessage().contains("exist"))`的代码，只需要使用`e.getStatusType().getCode() == TSStatusType.TIME_SERIES_NOT_EXIST_ERROR.getStatusCode()`。 + +这里是状态码和相对应信息的列表： + +|状态码|状态类型|状态信息| +|:---|:---|:---| +|200|SUCCESS_STATUS|| +|201|STILL_EXECUTING_STATUS|| +|202|INVALID_HANDLE_STATUS|| +|301|TIMESERIES_NOT_EXIST_ERROR|时间序列不存在| +|302|UNSUPPORTED_FETCH_METADATA_OPERATION_ERROR|不支持的获取元数据操作| +|303|FETCH_METADATA_ERROR|获取元数据失败| +|304|CHECK_FILE_LEVEL_ERROR|检查文件层级错误| +|400|EXECUTE_STATEMENT_ERROR|执行语句错误| +|401|SQL_PARSE_ERROR|SQL语句分析错误| +|402|GENERATE_TIME_ZONE_ERROR|生成时区错误| +|403|SET_TIME_ZONE_ERROR|设置时区错误| +|500|INTERNAL_SERVER_ERROR|服务器内部错误| +|600|WRONG_LOGIN_PASSWORD_ERROR|用户名或密码错误| +|601|NOT_LOGIN_ERROR|没有登录| +|602|NO_PERMISSION_ERROR|没有操作权限| +|603|UNINITIALIZED_AUTH_ERROR|授权人未初始化| diff --git a/docs/Documentation/UserGuide/0-Content.md b/docs/Documentation/UserGuide/0-Content.md index 1b245d2135fb2a5c3c24b614304511e0a3f825c2..07aa1cf618e8196529a1a164410687fd411b6992 100644 --- a/docs/Documentation/UserGuide/0-Content.md +++ b/docs/Documentation/UserGuide/0-Content.md @@ -49,10 +49,13 @@ * 2-Reference # Chapter 6: JDBC API * 1-JDBC API -# Chapter 7: TsFile +* 2-Status Code +# Chapter 7: Session API +* 1-Session API +# Chapter 8: TsFile * 1-Installation * 2-Usage * 3-Hierarchy -# Chapter 8: System Tools +# Chapter 9: System Tools * 1-Sync * 2-Memory Estimation Tool diff --git a/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md b/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md new file mode 100644 index 0000000000000000000000000000000000000000..ea62c35776b7a5b93691a52d9fc42e52ccd4a26c --- /dev/null +++ b/docs/Documentation/UserGuide/6-JDBC API/2-Status Code.md @@ -0,0 +1,64 @@ + + +# Chapter6: JDBC API + +## Status Code + +**Status Code** is introduced in the latest version. For example, as IoTDB requires registering the time series first before writing data, a kind of solution is: + +``` +try { + writeData(); +} catch (SQLException e) { + // the most case is that the time series does not exist + if (e.getMessage().contains("exist")) { + //However, using the content of the error message is not so efficient + registerTimeSeries(); + //write data once again + writeData(); + } +} + +``` + +With Status Code, instead of writing codes like `if (e.getErrorMessage().contains("exist"))`, we can simply use `e.getStatusType().getCode() == TSStatusType.TIME_SERIES_NOT_EXIST_ERROR.getStatusCode()`. + +Here is a list of Status Code and related message: + +|Status Code|Status Type|Status Message| +|:---|:---|:---| +|200|SUCCESS_STATUS|| +|201|STILL_EXECUTING_STATUS|| +|202|INVALID_HANDLE_STATUS|| +|301|TIMESERIES_NOT_EXIST_ERROR|Timeseries does not exist| +|302|UNSUPPORTED_FETCH_METADATA_OPERATION_ERROR|Unsupported fetch metadata operation| +|303|FETCH_METADATA_ERROR|Failed to fetch metadata| +|304|CHECK_FILE_LEVEL_ERROR|Meet error while checking file level| +|400|EXECUTE_STATEMENT_ERROR|Execute statement error| +|401|SQL_PARSE_ERROR|Meet error while parsing SQL| +|402|GENERATE_TIME_ZONE_ERROR|Meet error while generating time zone| +|403|SET_TIME_ZONE_ERROR|Meet error while setting time zone| +|500|INTERNAL_SERVER_ERROR|Internal server error| +|600|WRONG_LOGIN_PASSWORD_ERROR|Username or password is wrong| +|601|NOT_LOGIN_ERROR|Has not logged in| +|602|NO_PERMISSION_ERROR|No permissions for this operation| +|603|UNINITIALIZED_AUTH_ERROR|Uninitialized authorizer| diff --git a/docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md b/docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md index e50bcc66e4ecaf8b122ba90370c3d0c620e1d3b9..732c5e688e78da2b1acaefc5ecde125ea65f3c6d 100644 --- a/docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md +++ b/docs/Documentation/UserGuide/8-TsFile/3-Hierarchy.md @@ -7,9 +7,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,53 +18,63 @@ under the License. --> + # Chapter 8: TsFile + ## TsFile Hierarchy + Here is a brief introduction of the structure of a TsFile file. - + ## Variable Storage - * **Big Endian** - - * For Example, the `int` `0x8` will be stored as `00 00 00 08`, not `08 00 00 00` - - * **String with Variable Length** - - * The format is `int size` plus `String literal`. Size can be zero. - - * Size equals the number of bytes this string will take, and it may not equal to the length of the string. - - * For example "sensor_1" will be stored as `00 00 00 08` plus the encoding(ASCII) of "sensor_1". - - * Note that for the "Magic String"(file signature) "TsFilev0.8.0", the size(12) and encoding(ASCII) + +- **Big Endian** + + - For Example, the `int` `0x8` will be stored as `00 00 00 08`, not `08 00 00 00` +- **String with Variable Length** + - The format is `int size` plus `String literal`. Size can be zero. + - Size equals the number of bytes this string will take, and it may not equal to the length of the string. + - For example "sensor_1" will be stored as `00 00 00 08` plus the encoding(ASCII) of "sensor_1". + - Note that for the "Magic String"(file signature) "TsFilev0.8.0", the size(12) and encoding(ASCII) is fixed so there is no need to put the size before this string literal. - - * **Data Type Hardcode** - * 0: BOOLEAN - * 1: INT32 (`int`) - * 2: INT64 (`long`) - * 3: FLOAT - * 4: DOUBLE - * 5: TEXT (`String`) - * **Encoding Type Hardcode** - * 0: PLAIN - * 1: PLAIN_DICTIONARY - * 2: RLE - * 3: DIFF - * 4: TS_2DIFF - * 5: BITMAP - * 6: GORILLA - * 7: REGULAR - * **Compressing Type Hardcode** - * 0: UNCOMPRESSED - * 1: SNAPPY - - +- **Data Type Hardcode** + - 0: BOOLEAN + - 1: INT32 (`int`) + - 2: INT64 (`long`) + - 3: FLOAT + - 4: DOUBLE + - 5: TEXT (`String`) +- **Encoding Type Hardcode** + - 0: PLAIN + - 1: PLAIN_DICTIONARY + - 2: RLE + - 3: DIFF + - 4: TS_2DIFF + - 5: BITMAP + - 6: GORILLA + - 7: REGULAR +- **Compressing Type Hardcode** + - 0: UNCOMPRESSED + - 1: SNAPPY + - 2: GZIP + - 3: LZO + - 4: SDT + - 5: PAA + - 6: PLA +- **TsDigest Statistics Type Hardcode** + - 0: min_value + - 1: max_value + - 2: first_value + - 3: last_value + - 4: sum_value + ## TsFile Overview + Here is a graph about the TsFile structure. ![TsFile Breakdown](https://user-images.githubusercontent.com/40447846/61616997-6fad1300-ac9c-11e9-9c17-46785ebfbc88.png) ## Magic String + There is a 12 bytes magic string: `TsFilev0.8.0` @@ -87,6 +97,7 @@ The `ChunkGroup` has an array of `Chunk`, a following byte `0x00` as the marker, A `Chunk` represents a *sensor*. There is a byte `0x01` as the marker, following a `ChunkHeader` and an array of `Page`. ###### ChunkHeader +
@@ -136,6 +147,7 @@ PageHeader Structure ## Metadata ### TsDeviceMetaData + The first part of metadata is `TsDeviceMetaData`
@@ -148,6 +160,7 @@ The first part of metadata is `TsDeviceMetaData`
Then there is an array of `ChunkGroupMetaData` after `TsDeviceMetaData` + ### ChunkGroupMetaData
@@ -179,14 +192,57 @@ Then there is an array of `ChunkMetadata` for each `ChunkGroupMetadata`
Member DescriptionMember Type
-###### TsDigest +###### TsDigest (updated on 2019/8/27) + +Right now there are five statistics: `min_value, max_value, first_value, last_value, sum_value`. + +In v0.8.0, the storage format of statistics is a name-value pair. That is, `Map statistics`. The name is a string (remember the length is before the literal). But for the value, there is also an integer byteLength acting as the self description length of the following value because the value may be of various type. For example, if the `min_value` is an integer 0, then it will be stored as [9 "min_value" 4 0] in the TsFile. + +The figure below shows an example of `TsDigest.deserializeFrom(buffer)`. In v0.8.0, we will get -There are five statistics: `min, last, sum, first, max` +``` +Map statistics = { + "min_value" -> ByteBuffer of int value 0, + "last" -> ByteBuffer of int value 19, + "sum" -> ByteBuffer of double value 1093347116, + "first" -> ByteBuffer of int value 0, + "max_value" -> ByteBuffer of int value 99 +} +``` -The storage format is a name-value pair. The name is a string (remember the length is before the literal). +![TsDigest ByteBuffer Breakdown comparison](https://user-images.githubusercontent.com/33376433/63765352-664a4280-c8fb-11e9-869e-859edf6d00bb.png) -But for the value, there is also a size integer before the data even if it is not string. For example, if the `min` is 3, then it will be -stored as 3 "min" 4 3 in the TsFile. +In v0.9.0, the storage format is changed to an array for space and time efficiency. That is, `ByteBuffer[] statistics`. Each position of the array has a fixed association with a specific type of statistic, following the order defined in StatisticType: + +``` +enum StatisticType { + min_value, max_value, first_value, last_value, sum_value +} +``` + +Therefore, in the above example, we will get + +``` +ByteBuffer[] statistics = [ + ByteBuffer of int value 0, // associated with "min_value" + ByteBuffer of int value 99, // associated with "max_value" + ByteBuffer of int value 0, // associated with "first_value" + ByteBuffer of int value 19, // associated with "last_value" + ByteBuffer of double value 1093347116 // associated with "sum_value" +] +``` + +As another example in v0.9.0, when deserializing a TsDigest from buffer [3, 0,4,0, 1,4,99, 3,4,19], we get + +``` +ByteBuffer[] statistics = [ + ByteBuffer of int value 0, // associated with "min_value" + ByteBuffer of int value 99, // associated with "max_value" + null, // associated with "first_value" + ByteBuffer of int value 19, // associated with "last_value" + null // associated with "sum_value" +] +``` #### File Metadata @@ -207,6 +263,7 @@ After the array of `ChunkGroupMetadata`, here is the last part of the metadata. ##### DeviceIndexMetadata +
@@ -219,6 +276,7 @@ After the array of `ChunkGroupMetadata`, here is the last part of the metadata. ##### MeasurementSchema +
Member DescriptionMember Type
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java index 62409924cb6f0c0beff396973de77c34cb46cdf9..83a96abe8c2438255d1979f5cf3870a95f796836 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java @@ -424,7 +424,7 @@ public class IoTDBConnection implements Connection { } catch (IoTDBRPCException e) { // failed to connect, disconnect from the server transport.close(); - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } if (protocolVersion.getValue() != openResp.getServerProtocolVersion().getValue()) { throw new TException(String @@ -485,7 +485,7 @@ public class IoTDBConnection implements Connection { try { RpcUtils.verifySuccess(resp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } return resp.getTimeZone(); } @@ -496,7 +496,7 @@ public class IoTDBConnection implements Connection { try { RpcUtils.verifySuccess(resp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } this.zoneId = ZoneId.of(zoneId); } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java index 209a0fa8e4ce74f24e43f7902ab0054f0fd69091..36d01c2c830a2c28afe8ec7e7c4d67a490561819 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java @@ -87,7 +87,7 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { try { RpcUtils.verifySuccess(resp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } return new IoTDBMetadataResultSet(resp.getColumnsList(), null, null); } catch (TException e) { @@ -101,7 +101,7 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { try { RpcUtils.verifySuccess(resp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } return new IoTDBMetadataResultSet(resp.getColumnsList(), null, null); } catch (TException e) { @@ -114,7 +114,7 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { try { RpcUtils.verifySuccess(resp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } Set showStorageGroup = resp.getShowStorageGroups(); return new IoTDBMetadataResultSet(null, showStorageGroup, null); @@ -129,7 +129,7 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { try { RpcUtils.verifySuccess(resp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } List> showTimeseriesList = resp.getShowTimeseriesList(); return new IoTDBMetadataResultSet(null, null, showTimeseriesList); @@ -1268,7 +1268,7 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { try { RpcUtils.verifySuccess(resp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } return resp.getMetadataInJson(); } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java index 3065d8f5f95113b45b2f20f2142b8e90f5cc2318..2fe037073d9bcbac404ec8d01a5d95e460c3f90e 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedInsertionStatement.java @@ -22,11 +22,12 @@ package org.apache.iotdb.jdbc; import java.sql.SQLException; import java.time.ZoneId; import java.util.List; + +import org.apache.iotdb.rpc.TSStatusType; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSIService.Iface; import org.apache.iotdb.service.rpc.thrift.TSInsertionReq; import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.apache.thrift.TException; public class IoTDBPreparedInsertionStatement extends IoTDBPreparedStatement { @@ -49,7 +50,7 @@ public class IoTDBPreparedInsertionStatement extends IoTDBPreparedStatement { req.unsetMeasurements(); req.unsetTimestamp(); req.unsetValues(); - return resp.getStatus().getStatusCode() == TS_StatusCode.SUCCESS_STATUS; + return resp.getStatus().getStatusType().getCode() == TSStatusType.SUCCESS_STATUS.getStatusCode(); } catch (TException e) { throw new SQLException(e); } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java index ab675d433b07b3cf59c317724d4b19cbdfc9f53f..a865a9022c75e6d811a53e1af40c60bfa34ce53b 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBQueryResultSet.java @@ -713,7 +713,7 @@ public class IoTDBQueryResultSet implements ResultSet { try { RpcUtils.verifySuccess(resp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } if (!resp.hasResultSet) { emptyResultSet = true; diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java index 393e5d2805a3bb1208cbe4915742288768bf6e18..680a0a3f173e81577aabe7d970d8ebb84110fd8d 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java @@ -30,20 +30,11 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; + import org.apache.iotdb.rpc.IoTDBRPCException; import org.apache.iotdb.rpc.RpcUtils; -import org.apache.iotdb.service.rpc.thrift.TSCancelOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCancelOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.thrift.TException; public class IoTDBStatement implements Statement { @@ -51,6 +42,7 @@ public class IoTDBStatement implements Statement { private static final String SHOW_TIMESERIES_COMMAND_LOWERCASE = "show timeseries"; private static final String SHOW_STORAGE_GROUP_COMMAND_LOWERCASE = "show storage group"; private static final String METHOD_NOT_SUPPORTED_STRING = "Method not supported"; + ZoneId zoneId; private ResultSet resultSet = null; private IoTDBConnection connection; @@ -251,7 +243,7 @@ public class IoTDBStatement implements Statement { try { RpcUtils.verifySuccess(execResp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } if (execResp.getOperationHandle().hasResultSet) { IoTDBQueryResultSet resSet = new IoTDBQueryResultSet(this, @@ -294,7 +286,7 @@ public class IoTDBStatement implements Statement { TSExecuteBatchStatementReq execReq = new TSExecuteBatchStatementReq(sessionHandle, batchSQLList); TSExecuteBatchStatementResp execResp = client.executeBatchStatement(execReq); - if (execResp.getStatus().statusCode == TS_StatusCode.SUCCESS_STATUS) { + if (execResp.getStatus().getStatusType().getCode() == TSStatusType.SUCCESS_STATUS.getStatusCode()) { if (execResp.getResult() == null) { return new int[0]; } else { @@ -309,7 +301,7 @@ public class IoTDBStatement implements Statement { } else { BatchUpdateException exception; if (execResp.getResult() == null) { - exception = new BatchUpdateException(execResp.getStatus().errorMessage, new int[0]); + exception = new BatchUpdateException(execResp.getStatus().getStatusType().getMessage(), new int[0]); } else { List result = execResp.getResult(); int len = result.size(); @@ -317,7 +309,7 @@ public class IoTDBStatement implements Statement { for (int i = 0; i < len; i++) { updateArray[i] = result.get(i); } - exception = new BatchUpdateException(execResp.getStatus().errorMessage, updateArray); + exception = new BatchUpdateException(execResp.getStatus().getStatusType().getMessage(), updateArray); } throw exception; } @@ -355,7 +347,7 @@ public class IoTDBStatement implements Statement { try { RpcUtils.verifySuccess(execResp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } IoTDBQueryResultSet resSet = new IoTDBQueryResultSet(this, execResp.getColumns(), client, operationHandle, sql, execResp.getOperationType(), execResp.getDataTypeList(), @@ -412,7 +404,7 @@ public class IoTDBStatement implements Statement { try { RpcUtils.verifySuccess(execResp.getStatus()); } catch (IoTDBRPCException e) { - throw new IoTDBSQLException(e); + throw new IoTDBSQLException(e.getMessage()); } return 0; } diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java index 7cf52deb3bf7a382b779995d86fae2a0d4ef7b8d..9ca73b268c05ea1050c64d46869b2f9a4f31ed94 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/Utils.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.jdbc; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Properties; @@ -32,8 +31,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Field; import org.apache.iotdb.tsfile.read.common.RowRecord; import org.apache.iotdb.tsfile.utils.Binary; -import org.apache.iotdb.tsfile.utils.BytesUtils; -import org.apache.iotdb.tsfile.write.record.RowBatch; /** * Utils to convert between thrift format and TsFile format. diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java index 912019d3c07d601c579457a990b54f2df52c8d39..09505f44d1348ddbe3046b4fc17b3fc5de2717fb 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/BatchTest.java @@ -29,12 +29,9 @@ import java.sql.Statement; import java.time.ZoneId; import java.util.ArrayList; import java.util.List; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.thrift.TException; import org.junit.After; import org.junit.Before; @@ -50,8 +47,10 @@ public class BatchTest { private TSIService.Iface client; @Mock private TS_SessionHandle sessHandle; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); - private TS_Status Status_ERROR = new TS_Status(TS_StatusCode.ERROR_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_StatusType errorStatus = new TS_StatusType(TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); + private TS_Status Status_ERROR = new TS_Status(errorStatus); private TSExecuteBatchStatementResp resp; private ZoneId zoneID = ZoneId.systemDefault(); diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java index e374064ef2126c22c771bd002419ccbd83d6f0c1..36bc42b9a46d67f95bef6dad31c5f69cb8916a12 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java @@ -24,13 +24,9 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.List; -import org.apache.iotdb.service.rpc.thrift.ServerProperties; -import org.apache.iotdb.service.rpc.thrift.TSGetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.thrift.TException; import org.junit.After; import org.junit.Before; @@ -44,7 +40,8 @@ public class IoTDBConnectionTest { private TSIService.Iface client; private IoTDBConnection connection = new IoTDBConnection(); - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); @Before public void setUp() throws Exception { diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java index 3c8e6fd22ee37204d9b44c3c5023d76eecb66144..2fdb40667ae688d2f61ba73e418edfcd3eb3ffb7 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java @@ -30,11 +30,9 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -66,7 +64,8 @@ public class IoTDBDatabaseMetadataTest { @Mock private TSFetchMetadataResp fetchMetadataResp; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); private DatabaseMetaData databaseMetaData; diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java index ca9928efc738d45f2d04dd7b2c38df4ac375737e..f690de2540373de4b5b81905d8bd98b3da32399b 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java @@ -26,14 +26,10 @@ import static org.mockito.Mockito.when; import java.sql.SQLException; import java.sql.Timestamp; import java.time.ZoneId; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSGetOperationStatusResp; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.service.rpc.thrift.TSIService.Iface; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -53,7 +49,8 @@ public class IoTDBPreparedStatementTest { private Iface client; @Mock private TS_SessionHandle sessHandle; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); @Mock private TSOperationHandle tOperationHandle; diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java index bbaadaa347b7409f58bd03bafe79a5d7b5187aed..70de759f234af0e6f49b5d9469894d2319b1663d 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBQueryResultSetTest.java @@ -32,22 +32,10 @@ import java.sql.Types; import java.time.ZoneId; import java.util.ArrayList; import java.util.List; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSDataValue; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; -import org.apache.iotdb.service.rpc.thrift.TSRowRecord; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; + import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.junit.Assert; import org.junit.Before; @@ -120,7 +108,8 @@ public class IoTDBQueryResultSetTest { @Mock private TSFetchResultsResp fetchResultsResp; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); private ZoneId zoneID = ZoneId.systemDefault(); @Before diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java index b96878f4fae6cc266e3e93f0502ac421281d019e..44fd2f4d224680c05068fd394b9114e9f14b01e4 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBStatementTest.java @@ -31,12 +31,10 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; + +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.service.rpc.thrift.TSIService.Iface; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.apache.thrift.TException; import org.junit.After; import org.junit.Assert; @@ -59,7 +57,8 @@ public class IoTDBStatementTest { @Mock private TSFetchMetadataResp fetchMetadataResp; - private TS_Status Status_SUCCESS = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + private TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + private TS_Status Status_SUCCESS = new TS_Status(successStatus); private ZoneId zoneID = ZoneId.systemDefault(); @Before diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java index 123d2e300b9ecfaff6e34b69c4405406a21741ba..bc3a766823363e5554b4e70c4175e17ac044baaa 100644 --- a/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java +++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/UtilsTest.java @@ -27,11 +27,14 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusType; import org.apache.iotdb.service.rpc.thrift.TSDataValue; import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; import org.apache.iotdb.service.rpc.thrift.TSRowRecord; import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; + +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Field; import org.apache.iotdb.tsfile.read.common.RowRecord; @@ -70,13 +73,15 @@ public class UtilsTest { @Test public void testVerifySuccess() { try { - RpcUtils.verifySuccess(new TS_Status(TS_StatusCode.SUCCESS_STATUS)); + TS_StatusType successStatus = new TS_StatusType(TSStatusType.SUCCESS_STATUS.getStatusCode(), ""); + RpcUtils.verifySuccess(new TS_Status(successStatus)); } catch (Exception e) { fail(); } try { - RpcUtils.verifySuccess(new TS_Status(TS_StatusCode.ERROR_STATUS)); + TS_StatusType errorStatus = new TS_StatusType(TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(), ""); + RpcUtils.verifySuccess(new TS_Status(errorStatus)); } catch (Exception e) { return; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index b5d47127c3aa7d72b185384aab9b08a5ba210433..1e41c57db7816d11800f751e7cc84ba30ef2cb01 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@ -67,9 +67,9 @@ import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.JobFileManager; +import org.apache.iotdb.rpc.TSStatusType; import org.apache.iotdb.db.utils.CopyOnReadLinkedList; import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -361,7 +361,7 @@ public class StorageGroupProcessor { List unsequenceIndexes = new ArrayList<>(); for (int i = 0; i < batchInsertPlan.getRowCount(); i++) { - results[i] = TS_StatusCode.SUCCESS_STATUS.getValue(); + results[i] = TSStatusType.SUCCESS_STATUS.getStatusCode(); if (batchInsertPlan.getTimes()[i] > latestFlushedTimeForEachDevice .get(batchInsertPlan.getDeviceId())) { sequenceIndexes.add(i); @@ -389,7 +389,7 @@ public class StorageGroupProcessor { TsFileProcessor tsFileProcessor = getOrCreateTsFileProcessor(sequence); if (tsFileProcessor == null) { for (int index : indexes) { - results[index] = TS_StatusCode.ERROR_STATUS.getValue(); + results[index] = TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(); } return; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index b46f0586f90ff8e972db1f5d17f7b5b4d1087cb9..5d237d19f7c4817b26c10527681c25d47cab64b9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -48,10 +48,10 @@ import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.rescon.MemTablePool; +import org.apache.iotdb.rpc.TSStatusType; import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager; import org.apache.iotdb.db.writelog.node.WriteLogNode; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.utils.Pair; @@ -173,7 +173,7 @@ public class TsFileProcessor { } catch (IOException e) { logger.error("write WAL failed", e); for (int index: indexes) { - results[index] = TS_StatusCode.ERROR_STATUS.getValue(); + results[index] = TSStatusType.INTERNAL_SERVER_ERROR.getStatusCode(); } return false; } diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java index 1aa830a00e4bffb475e547a2bb4a9d17ee0fc787..9d8cf965bd869dac1baa2c2077c9c6c143ba0163 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/resourceRelated/UnseqResourceMergeReader.java @@ -16,9 +16,11 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.query.reader.resourceRelated; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; import org.apache.iotdb.db.engine.cache.DeviceMetaDataCache; import org.apache.iotdb.db.engine.modification.Modification; @@ -29,8 +31,8 @@ import org.apache.iotdb.db.query.reader.chunkRelated.DiskChunkReader; import org.apache.iotdb.db.query.reader.chunkRelated.MemChunkReader; import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader; import org.apache.iotdb.db.utils.QueryUtils; -import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.common.Path; @@ -96,11 +98,16 @@ public class UnseqResourceMergeReader extends PriorityMergeReader { for (ChunkMetaData chunkMetaData : metaDataList) { if (filter != null) { + ByteBuffer minValue = null; + ByteBuffer maxValue = null; + ByteBuffer[] statistics = chunkMetaData.getDigest().getStatistics(); + if (statistics != null) { + minValue = statistics[StatisticType.min_value.ordinal()]; // note still CAN be null + maxValue = statistics[StatisticType.max_value.ordinal()]; // note still CAN be null + } + DigestForFilter digest = new DigestForFilter(chunkMetaData.getStartTime(), - chunkMetaData.getEndTime(), - chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MIN_VALUE), - chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MAX_VALUE), - chunkMetaData.getTsDataType()); + chunkMetaData.getEndTime(), minValue, maxValue, chunkMetaData.getTsDataType()); if (!filter.satisfy(digest)) { continue; } diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java index 79b7d6095755f74e9ce9691828cd6bd5c84e3325..f39046a6ff2518c69c7f5af7acf27c19a77c757c 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java @@ -18,24 +18,6 @@ */ package org.apache.iotdb.db.service; -import static org.apache.iotdb.db.conf.IoTDBConstant.PRIVILEGE; -import static org.apache.iotdb.db.conf.IoTDBConstant.ROLE; -import static org.apache.iotdb.db.conf.IoTDBConstant.USER; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.regex.Pattern; import org.apache.iotdb.db.auth.AuthException; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.auth.authorizer.IAuthorizer; @@ -49,12 +31,7 @@ import org.apache.iotdb.db.cost.statistic.Measurement; import org.apache.iotdb.db.cost.statistic.Operation; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.flush.pool.FlushTaskPoolManager; -import org.apache.iotdb.db.exception.ArgsErrorException; -import org.apache.iotdb.db.exception.MetadataErrorException; -import org.apache.iotdb.db.exception.PathErrorException; -import org.apache.iotdb.db.exception.ProcessorException; -import org.apache.iotdb.db.exception.QueryInBatchStmtException; -import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.exception.*; import org.apache.iotdb.db.exception.qp.IllegalASTFormatException; import org.apache.iotdb.db.exception.qp.QueryProcessorException; import org.apache.iotdb.db.metadata.MManager; @@ -68,36 +45,8 @@ import org.apache.iotdb.db.qp.physical.sys.AuthorPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.QueryResourceManager; import org.apache.iotdb.db.utils.QueryDataSetUtils; -import org.apache.iotdb.service.rpc.thrift.ServerProperties; -import org.apache.iotdb.service.rpc.thrift.TSBatchInsertionReq; -import org.apache.iotdb.service.rpc.thrift.TSCancelOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCancelOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationReq; -import org.apache.iotdb.service.rpc.thrift.TSCloseOperationResp; -import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq; -import org.apache.iotdb.service.rpc.thrift.TSCloseSessionResp; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteBatchStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchMetadataResp; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsReq; -import org.apache.iotdb.service.rpc.thrift.TSFetchResultsResp; -import org.apache.iotdb.service.rpc.thrift.TSGetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TSHandleIdentifier; -import org.apache.iotdb.service.rpc.thrift.TSIService; -import org.apache.iotdb.service.rpc.thrift.TSInsertionReq; -import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; -import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; -import org.apache.iotdb.service.rpc.thrift.TSOperationHandle; -import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; -import org.apache.iotdb.service.rpc.thrift.TSQueryDataSet; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneReq; -import org.apache.iotdb.service.rpc.thrift.TSSetTimeZoneResp; -import org.apache.iotdb.service.rpc.thrift.TS_SessionHandle; -import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; +import org.apache.iotdb.rpc.TSStatusType; +import org.apache.iotdb.service.rpc.thrift.*; import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -108,6 +57,18 @@ import org.apache.thrift.server.ServerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.ZoneId; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; + +import static org.apache.iotdb.db.conf.IoTDBConstant.*; + /** * Thrift RPC implementation at server side. */ @@ -116,7 +77,6 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { private static final Logger logger = LoggerFactory.getLogger(TSServiceImpl.class); private static final String INFO_NOT_LOGIN = "{}: Not login."; - private static final String ERROR_NOT_LOGIN = "Not login"; protected QueryProcessor processor; // Record the username for every rpc connection. Username.get() is null if @@ -157,14 +117,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } TS_Status tsStatus; if (status) { - tsStatus = new TS_Status(TS_StatusCode.SUCCESS_STATUS); - tsStatus.setErrorMessage("login successfully."); + tsStatus = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS, "Login successfully")); username.set(req.getUsername()); zoneIds.set(config.getZoneID()); initForOneSession(); } else { - tsStatus = new TS_Status(TS_StatusCode.ERROR_STATUS); - tsStatus.setErrorMessage("login failed. Username or password is wrong."); + tsStatus = getStatus(TSStatusType.WRONG_LOGIN_PASSWORD_ERROR); } TSOpenSessionResp resp = new TSOpenSessionResp(tsStatus, TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V1); @@ -172,8 +130,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { new TS_SessionHandle(new TSHandleIdentifier(ByteBuffer.wrap(req.getUsername().getBytes()), ByteBuffer.wrap(req.getPassword().getBytes())))); logger.info("{}: Login status: {}. User : {}", IoTDBConstant.GLOBAL_DB_NAME, - tsStatus.getErrorMessage(), - req.getUsername()); + tsStatus.getStatusType().getMessage(), req.getUsername()); return resp; } @@ -188,13 +145,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { logger.info("{}: receive close session", IoTDBConstant.GLOBAL_DB_NAME); TS_Status tsStatus; if (username.get() == null) { - tsStatus = new TS_Status(TS_StatusCode.ERROR_STATUS); - tsStatus.setErrorMessage("Has not logged in"); + tsStatus = getStatus(TSStatusType.NOT_LOGIN_ERROR); if (zoneIds.get() != null) { zoneIds.remove(); } } else { - tsStatus = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + tsStatus = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); username.remove(); if (zoneIds.get() != null) { zoneIds.remove(); @@ -205,7 +161,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { @Override public TSCancelOperationResp cancelOperation(TSCancelOperationReq req) { - return new TSCancelOperationResp(new TS_Status(TS_StatusCode.SUCCESS_STATUS)); + return new TSCancelOperationResp(getStatus(TSStatusType.SUCCESS_STATUS)); } @Override @@ -224,7 +180,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } catch (Exception e) { logger.error("Error in closeOperation : ", e); } - return new TSCloseOperationResp(new TS_Status(TS_StatusCode.SUCCESS_STATUS)); + return new TSCloseOperationResp(getStatus(TSStatusType.SUCCESS_STATUS)); } private void releaseQueryResource(TSCloseOperationReq req) throws StorageEngineException { @@ -253,9 +209,29 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } } - private TS_Status getErrorStatus(String message) { - TS_Status status = new TS_Status(TS_StatusCode.ERROR_STATUS); - status.setErrorMessage(message); + /** + * convert from TSStatusType to TS_Status according to status code and status message + * + * @param statusType status type + * @return + */ + private TS_Status getStatus(TSStatusType statusType) { + TS_StatusType statusCodeAndMessage = new TS_StatusType(statusType.getStatusCode(), statusType.getStatusMessage()); + TS_Status status = new TS_Status(statusCodeAndMessage); + return status; + } + + /** + * convert from TSStatusType to TS_Status, which has message appending with existed status message + * + * @param statusType status type + * @param appendMessage appending message + * @return + */ + private TS_Status getStatus(TSStatusType statusType, String appendMessage) { + TS_StatusType statusCodeAndMessage = new TS_StatusType(statusType.getStatusCode(), + statusType.getStatusMessage() + ": " + appendMessage); + TS_Status status = new TS_Status(statusCodeAndMessage); return status; } @@ -264,7 +240,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { TS_Status status; if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - status = getErrorStatus(ERROR_NOT_LOGIN); + status = getStatus(TSStatusType.NOT_LOGIN_ERROR); return new TSFetchMetadataResp(status); } TSFetchMetadataResp resp = new TSFetchMetadataResp(); @@ -274,17 +250,17 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { String path = req.getColumnPath(); List> showTimeseriesList = getTimeSeriesForPath(path); resp.setShowTimeseriesList(showTimeseriesList); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "SHOW_STORAGE_GROUP": Set storageGroups = getAllStorageGroups(); resp.setShowStorageGroups(storageGroups); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "METADATA_IN_JSON": String metadataInJson = getMetadataInString(); resp.setMetadataInJson(metadataInJson); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "DELTA_OBEJECT": Metadata metadata = getMetadata(); @@ -295,21 +271,18 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } else { resp.setColumnsList(deviceMap.get(column)); } - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "COLUMN": resp.setDataType(getSeriesType(req.getColumnPath()).toString()); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; case "ALL_COLUMNS": resp.setColumnsList(getPaths(req.getColumnPath())); - status = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + status = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); break; default: - status = new TS_Status(TS_StatusCode.ERROR_STATUS); - status - .setErrorMessage( - String.format("Unsupported fetch metadata operation %s", req.getType())); + status = getStatus(TSStatusType.FETCH_METADATA_ERROR, req.getType()); break; } } catch (PathErrorException | MetadataErrorException | OutOfMemoryError e) { @@ -317,8 +290,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { .error(String.format("Failed to fetch timeseries %s's metadata", req.getColumnPath()), e); Thread.currentThread().interrupt(); - status = getErrorStatus( - String.format("Failed to fetch metadata because: %s", e)); + status = getStatus(TSStatusType.FETCH_METADATA_ERROR, e.getMessage()); resp.setStatus(status); return resp; } @@ -425,7 +397,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { try { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN, null); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR), null); } List statements = req.getStatements(); @@ -440,16 +412,15 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } if (isAllSuccessful) { - return getTSBathExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, - "Execute batch statements successfully", result); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, + "Execute batch statements successfully"), result); } else { - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - batchErrorMessage.toString(), - result); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, + batchErrorMessage.toString()), result); } } catch (Exception e) { logger.error("{}: error occurs when executing statements", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage(), null); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage()), null); } finally { Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_JDBC_BATCH, t1); } @@ -465,11 +436,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { throw new QueryInBatchStmtException("Query statement not allowed in batch: " + statement); } TSExecuteStatementResp resp = executeUpdateStatement(physicalPlan); - if (resp.getStatus().getStatusCode().equals(TS_StatusCode.SUCCESS_STATUS)) { + if (resp.getStatus().getStatusType().getCode() == TSStatusType.SUCCESS_STATUS.getStatusCode()) { result.add(Statement.SUCCESS_NO_INFO); } else { result.add(Statement.EXECUTE_FAILED); - batchErrorMessage.append(resp.getStatus().getErrorMessage()).append("\n"); + batchErrorMessage.append(resp.getStatus().getStatusType().getCode()).append("\n"); return false; } } catch (Exception e) { @@ -491,12 +462,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { try { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN); + return getTSExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } String statement = req.getStatement(); if (execAdminCommand(statement)) { - return getTSExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, "ADMIN_COMMAND_SUCCESS"); + return getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, "ADMIN_COMMAND_SUCCESS")); } if (execShowFlushInfo(statement)) { @@ -505,7 +476,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { FlushTaskPoolManager.getInstance().getTotalTasks(), FlushTaskPoolManager.getInstance().getWorkingTasksNumber(), FlushTaskPoolManager.getInstance().getWaitingTasksNumber()); - return getTSExecuteStatementResp(TS_StatusCode.SUCCESS_WITH_INFO_STATUS, msg); + return getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, msg)); } if (execShowDynamicParameters(statement)) { @@ -519,12 +490,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { MManager.getInstance().getAllStorageGroup().size(), IoTDBConfigDynamicAdapter.getInstance().getTotalTimeseries(), MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups()); - return getTSExecuteStatementResp(TS_StatusCode.SUCCESS_WITH_INFO_STATUS, msg); + return getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, msg)); } if (execSetConsistencyLevel(statement)) { - return getTSExecuteStatementResp(TS_StatusCode.SUCCESS_WITH_INFO_STATUS, - "Execute set consistency level successfully"); + return getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS, + "Execute set consistency level successfully")); } PhysicalPlan physicalPlan; @@ -536,11 +507,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } } catch (IllegalASTFormatException e) { logger.debug("meet error while parsing SQL to physical plan: ", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Statement format is not right:" + e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.SQL_PARSE_ERROR, + "Statement format is not right: " + e.getMessage())); } catch (Exception e) { logger.info("meet error while executing statement: {}", req.getStatement(), e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage())); } } @@ -612,7 +583,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { return resp; } catch (Exception e) { logger.error("{}: Internal server error: ", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.INTERNAL_SERVER_ERROR, e.getMessage())); } finally { Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_QUERY, t1); } @@ -622,7 +593,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { public TSExecuteStatementResp executeQueryStatement(TSExecuteStatementReq req) { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN); + return getTSExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } String statement = req.getStatement(); @@ -631,12 +602,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { physicalPlan = processor.parseSQLToPhysicalPlan(statement, zoneIds.get()); } catch (QueryProcessorException | ArgsErrorException | MetadataErrorException e) { logger.error("meet error while parsing SQL to physical plan!", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.SQL_PARSE_ERROR, e.getMessage())); } if (!physicalPlan.isQuery()) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Statement is not a query statement."); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, + "Statement is not a query statement.")); } return executeQueryStatement(statement, physicalPlan); } @@ -650,7 +621,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { } private TSExecuteStatementResp executeAuthQuery(PhysicalPlan plan, List columns) { - TSExecuteStatementResp resp = getTSExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, ""); + TSExecuteStatementResp resp = getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS)); resp.setIgnoreTimeStamp(true); AuthorPlan authorPlan = (AuthorPlan) plan; switch (authorPlan.getAuthorType()) { @@ -674,8 +645,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { columns.add(PRIVILEGE); break; default: - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, String.format("%s is not an " - + "auth query", authorPlan.getAuthorType())); + return getTSExecuteStatementResp(getStatus(TSStatusType.SQL_PARSE_ERROR, + String.format("%s is not an auth query", authorPlan.getAuthorType()))); } return resp; } @@ -686,7 +657,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { // check seriesPath exists if (paths.isEmpty()) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, "Timeseries does not exist."); + return getTSExecuteStatementResp(getStatus(TSStatusType.TIMESERIES_NOT_EXIST_ERROR)); } // check file level set @@ -694,16 +665,15 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { checkFileLevelSet(paths); } catch (PathErrorException e) { logger.error("meet error while checking file level.", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.CHECK_FILE_LEVEL_ERROR, e.getMessage())); } // check permissions if (!checkAuthorization(paths, plan)) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "No permissions for this query."); + return getTSExecuteStatementResp(getStatus(TSStatusType.NO_PERMISSION_ERROR)); } - TSExecuteStatementResp resp = getTSExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, ""); + TSExecuteStatementResp resp = getTSExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS)); // Restore column header of aggregate to func(column_name), only // support single aggregate function for now switch (plan.getOperatorType()) { @@ -739,12 +709,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { public TSFetchResultsResp fetchResults(TSFetchResultsReq req) { try { if (!checkLogin()) { - return getTSFetchResultsResp(TS_StatusCode.ERROR_STATUS, "Not login."); + return getTSFetchResultsResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } String statement = req.getStatement(); if (!queryStatus.get().containsKey(statement)) { - return getTSFetchResultsResp(TS_StatusCode.ERROR_STATUS, "Has not executed statement"); + return getTSFetchResultsResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, "Has not executed statement")); } QueryDataSet queryDataSet; @@ -763,14 +733,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { queryRet.get().remove(statement); } - TSFetchResultsResp resp = getTSFetchResultsResp(TS_StatusCode.SUCCESS_STATUS, - "FetchResult successfully. Has more result: " + hasResultSet); + TSFetchResultsResp resp = getTSFetchResultsResp(getStatus(TSStatusType.SUCCESS_STATUS, + "FetchResult successfully. Has more result: " + hasResultSet)); resp.setHasResultSet(hasResultSet); resp.setQueryDataSet(result); return resp; } catch (Exception e) { logger.error("{}: Internal server error: ", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSFetchResultsResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSFetchResultsResp(getStatus(TSStatusType.INTERNAL_SERVER_ERROR, e.getMessage())); } } @@ -805,13 +775,13 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { try { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN); + return getTSExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } String statement = req.getStatement(); return executeUpdateStatement(statement); } catch (Exception e) { logger.error("{}: server Internal Error: ", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.INTERNAL_SERVER_ERROR, e.getMessage())); } } @@ -819,13 +789,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { List paths = plan.getPaths(); try { if (!checkAuthorization(paths, plan)) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "No permissions for this operation " + plan.getOperatorType()); + return getTSExecuteStatementResp(getStatus(TSStatusType.NO_PERMISSION_ERROR, plan.getOperatorType().toString())); } } catch (AuthException e) { logger.error("meet error while checking authorization.", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Uninitialized authorizer " + e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.UNINITIALIZED_AUTH_ERROR, e.getMessage())); } // TODO // In current version, we only return OK/ERROR @@ -835,12 +803,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { execRet = executeNonQuery(plan); } catch (ProcessorException e) { logger.debug("meet error while processing non-query. ", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage())); } - TS_StatusCode statusCode = execRet ? TS_StatusCode.SUCCESS_STATUS : TS_StatusCode.ERROR_STATUS; + TSStatusType statusType = execRet ? TSStatusType.SUCCESS_STATUS : TSStatusType.EXECUTE_STATEMENT_ERROR; String msg = execRet ? "Execute successfully" : "Execute statement error."; - TSExecuteStatementResp resp = getTSExecuteStatementResp(statusCode, msg); + TSExecuteStatementResp resp = getTSExecuteStatementResp(getStatus(statusType, msg)); TSHandleIdentifier operationId = new TSHandleIdentifier( ByteBuffer.wrap(username.get().getBytes()), ByteBuffer.wrap("PASS".getBytes())); @@ -865,12 +833,12 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { physicalPlan = processor.parseSQLToPhysicalPlan(statement, zoneIds.get()); } catch (QueryProcessorException | ArgsErrorException | MetadataErrorException e) { logger.error("meet error while parsing SQL to physical plan!", e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.SQL_PARSE_ERROR, e.getMessage())); } if (physicalPlan.isQuery()) { - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Statement is a query statement."); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, + "Statement is a query statement.")); } return executeUpdateStatement(physicalPlan); @@ -899,10 +867,9 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { return AuthorityChecker.check(username.get(), paths, plan.getOperatorType(), targetUser); } - private TSExecuteStatementResp getTSExecuteStatementResp(TS_StatusCode code, String msg) { + private TSExecuteStatementResp getTSExecuteStatementResp(TS_Status status) { TSExecuteStatementResp resp = new TSExecuteStatementResp(); - TS_Status tsStatus = new TS_Status(code); - tsStatus.setErrorMessage(msg); + TS_Status tsStatus = new TS_Status(status); resp.setStatus(tsStatus); TSHandleIdentifier operationId = new TSHandleIdentifier( ByteBuffer.wrap(username.get().getBytes()), @@ -912,21 +879,17 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { return resp; } - private TSExecuteBatchStatementResp getTSBathExecuteStatementResp(TS_StatusCode code, - String msg, - List result) { + private TSExecuteBatchStatementResp getTSBatchExecuteStatementResp(TS_Status status, List result) { TSExecuteBatchStatementResp resp = new TSExecuteBatchStatementResp(); - TS_Status tsStatus = new TS_Status(code); - tsStatus.setErrorMessage(msg); + TS_Status tsStatus = new TS_Status(status); resp.setStatus(tsStatus); resp.setResult(result); return resp; } - private TSFetchResultsResp getTSFetchResultsResp(TS_StatusCode code, String msg) { + private TSFetchResultsResp getTSFetchResultsResp(TS_Status status) { TSFetchResultsResp resp = new TSFetchResultsResp(); - TS_Status tsStatus = new TS_Status(code); - tsStatus.setErrorMessage(msg); + TS_Status tsStatus = new TS_Status(status); resp.setStatus(tsStatus); return resp; } @@ -941,12 +904,11 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { TS_Status tsStatus; TSGetTimeZoneResp resp; try { - tsStatus = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + tsStatus = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); resp = new TSGetTimeZoneResp(tsStatus, zoneIds.get().toString()); } catch (Exception e) { logger.error("meet error while generating time zone.", e); - tsStatus = new TS_Status(TS_StatusCode.ERROR_STATUS); - tsStatus.setErrorMessage(e.getMessage()); + tsStatus = getStatus(TSStatusType.GENERATE_TIME_ZONE_ERROR); resp = new TSGetTimeZoneResp(tsStatus, "Unknown time zone"); } return resp; @@ -958,11 +920,10 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { try { String timeZoneID = req.getTimeZone(); zoneIds.set(ZoneId.of(timeZoneID)); - tsStatus = new TS_Status(TS_StatusCode.SUCCESS_STATUS); + tsStatus = new TS_Status(getStatus(TSStatusType.SUCCESS_STATUS)); } catch (Exception e) { logger.error("meet error while setting time zone.", e); - tsStatus = new TS_Status(TS_StatusCode.ERROR_STATUS); - tsStatus.setErrorMessage(e.getMessage()); + tsStatus = getStatus(TSStatusType.SET_TIME_ZONE_ERROR); } return new TSSetTimeZoneResp(tsStatus); } @@ -983,7 +944,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { public TSExecuteStatementResp insert(TSInsertionReq req) { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN); + return getTSExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR)); } long stmtId = req.getStmtId(); @@ -1007,7 +968,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { return executeUpdateStatement(plan); } catch (Exception e) { logger.info("meet error while executing an insertion into {}", req.getDeviceId(), e); - return getTSExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage()); + return getTSExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage())); } } @@ -1017,7 +978,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { try { if (!checkLogin()) { logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME); - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, ERROR_NOT_LOGIN, null); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.NOT_LOGIN_ERROR), null); } BatchInsertPlan batchInsertPlan = new BatchInsertPlan(req.deviceId, req.measurements); @@ -1034,19 +995,19 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { List paths = batchInsertPlan.getPaths(); try { if (!checkAuthorization(paths, batchInsertPlan)) { - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "No permissions for this operation " + batchInsertPlan.getOperatorType(), null); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.NO_PERMISSION_ERROR, + "No permissions for this operation " + batchInsertPlan.getOperatorType()), null); } } catch (AuthException e) { logger.error("meet error while checking authorization.", e); - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, - "Uninitialized authorizer " + e.getMessage(), null); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.UNINITIALIZED_AUTH_ERROR, + "Uninitialized authorizer " + e.getMessage()), null); } Integer[] results = processor.getExecutor().insertBatch(batchInsertPlan); for (Integer result : results) { - if (result != TS_StatusCode.SUCCESS_STATUS.getValue()) { + if (result != TSStatusType.SUCCESS_STATUS.getStatusCode()) { isAllSuccessful = false; break; } @@ -1054,14 +1015,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext { if (isAllSuccessful) { logger.debug("Insert one RowBatch successfully"); - return getTSBathExecuteStatementResp(TS_StatusCode.SUCCESS_STATUS, "", Arrays.asList(results)); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.SUCCESS_STATUS), Arrays.asList(results)); } else { logger.debug("Insert one RowBatch failed!"); - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, "", Arrays.asList(results)); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.INTERNAL_SERVER_ERROR), Arrays.asList(results)); } } catch (Exception e) { logger.error("{}: error occurs when executing statements", IoTDBConstant.GLOBAL_DB_NAME, e); - return getTSBathExecuteStatementResp(TS_StatusCode.ERROR_STATUS, e.getMessage(), null); + return getTSBatchExecuteStatementResp(getStatus(TSStatusType.EXECUTE_STATEMENT_ERROR, e.getMessage()), null); } finally { Measurement.INSTANCE.addOperationLatency(Operation.EXECUTE_RPC_BATCH_INSERT, t1); } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java index 1f4dc0261ee8aa7cc72b9a4ed8ebfe73b1a0d086..50f5cd091aeaae999e161d1b08388695f7e664ed 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxSeriesMergeFileSelectorTest.java @@ -97,7 +97,7 @@ public class MaxSeriesMergeFileSelectorTest extends MergeTest{ List unseqSelected = result[1]; assertEquals(seqResources.subList(0, 1), seqSelected); assertEquals(unseqResources.subList(0, 1), unseqSelected); - assertEquals(12, mergeFileSelector.getConcurrentMergeNum()); + assertEquals(34, mergeFileSelector.getConcurrentMergeNum()); resource.clear(); } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java index dcc7653f55885cebf9494032cd771186848a9148..ab613f3c993c1cad2f6f57e796cb7631a34cec6f 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java @@ -471,7 +471,7 @@ public class IoTDBAggregationIT { resultSet.next(); fail(); } catch (Exception e) { - Assert.assertEquals("org.apache.iotdb.rpc.IoTDBRPCException: Unsupported data type in aggregation MEAN : TEXT", e.getMessage()); + Assert.assertEquals("Internal server error: Unsupported data type in aggregation MEAN : TEXT", e.getMessage()); } hasResultSet = statement.execute("select sum(s3)" + @@ -481,7 +481,7 @@ public class IoTDBAggregationIT { resultSet.next(); fail(); } catch (Exception e) { - Assert.assertEquals("org.apache.iotdb.rpc.IoTDBRPCException: Unsupported data type in aggregation SUM : TEXT", e.getMessage()); + Assert.assertEquals("Internal server error: Unsupported data type in aggregation SUM : TEXT", e.getMessage()); } hasResultSet = statement.execute("select mean(s4)" + @@ -492,7 +492,7 @@ public class IoTDBAggregationIT { resultSet.next(); fail(); } catch (Exception e) { - Assert.assertEquals("org.apache.iotdb.rpc.IoTDBRPCException: Unsupported data type in aggregation MEAN : BOOLEAN", e.getMessage()); + Assert.assertEquals("Internal server error: Unsupported data type in aggregation MEAN : BOOLEAN", e.getMessage()); } hasResultSet = statement.execute("select sum(s4)" + @@ -502,7 +502,7 @@ public class IoTDBAggregationIT { resultSet.next(); fail(); } catch (Exception e) { - Assert.assertEquals("org.apache.iotdb.rpc.IoTDBRPCException: Unsupported data type in aggregation SUM : BOOLEAN", e.getMessage()); + Assert.assertEquals("Internal server error: Unsupported data type in aggregation SUM : BOOLEAN", e.getMessage()); } } catch (Exception e) { e.printStackTrace(); diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java index 908812c66039efdbcb9392194a1f1bba6a5e7f91..74df2bc5d509fd2ebc132ddec8d85c2095317b7f 100644 --- a/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java @@ -21,7 +21,6 @@ package org.apache.iotdb.rpc; import java.lang.reflect.Proxy; import org.apache.iotdb.service.rpc.thrift.TSIService; import org.apache.iotdb.service.rpc.thrift.TS_Status; -import org.apache.iotdb.service.rpc.thrift.TS_StatusCode; public class RpcUtils { @@ -36,8 +35,8 @@ public class RpcUtils { * @param status -status */ public static void verifySuccess(TS_Status status) throws IoTDBRPCException { - if (status.getStatusCode() != TS_StatusCode.SUCCESS_STATUS) { - throw new IoTDBRPCException(status.errorMessage); + if (status.getStatusType().getCode() != TSStatusType.SUCCESS_STATUS.getStatusCode()) { + throw new IoTDBRPCException(status.getStatusType().getMessage()); } } diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusType.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusType.java new file mode 100644 index 0000000000000000000000000000000000000000..5a4acf6b4636b80d1b5b78222fa9ad8b86c6c592 --- /dev/null +++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusType.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.rpc; + +public enum TSStatusType { + SUCCESS_STATUS(200, ""), + STILL_EXECUTING_STATUS(201, ""), + INVALID_HANDLE_STATUS(202, ""), + TIMESERIES_NOT_EXIST_ERROR(301, "Timeseries does not exist"), + UNSUPPORTED_FETCH_METADATA_OPERATION_ERROR(302, "Unsupported fetch metadata operation"), + FETCH_METADATA_ERROR(303, "Failed to fetch metadata"), + CHECK_FILE_LEVEL_ERROR(304, "Meet error while checking file level"), + EXECUTE_STATEMENT_ERROR(400, "Execute statement error"), + SQL_PARSE_ERROR(401, "Meet error while parsing SQL"), + GENERATE_TIME_ZONE_ERROR(402, "Meet error while generating time zone"), + SET_TIME_ZONE_ERROR(403, "Meet error while setting time zone"), + INTERNAL_SERVER_ERROR(500, "Internal server error"), + WRONG_LOGIN_PASSWORD_ERROR(600, "Username or password is wrong"), + NOT_LOGIN_ERROR(601, "Has not logged in"), + NO_PERMISSION_ERROR(602, "No permissions for this operation"), + UNINITIALIZED_AUTH_ERROR(603, "Uninitialized authorizer"); + + private int statusCode; + private String statusMessage; + + private TSStatusType(int statusCode, String statusMessage) { + this.statusCode = statusCode; + this.statusMessage = statusMessage; + } + + public int getStatusCode() { + return statusCode; + } + + public String getStatusMessage() { + return statusMessage; + } +} diff --git a/service-rpc/src/main/thrift/rpc.thrift b/service-rpc/src/main/thrift/rpc.thrift index a96284bfe6cec620d1fb92b11fa3e2c84c487d2a..13aae094eb5958109dbc3efc95683e550c045196 100644 --- a/service-rpc/src/main/thrift/rpc.thrift +++ b/service-rpc/src/main/thrift/rpc.thrift @@ -18,28 +18,17 @@ */ namespace java org.apache.iotdb.service.rpc.thrift - -// The return status code contained in each response. -enum TS_StatusCode { - SUCCESS_STATUS, - SUCCESS_WITH_INFO_STATUS, - STILL_EXECUTING_STATUS, - ERROR_STATUS, - INVALID_HANDLE_STATUS +// The return status code and message in each response. +struct TS_StatusType { + 1: required i32 code + 2: required string message } // The return status of a remote request struct TS_Status { - 1: required TS_StatusCode statusCode - - // If status is SUCCESS_WITH_INFO, info_msgs may be populated with - // additional diagnostic information. + 1: required TS_StatusType statusType 2: optional list infoMessages - - // If status is ERROR, then the following fields may be set 3: optional string sqlState // as defined in the ISO/IEF CLIENT specification - 4: optional i32 errorCode // internal error code - 5: optional string errorMessage } struct TSHandleIdentifier { diff --git a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java index 5f6da88b1abaea516fcd948ad5597e3c0acdc830..9aab7f87c19187c6d155e36981abb7acf8f37c21 100644 --- a/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java +++ b/spark-tsfile/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java @@ -64,14 +64,14 @@ public class HDFSInputTest { @Test public void test_read1() throws IOException { - int size = 2000; + int size = 1000; ByteBuffer buffer = ByteBuffer.allocate(size); Assert.assertEquals(size, in.read(buffer)); } @Test public void test_read2() throws IOException { - int size = 2000; + int size = 1000; long pos = 20L; ByteBuffer buffer = ByteBuffer.allocate(size); Assert.assertEquals(size, in.read(buffer, pos)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDigest.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDigest.java index 1b27efaf527a45fafc908c7d127f6c8268b8b524..0fa252bee69d3e17e0819df8c95cbdfe7e2769fc 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDigest.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsDigest.java @@ -23,10 +23,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; +import java.util.Arrays; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; /** @@ -34,11 +31,15 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class TsDigest { - private Map statistics; + private ByteBuffer[] statistics; - private int serializedSize = Integer.BYTES; + /** + * size of valid values in statistics. Note that some values in statistics can be null and thus + * invalid. + */ + private int validSizeOfArray = 0; - private int sizeOfList; + private int serializedSize = Integer.BYTES; // initialize for number of statistics public TsDigest() { // allowed to clair an empty TsDigest whose fields will be assigned later. @@ -66,17 +67,18 @@ public class TsDigest { TsDigest digest = new TsDigest(); int size = ReadWriteIOUtils.readInt(inputStream); + digest.validSizeOfArray = size; + digest.serializedSize = Integer.BYTES; if (size > 0) { - Map statistics = new HashMap<>(); - String key; + digest.statistics = new ByteBuffer[StatisticType.getTotalTypeNum()]; ByteBuffer value; for (int i = 0; i < size; i++) { - key = ReadWriteIOUtils.readString(inputStream); + short n = ReadWriteIOUtils.readShort(inputStream); value = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(inputStream); - statistics.put(key, value); + digest.statistics[n] = value; + digest.serializedSize += Short.BYTES + Integer.BYTES + value.remaining(); } - digest.statistics = statistics; - } + } // else left digest.statistics as null return digest; } @@ -91,69 +93,56 @@ public class TsDigest { TsDigest digest = new TsDigest(); int size = ReadWriteIOUtils.readInt(buffer); + digest.validSizeOfArray = size; + digest.serializedSize = Integer.BYTES; if (size > 0) { - Map statistics = new HashMap<>(); - String key; + digest.statistics = new ByteBuffer[StatisticType.getTotalTypeNum()]; ByteBuffer value; for (int i = 0; i < size; i++) { - key = ReadWriteIOUtils.readString(buffer); + short n = ReadWriteIOUtils.readShort(buffer); value = ReadWriteIOUtils.readByteBufferWithSelfDescriptionLength(buffer); - statistics.put(key, value); + digest.statistics[n] = value; + digest.serializedSize += Short.BYTES + Integer.BYTES + value.remaining(); } - digest.statistics = statistics; - } + } // else left digest.statistics as null return digest; } - private void reCalculateSerializedSize() { + private void reCalculate() { + validSizeOfArray = 0; serializedSize = Integer.BYTES; if (statistics != null) { - for (Map.Entry entry : statistics.entrySet()) { - serializedSize += Integer.BYTES + entry.getKey().length() + Integer.BYTES - + entry.getValue().remaining(); + for (ByteBuffer value : statistics) { + if (value != null) { + // StatisticType serialized value, byteBuffer.capacity and byteBuffer.array + serializedSize += Short.BYTES + Integer.BYTES + value.remaining(); + validSizeOfArray++; + } } - sizeOfList = statistics.size(); - } else { - sizeOfList = 0; } } /** * get statistics of the current object. - * - * @return -unmodifiableMap of the current object's statistics */ - public Map getStatistics() { - if (statistics == null) { - return null; - } - return Collections.unmodifiableMap(this.statistics); + public ByteBuffer[] getStatistics() { + return statistics; //TODO unmodifiable } - public void setStatistics(Map statistics) { - this.statistics = statistics; - reCalculateSerializedSize(); - } - - /** - * add statistics using given param. - * - * @param key -key of the entry - * @param value -value of the entry - */ - public void addStatistics(String key, ByteBuffer value) { - if (statistics == null) { - statistics = new HashMap<>(); + public void setStatistics(ByteBuffer[] statistics) throws IOException { + if (statistics != null && statistics.length != StatisticType.getTotalTypeNum()) { + throw new IOException(String.format( + "The length of array of statistics doesn't equal StatisticType.getTotalTypeNum() %d", + StatisticType.getTotalTypeNum())); } - statistics.put(key, value); - serializedSize += Integer.BYTES + key.length() + Integer.BYTES + value.remaining(); - sizeOfList++; + this.statistics = statistics; + reCalculate(); // DO NOT REMOVE THIS } @Override public String toString() { - return statistics != null ? statistics.toString() : ""; + return statistics != null ? Arrays.toString(statistics) : ""; } /** @@ -163,20 +152,16 @@ public class TsDigest { * @return -byte length */ public int serializeTo(OutputStream outputStream) throws IOException { - if ((statistics != null && sizeOfList != statistics.size()) || (statistics == null - && sizeOfList != 0)) { - reCalculateSerializedSize(); - } int byteLen = 0; - if (statistics == null || statistics.size() == 0) { + if (validSizeOfArray == 0) { byteLen += ReadWriteIOUtils.write(0, outputStream); } else { - byteLen += ReadWriteIOUtils.write(statistics.size(), outputStream); - for (Map.Entry entry : statistics.entrySet()) { - byteLen += ReadWriteIOUtils - .write(entry.getKey(), outputStream); - byteLen += ReadWriteIOUtils - .write(entry.getValue(), outputStream); + byteLen += ReadWriteIOUtils.write(validSizeOfArray, outputStream); + for (int i = 0; i < statistics.length; i++) { + if (statistics[i] != null) { + byteLen += ReadWriteIOUtils.write((short) i, outputStream); + byteLen += ReadWriteIOUtils.write(statistics[i], outputStream); + } } } return byteLen; @@ -189,20 +174,16 @@ public class TsDigest { * @return -byte length */ public int serializeTo(ByteBuffer buffer) { - if ((statistics != null && sizeOfList != statistics.size()) || (statistics == null - && sizeOfList != 0)) { - reCalculateSerializedSize(); - } int byteLen = 0; - - if (statistics == null || statistics.size() == 0) { + if (validSizeOfArray == 0) { byteLen += ReadWriteIOUtils.write(0, buffer); } else { - byteLen += ReadWriteIOUtils.write(statistics.size(), buffer); - for (Map.Entry entry : statistics.entrySet()) { - byteLen += ReadWriteIOUtils.write(entry.getKey(), buffer); - byteLen += ReadWriteIOUtils - .write(entry.getValue(), buffer); + byteLen += ReadWriteIOUtils.write(validSizeOfArray, buffer); + for (int i = 0; i < statistics.length; i++) { + if (statistics[i] != null) { + byteLen += ReadWriteIOUtils.write((short) i, buffer); + byteLen += ReadWriteIOUtils.write(statistics[i], buffer); + } } } return byteLen; @@ -214,9 +195,6 @@ public class TsDigest { * @return -serializedSize */ public int getSerializedSize() { - if (statistics == null || (sizeOfList != statistics.size())) { - reCalculateSerializedSize(); - } return serializedSize; } @@ -229,17 +207,44 @@ public class TsDigest { return false; } TsDigest digest = (TsDigest) o; - if (serializedSize != digest.serializedSize || sizeOfList != digest.sizeOfList - || statistics.size() != digest.statistics.size()) { + if (serializedSize != digest.serializedSize || validSizeOfArray != digest.validSizeOfArray + || ((statistics == null) ^ (digest.statistics == null))) { return false; } - for (Entry entry : statistics.entrySet()) { - String key = entry.getKey(); - ByteBuffer value = entry.getValue(); - if (!digest.statistics.containsKey(key) || !value.equals(digest.statistics.get(key))) { - return false; + + if (statistics != null) { + for (int i = 0; i < statistics.length; i++) { + if ((statistics[i] == null) ^ (digest.statistics[i] == null)) { + // one is null and the other is not null + return false; + } + if (statistics[i] != null) { + if (!statistics[i].equals(digest.statistics[i])) { + return false; + } + } } } return true; } + + public enum StatisticType { + min_value, max_value, first_value, last_value, sum_value; + + public static int getTotalTypeNum() { + return StatisticType.values().length; + } + + public static StatisticType deserialize(short i) { + return StatisticType.values()[i]; + } + + public static int getSerializedSize() { + return Short.BYTES; + } + + public short serialize() { + return (short) this.ordinal(); + } + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java index 2a57fc256b272d7560cbae67cbf13c66151f71c6..54fd32cb14b7f16ebc40d03cbbd73bb192fabd4f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BinaryStatistics.java @@ -33,16 +33,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class BinaryStatistics extends Statistics { - private Binary max = new Binary(""); private Binary min = new Binary(""); + private Binary max = new Binary(""); private Binary first = new Binary(""); - private double sum;// FIXME sum is meaningless private Binary last = new Binary(""); + private double sum;// FIXME sum is meaningless @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = new Binary(maxBytes); min = new Binary(minBytes); + max = new Binary(maxBytes); } @Override @@ -61,13 +61,13 @@ public class BinaryStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Binary getLast() { + return last; } @Override - public Binary getLast() { - return last; + public double getSum() { + return sum; } /** @@ -76,15 +76,15 @@ public class BinaryStatistics extends Statistics { * @param min minimum value * @param max maximum value * @param first the first value - * @param sum sum * @param last the last value + * @param sum sum */ - public void initializeStats(Binary min, Binary max, Binary first, double sum, Binary last) { + private void initializeStats(Binary min, Binary max, Binary first, Binary last, double sum) { this.min = min; this.max = max; this.first = first; - this.sum = sum; this.last = last; + this.sum = sum; } @Override @@ -92,23 +92,21 @@ public class BinaryStatistics extends Statistics { BinaryStatistics stringStats = (BinaryStatistics) stats; if (isEmpty) { initializeStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(), - stringStats.getSum(), - stringStats.getLast()); + stringStats.getLast(), stringStats.getSum()); isEmpty = false; } else { updateStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(), - stringStats.getSum(), - stringStats.getLast()); + stringStats.getLast(), stringStats.getSum()); } } @Override public void updateStats(Binary value) { if (isEmpty) { - initializeStats(value, value, value, 0, value); + initializeStats(value, value, value, value, 0); isEmpty = false; } else { - updateStats(value, value, value, 0, value); + updateStats(value, value, value, value, 0); isEmpty = false; } } @@ -117,17 +115,17 @@ public class BinaryStatistics extends Statistics { public void updateStats(Binary[] values) { for (Binary value : values) { if (isEmpty) { - initializeStats(value, value, value, 0, value); + initializeStats(value, value, value, value, 0); isEmpty = false; } else { - updateStats(value, value, value, 0, value); + updateStats(value, value, value, value, 0); isEmpty = false; } } } - private void updateStats(Binary minValue, Binary maxValue, Binary firstValue, double sum, - Binary lastValue) { + private void updateStats(Binary minValue, Binary maxValue, Binary firstValue, Binary lastValue, + double sum) { if (minValue.compareTo(min) < 0) { min = minValue; } @@ -137,24 +135,19 @@ public class BinaryStatistics extends Statistics { this.last = lastValue; } - @Override - public byte[] getMaxBytes() { - return max.getValues(); - } - @Override public byte[] getMinBytes() { return min.getValues(); } @Override - public byte[] getFirstBytes() { - return first.getValues(); + public byte[] getMaxBytes() { + return max.getValues(); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return first.getValues(); } @Override @@ -163,8 +156,8 @@ public class BinaryStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ByteBuffer.wrap(max.getValues()); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -173,13 +166,13 @@ public class BinaryStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ByteBuffer.wrap(first.getValues()); + public ByteBuffer getMaxBytebuffer() { + return ByteBuffer.wrap(max.getValues()); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ByteBuffer.wrap(first.getValues()); } @Override @@ -187,6 +180,11 @@ public class BinaryStatistics extends Statistics { return ByteBuffer.wrap(last.getValues()); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override public int sizeOfDatum() { return -1; @@ -194,7 +192,7 @@ public class BinaryStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + "]"; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java index ff04992bfbc9d442d6cc161c614f58dea9cc1b7d..7c2a7e8ce4ff28c01a9f293ae12e6a217b7b29a7 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/BooleanStatistics.java @@ -31,25 +31,25 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class BooleanStatistics extends Statistics { - private boolean max; private boolean min; + private boolean max; private boolean first; - private double sum; private boolean last; + private double sum; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToBool(maxBytes); min = BytesUtils.bytesToBool(minBytes); + max = BytesUtils.bytesToBool(maxBytes); } @Override public void updateStats(boolean value) { if (isEmpty) { - initializeStats(value, value, value, 0, value); + initializeStats(value, value, value, value, 0); isEmpty = false; } else { - updateStats(value, value, value, 0, value); + updateStats(value, value, value, value, 0); isEmpty = false; } } @@ -58,17 +58,17 @@ public class BooleanStatistics extends Statistics { public void updateStats(boolean[] values) { for (boolean value : values) { if (isEmpty) { - initializeStats(value, value, value, 0, value); + initializeStats(value, value, value, value, 0); isEmpty = false; } else { - updateStats(value, value, value, 0, value); + updateStats(value, value, value, value, 0); isEmpty = false; } } } - private void updateStats(boolean minValue, boolean maxValue, boolean firstValue, double sumValue, - boolean lastValue) { + private void updateStats(boolean minValue, boolean maxValue, boolean firstValue, + boolean lastValue, double sumValue) { if (!minValue && min) { min = false; } @@ -78,24 +78,19 @@ public class BooleanStatistics extends Statistics { this.last = lastValue; } - @Override - public Boolean getMax() { - return max; - } - @Override public Boolean getMin() { return min; } @Override - public Boolean getFirst() { - return first; + public Boolean getMax() { + return max; } @Override - public double getSum() { - return sum; + public Boolean getFirst() { + return first; } @Override @@ -104,8 +99,8 @@ public class BooleanStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + public double getSum() { + return sum; } @Override @@ -114,13 +109,13 @@ public class BooleanStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -128,17 +123,21 @@ public class BooleanStatistics extends Statistics { return ReadWriteIOUtils.getByteBuffer(last); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override protected void mergeStatisticsValue(Statistics stats) { BooleanStatistics boolStats = (BooleanStatistics) stats; if (isEmpty) { initializeStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), - boolStats.getSum(), - boolStats.getLast()); + boolStats.getLast(), boolStats.getSum()); isEmpty = false; } else { - updateStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), boolStats.getSum(), - boolStats.getLast()); + updateStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), + boolStats.getLast(), boolStats.getSum()); } } @@ -148,11 +147,11 @@ public class BooleanStatistics extends Statistics { * @param min min boolean * @param max max boolean * @param firstValue first boolean value - * @param sumValue sum value (double type) * @param lastValue last boolean value + * @param sumValue sum value (double type) */ - public void initializeStats(boolean min, boolean max, boolean firstValue, double sumValue, - boolean lastValue) { + private void initializeStats(boolean min, boolean max, boolean firstValue, boolean lastValue, + double sumValue) { this.min = min; this.max = max; this.first = firstValue; @@ -160,13 +159,13 @@ public class BooleanStatistics extends Statistics { } @Override - public byte[] getMaxBytes() { - return BytesUtils.boolToBytes(max); + public byte[] getMinBytes() { + return BytesUtils.boolToBytes(min); } @Override - public byte[] getMinBytes() { - return BytesUtils.boolToBytes(min); + public byte[] getMaxBytes() { + return BytesUtils.boolToBytes(max); } @Override @@ -175,13 +174,13 @@ public class BooleanStatistics extends Statistics { } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getLastBytes() { + return BytesUtils.boolToBytes(last); } @Override - public byte[] getLastBytes() { - return BytesUtils.boolToBytes(last); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -191,7 +190,7 @@ public class BooleanStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + "]"; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java index 57d500e1d54624e94e946bc979faf3fe8d6e095a..e0e2998342a9788668750a196a30cb988facf4e5 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatistics.java @@ -31,16 +31,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class DoubleStatistics extends Statistics { - private double max; private double min; + private double max; private double first; - private double sum; private double last; + private double sum; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToDouble(maxBytes); min = BytesUtils.bytesToDouble(minBytes); + max = BytesUtils.bytesToDouble(maxBytes); } @Override @@ -65,8 +65,8 @@ public class DoubleStatistics extends Statistics { } } - private void updateStats(double minValue, double maxValue, double firstValue, double sumValue, - double lastValue) { + private void updateStats(double minValue, double maxValue, double firstValue, double lastValue, + double sumValue) { if (minValue < min) { min = minValue; } @@ -78,13 +78,13 @@ public class DoubleStatistics extends Statistics { } @Override - public Double getMax() { - return max; + public Double getMin() { + return min; } @Override - public Double getMin() { - return min; + public Double getMax() { + return max; } @Override @@ -93,13 +93,13 @@ public class DoubleStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Double getLast() { + return last; } @Override - public Double getLast() { - return last; + public double getSum() { + return sum; } @Override @@ -107,13 +107,11 @@ public class DoubleStatistics extends Statistics { DoubleStatistics doubleStats = (DoubleStatistics) stats; if (this.isEmpty) { initializeStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(), - doubleStats.getSum(), - doubleStats.getLast()); + doubleStats.getLast(), doubleStats.getSum()); isEmpty = false; } else { updateStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(), - doubleStats.getSum(), - doubleStats.getLast()); + doubleStats.getLast(), doubleStats.getSum()); } } @@ -124,20 +122,15 @@ public class DoubleStatistics extends Statistics { * @param min min value * @param max max value * @param first the first value - * @param sum sum value * @param last the last value + * @param sum sum value */ - public void initializeStats(double min, double max, double first, double sum, double last) { + private void initializeStats(double min, double max, double first, double last, double sum) { this.min = min; this.max = max; this.first = first; - this.sum = sum; this.last = last; - } - - @Override - public byte[] getMaxBytes() { - return BytesUtils.doubleToBytes(max); + this.sum = sum; } @Override @@ -146,13 +139,13 @@ public class DoubleStatistics extends Statistics { } @Override - public byte[] getFirstBytes() { - return BytesUtils.doubleToBytes(first); + public byte[] getMaxBytes() { + return BytesUtils.doubleToBytes(max); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return BytesUtils.doubleToBytes(first); } @Override @@ -161,8 +154,8 @@ public class DoubleStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -171,13 +164,13 @@ public class DoubleStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -185,6 +178,11 @@ public class DoubleStatistics extends Statistics { return ReadWriteIOUtils.getByteBuffer(last); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override public int sizeOfDatum() { return 8; @@ -192,7 +190,7 @@ public class DoubleStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + "]"; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java index 2fa13f6ebc292b766ba013fa6ea7cfd698a972e4..b995d6d6c5780b557f7a11a8514c3b00bdf3cf3d 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatistics.java @@ -31,16 +31,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class FloatStatistics extends Statistics { - private float max; private float min; + private float max; private float first; private double sum; private float last; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToFloat(maxBytes); min = BytesUtils.bytesToFloat(minBytes); + max = BytesUtils.bytesToFloat(maxBytes); } @Override @@ -65,8 +65,8 @@ public class FloatStatistics extends Statistics { } } - private void updateStats(float minValue, float maxValue, float firstValue, - double sumValue, float last) { + private void updateStats(float minValue, float maxValue, float firstValue, float last, + double sumValue) { if (minValue < min) { min = minValue; } @@ -78,13 +78,13 @@ public class FloatStatistics extends Statistics { } @Override - public Float getMax() { - return max; + public Float getMin() { + return min; } @Override - public Float getMin() { - return min; + public Float getMax() { + return max; } @Override @@ -93,13 +93,13 @@ public class FloatStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Float getLast() { + return last; } @Override - public Float getLast() { - return last; + public double getSum() { + return sum; } @Override @@ -107,28 +107,21 @@ public class FloatStatistics extends Statistics { FloatStatistics floatStats = (FloatStatistics) stats; if (isEmpty) { initializeStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(), - floatStats.getSum(), - floatStats.getLast()); + floatStats.getLast(), floatStats.getSum()); isEmpty = false; } else { updateStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(), - floatStats.getSum(), - floatStats.getLast()); + floatStats.getLast(), floatStats.getSum()); } } - public void initializeStats(float min, float max, float first, double sum, float last) { + private void initializeStats(float min, float max, float first, float last, double sum) { this.min = min; this.max = max; this.first = first; - this.sum = sum; this.last = last; - } - - @Override - public byte[] getMaxBytes() { - return BytesUtils.floatToBytes(max); + this.sum = sum; } @Override @@ -137,13 +130,13 @@ public class FloatStatistics extends Statistics { } @Override - public byte[] getFirstBytes() { - return BytesUtils.floatToBytes(first); + public byte[] getMaxBytes() { + return BytesUtils.floatToBytes(max); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return BytesUtils.floatToBytes(first); } @Override @@ -152,8 +145,8 @@ public class FloatStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -162,13 +155,13 @@ public class FloatStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -176,6 +169,11 @@ public class FloatStatistics extends Statistics { return ReadWriteIOUtils.getByteBuffer(last); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override public int sizeOfDatum() { return 4; @@ -183,8 +181,8 @@ public class FloatStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" - + first + ",sum:" + sum + ",last:" + last + "]"; + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + + "]"; } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java index 9726e3c44a75f8dc59e07d8d4d2aa5a9085439cc..bb81e677b13b0375171288b9641007cdc572479a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatistics.java @@ -31,16 +31,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class IntegerStatistics extends Statistics { - private int max; private int min; + private int max; private int first; - private double sum; private int last; + private double sum; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToInt(maxBytes); min = BytesUtils.bytesToInt(minBytes); + max = BytesUtils.bytesToInt(maxBytes); } @Override @@ -67,8 +67,8 @@ public class IntegerStatistics extends Statistics { } } - private void updateStats(int minValue, int maxValue, - int firstValue, double sumValue, int lastValue) { + private void updateStats(int minValue, int maxValue, int firstValue, int lastValue, + double sumValue) { // TODO: unused parameter if (minValue < min) { min = minValue; @@ -81,13 +81,13 @@ public class IntegerStatistics extends Statistics { } @Override - public Integer getMax() { - return max; + public Integer getMin() { + return min; } @Override - public Integer getMin() { - return min; + public Integer getMax() { + return max; } @Override @@ -96,40 +96,35 @@ public class IntegerStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Integer getLast() { + return last; } @Override - public Integer getLast() { - return last; + public double getSum() { + return sum; } @Override protected void mergeStatisticsValue(Statistics stats) { IntegerStatistics intStats = (IntegerStatistics) stats; if (isEmpty) { - initializeStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getSum(), - intStats.getLast()); + initializeStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getLast(), + intStats.getSum()); isEmpty = false; } else { - updateStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getSum(), - intStats.getLast()); + updateStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getLast(), + intStats.getSum()); } } - void initializeStats(int min, int max, int first, double sum, int last) { + private void initializeStats(int min, int max, int first, int last, double sum) { this.min = min; this.max = max; this.first = first; - this.sum = sum; this.last = last; - } - - @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + this.sum = sum; } @Override @@ -138,13 +133,13 @@ public class IntegerStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -153,8 +148,8 @@ public class IntegerStatistics extends Statistics { } @Override - public byte[] getMaxBytes() { - return BytesUtils.intToBytes(max); + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); } @Override @@ -163,13 +158,13 @@ public class IntegerStatistics extends Statistics { } @Override - public byte[] getFirstBytes() { - return BytesUtils.intToBytes(first); + public byte[] getMaxBytes() { + return BytesUtils.intToBytes(max); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return BytesUtils.intToBytes(first); } @Override @@ -177,6 +172,11 @@ public class IntegerStatistics extends Statistics { return BytesUtils.intToBytes(last); } + @Override + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); + } + @Override public int sizeOfDatum() { return 4; @@ -184,8 +184,8 @@ public class IntegerStatistics extends Statistics { @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" - + first + ",sum:" + sum + ",last:" + last + "]"; + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + + "]"; } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java index 1c9ba6474a027173cc66fd434c8c26e051ebc126..d8e401f78e9e217eb33d1d03e168303fd39f950a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatistics.java @@ -31,16 +31,16 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public class LongStatistics extends Statistics { - private long max; private long min; + private long max; private long first; - private double sum; private long last; + private double sum; @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { - max = BytesUtils.bytesToLong(maxBytes); min = BytesUtils.bytesToLong(minBytes); + max = BytesUtils.bytesToLong(maxBytes); } @Override @@ -59,13 +59,13 @@ public class LongStatistics extends Statistics { } @Override - public double getSum() { - return sum; + public Long getLast() { + return last; } @Override - public Long getLast() { - return last; + public double getSum() { + return sum; } @Override @@ -90,8 +90,8 @@ public class LongStatistics extends Statistics { } } - private void updateStats(long minValue, long maxValue, long firstValue, double sumValue, - long lastValue) { + private void updateStats(long minValue, long maxValue, long firstValue, long lastValue, + double sumValue) { if (minValue < min) { min = minValue; } @@ -117,27 +117,21 @@ public class LongStatistics extends Statistics { LongStatistics longStats = (LongStatistics) stats; if (isEmpty) { initializeStats(longStats.getMin(), longStats.getMax(), longStats.getFirst(), - longStats.getSum(), - longStats.getLast()); + longStats.getLast(), longStats.getSum()); isEmpty = false; } else { - updateStats(longStats.getMin(), longStats.getMax(), longStats.getFirst(), longStats.getSum(), - longStats.getLast()); + updateStats(longStats.getMin(), longStats.getMax(), longStats.getFirst(), longStats.getLast(), + longStats.getSum()); } } - void initializeStats(long min, long max, long firstValue, double sum, long last) { + private void initializeStats(long min, long max, long firstValue, long last, double sum) { this.min = min; this.max = max; this.first = firstValue; - this.sum += sum; this.last = last; - } - - @Override - public byte[] getMaxBytes() { - return BytesUtils.longToBytes(max); + this.sum += sum; } @Override @@ -146,13 +140,13 @@ public class LongStatistics extends Statistics { } @Override - public byte[] getFirstBytes() { - return BytesUtils.longToBytes(first); + public byte[] getMaxBytes() { + return BytesUtils.longToBytes(max); } @Override - public byte[] getSumBytes() { - return BytesUtils.doubleToBytes(sum); + public byte[] getFirstBytes() { + return BytesUtils.longToBytes(first); } @Override @@ -161,8 +155,8 @@ public class LongStatistics extends Statistics { } @Override - public ByteBuffer getMaxBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(max); + public byte[] getSumBytes() { + return BytesUtils.doubleToBytes(sum); } @Override @@ -171,13 +165,13 @@ public class LongStatistics extends Statistics { } @Override - public ByteBuffer getFirstBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(first); + public ByteBuffer getMaxBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(max); } @Override - public ByteBuffer getSumBytebuffer() { - return ReadWriteIOUtils.getByteBuffer(sum); + public ByteBuffer getFirstBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(first); } @Override @@ -185,9 +179,14 @@ public class LongStatistics extends Statistics { return ReadWriteIOUtils.getByteBuffer(last); } + @Override + public ByteBuffer getSumBytebuffer() { + return ReadWriteIOUtils.getByteBuffer(sum); + } + @Override public String toString() { - return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + return "[min:" + min + ",max:" + max + ",first:" + first + ",last:" + last + ",sum:" + sum + "]"; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java index cdc13ad37320129b3ce04788c87456e59b437857..fb2d3e8cbca861eb784465f14264b17293bb07a1 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java @@ -105,30 +105,30 @@ public abstract class Statistics { public abstract T getFirst(); - public abstract double getSum(); - public abstract T getLast(); - public abstract byte[] getMaxBytes(); + public abstract double getSum(); public abstract byte[] getMinBytes(); - public abstract byte[] getFirstBytes(); + public abstract byte[] getMaxBytes(); - public abstract byte[] getSumBytes(); + public abstract byte[] getFirstBytes(); public abstract byte[] getLastBytes(); - public abstract ByteBuffer getMaxBytebuffer(); + public abstract byte[] getSumBytes(); public abstract ByteBuffer getMinBytebuffer(); - public abstract ByteBuffer getFirstBytebuffer(); + public abstract ByteBuffer getMaxBytebuffer(); - public abstract ByteBuffer getSumBytebuffer(); + public abstract ByteBuffer getFirstBytebuffer(); public abstract ByteBuffer getLastBytebuffer(); + public abstract ByteBuffer getSumBytebuffer(); + /** * merge parameter to this statistic. Including * @@ -262,7 +262,7 @@ public abstract class Statistics { } else if (sizeOfDatum() != -1) { return sizeOfDatum() * 4 + 8; } else { - return 4 * Integer.BYTES + getMaxBytes().length + getMinBytes().length + return 4 * Integer.BYTES + getMinBytes().length + getMaxBytes().length + getFirstBytes().length + getLastBytes().length + getSumBytes().length; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index d2ce51dab25b62316c75b7080c8d861231b7181c..f3694487127a5a4ba94099262ce4a808ca66bad4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -31,7 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.compress.IUnCompressor; import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter; @@ -42,6 +41,7 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; import org.apache.iotdb.tsfile.file.metadata.TsDigest; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -596,16 +596,21 @@ public class TsFileSequenceReader implements AutoCloseable { this.skipPageData(pageHeader); } currentChunk = new ChunkMetaData(measurementID, dataType, fileOffsetOfChunk, - startTimeOfChunk, endTimeOfChunk); + startTimeOfChunk, endTimeOfChunk); currentChunk.setNumOfPoints(numOfPoints); - Map statisticsMap = new HashMap<>(); - statisticsMap.put(StatisticConstant.MAX_VALUE, ByteBuffer.wrap(chunkStatistics.getMaxBytes())); - statisticsMap.put(StatisticConstant.MIN_VALUE, ByteBuffer.wrap(chunkStatistics.getMinBytes())); - statisticsMap.put(StatisticConstant.FIRST, ByteBuffer.wrap(chunkStatistics.getFirstBytes())); - statisticsMap.put(StatisticConstant.SUM, ByteBuffer.wrap(chunkStatistics.getSumBytes())); - statisticsMap.put(StatisticConstant.LAST, ByteBuffer.wrap(chunkStatistics.getLastBytes())); + ByteBuffer[] statisticsArray = new ByteBuffer[StatisticType.getTotalTypeNum()]; + statisticsArray[StatisticType.min_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getMinBytes()); + statisticsArray[StatisticType.max_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getMaxBytes()); + statisticsArray[StatisticType.first_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getFirstBytes()); + statisticsArray[StatisticType.last_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getLastBytes()); + statisticsArray[StatisticType.sum_value.ordinal()] = ByteBuffer + .wrap(chunkStatistics.getSumBytes()); TsDigest tsDigest = new TsDigest(); - tsDigest.setStatistics(statisticsMap); + tsDigest.setStatistics(statisticsArray); currentChunk.setDigest(tsDigest); chunks.add(currentChunk); numOfPoints = 0; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java index e3784afdc23156df464ab36f264e94d1592d8cae..9f8806393535c685e7f108e129bdd708d8db8203 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/DigestForFilter.java @@ -100,10 +100,24 @@ public class DigestForFilter { return maxTime; } + public boolean isMinValueNull() { + return minValue == null; + } + + public boolean isMaxValueNull() { + return maxValue == null; + } + + /** + * Note check isMinValueNull before its usage + */ public > T getMinValue() { return getValue(minValue); } + /** + * Note check isMaxValueNull before its usage + */ public > T getMaxValue() { return getValue(maxValue); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java index 6c3b587fd641cdc8ee6297c8e601151e0cd653ff..7af77061d6e2acecb80eaf09c0d6b6de20823d45 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java @@ -41,6 +41,9 @@ public class Eq> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) >= digest.getMinTime() && ((Long) value) <= digest.getMaxTime(); } else { + if (digest.isMinValueNull() || digest.isMaxValueNull()) { + return true; + } return value.compareTo(digest.getMinValue()) >= 0 && value.compareTo(digest.getMaxValue()) <= 0; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java index 0b12713e66ba53fbd1c0f08849341b09eb5f4277..eadc8ffc8b659015cc5308b1b0477ce20fbebf5c 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java @@ -41,6 +41,9 @@ public class Gt> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) < digest.getMaxTime(); } else { + if (digest.isMaxValueNull()) { + return true; + } return value.compareTo(digest.getMaxValue()) < 0; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java index 5d1cb6d42f7729cfd1725366a8d173314e5e6dc1..20f3e19b3f0fd2801d776a9717f07f6353ef394b 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java @@ -41,6 +41,9 @@ public class GtEq> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) <= digest.getMaxTime(); } else { + if (digest.isMaxValueNull()) { + return true; + } return value.compareTo(digest.getMaxValue()) <= 0; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java index 3385241e7b14583aa9368f0996ce26bbbc8404a3..e7449c7223bf99f4efab4ed1b836ca5a078543aa 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java @@ -41,6 +41,9 @@ public class Lt> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) > digest.getMinTime(); } else { + if (digest.isMinValueNull()) { + return true; + } return value.compareTo(digest.getMinValue()) > 0; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java index b1cef193cf2a1b8f47ee80ac1cbbf7bbd8a4922d..1076b63531d3da54b15f21c7903082be701613c6 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java @@ -41,6 +41,9 @@ public class LtEq> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return ((Long) value) >= digest.getMinTime(); } else { + if (digest.isMinValueNull()) { + return true; + } return value.compareTo(digest.getMinValue()) >= 0; } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java index 0b92639f9cdc9b2a2846611280f3571adc7f7640..634388e03a7e24d0e3159185b3961f5d8b12ff24 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java @@ -41,6 +41,9 @@ public class NotEq> extends UnaryFilter { if (filterType == FilterType.TIME_FILTER) { return !(((Long) value) == digest.getMinTime() && (Long) value == digest.getMaxTime()); } else { + if (digest.isMinValueNull() || digest.isMaxValueNull()) { + return true; + } return !(value.compareTo(digest.getMinValue()) == 0 && value.compareTo(digest.getMaxValue()) == 0); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java index 145b6715c33571d49d33abdef479740dbbfa5df7..588e1f509827598d7a2f305472a7fdb618218b90 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReaderWithFilter.java @@ -19,9 +19,10 @@ package org.apache.iotdb.tsfile.read.reader.series; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; -import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; import org.apache.iotdb.tsfile.read.common.Chunk; import org.apache.iotdb.tsfile.read.controller.IChunkLoader; import org.apache.iotdb.tsfile.read.filter.DigestForFilter; @@ -50,11 +51,16 @@ public class FileSeriesReaderWithFilter extends FileSeriesReader { @Override protected boolean chunkSatisfied(ChunkMetaData chunkMetaData) { + ByteBuffer minValue = null; + ByteBuffer maxValue = null; + ByteBuffer[] statistics = chunkMetaData.getDigest().getStatistics(); + if (statistics != null) { + minValue = statistics[StatisticType.min_value.ordinal()]; // note still CAN be null + maxValue = statistics[StatisticType.max_value.ordinal()]; // note still CAN be null + } + DigestForFilter digest = new DigestForFilter(chunkMetaData.getStartTime(), - chunkMetaData.getEndTime(), - chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MIN_VALUE), - chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MAX_VALUE), - chunkMetaData.getTsDataType()); + chunkMetaData.getEndTime(), minValue, maxValue, chunkMetaData.getTsDataType()); return filter.satisfy(digest); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index d7cb619273728645e3c412cb115baf30940005d7..ff2a24c87d7bf9dc85591978a2af5d1347a96afd 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -21,13 +21,26 @@ package org.apache.iotdb.tsfile.write.writer; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.iotdb.tsfile.common.constant.StatisticConstant; import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter; import org.apache.iotdb.tsfile.file.header.ChunkHeader; -import org.apache.iotdb.tsfile.file.metadata.*; +import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; +import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; +import org.apache.iotdb.tsfile.file.metadata.TsDigest; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; +import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; @@ -37,8 +50,8 @@ import org.apache.iotdb.tsfile.read.common.Path; import org.apache.iotdb.tsfile.utils.BytesUtils; import org.apache.iotdb.tsfile.utils.PublicBAOS; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.apache.iotdb.tsfile.write.schema.Schema; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.apache.iotdb.tsfile.write.schema.Schema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +104,6 @@ public class TsFileIOWriter { * for writing a new tsfile. * * @param output be used to output written data - * @throws IOException */ public TsFileIOWriter(TsFileOutput output) throws IOException { this.out = output; @@ -137,7 +149,8 @@ public class TsFileIOWriter { */ public void startChunkGroup(String deviceId) throws IOException { LOG.debug("start chunk group:{}, file position {}", deviceId, out.getPosition()); - currentChunkGroupMetaData = new ChunkGroupMetaData(deviceId, new ArrayList<>(), out.getPosition()); + currentChunkGroupMetaData = new ChunkGroupMetaData(deviceId, new ArrayList<>(), + out.getPosition()); } /** @@ -145,7 +158,8 @@ public class TsFileIOWriter { */ public void endChunkGroup(long version) throws IOException { long dataSize = out.getPosition() - currentChunkGroupMetaData.getStartOffsetOfChunkGroup(); - ChunkGroupFooter chunkGroupFooter = new ChunkGroupFooter(currentChunkGroupMetaData.getDeviceID(), + ChunkGroupFooter chunkGroupFooter = new ChunkGroupFooter( + currentChunkGroupMetaData.getDeviceID(), dataSize, currentChunkGroupMetaData.getChunkMetaDataList().size()); chunkGroupFooter.serializeTo(out.wrapAsStream()); currentChunkGroupMetaData.setEndOffsetOfChunkGroup(out.getPosition()); @@ -183,17 +197,19 @@ public class TsFileIOWriter { header.serializeTo(out.wrapAsStream()); LOG.debug("finish series chunk:{} header, file position {}", header, out.getPosition()); - Map statisticsMap = new HashMap<>(); // TODO add your statistics - statisticsMap.put(StatisticConstant.MAX_VALUE, ByteBuffer.wrap(statistics.getMaxBytes())); - statisticsMap.put(StatisticConstant.MIN_VALUE, ByteBuffer.wrap(statistics.getMinBytes())); - statisticsMap.put(StatisticConstant.FIRST, ByteBuffer.wrap(statistics.getFirstBytes())); - statisticsMap.put(StatisticConstant.SUM, ByteBuffer.wrap(statistics.getSumBytes())); - statisticsMap.put(StatisticConstant.LAST, ByteBuffer.wrap(statistics.getLastBytes())); + ByteBuffer[] statisticsArray = new ByteBuffer[StatisticType.getTotalTypeNum()]; + statisticsArray[StatisticType.max_value.ordinal()] = ByteBuffer.wrap(statistics.getMaxBytes()); + statisticsArray[StatisticType.min_value.ordinal()] = ByteBuffer.wrap(statistics.getMinBytes()); + statisticsArray[StatisticType.first_value.ordinal()] = ByteBuffer + .wrap(statistics.getFirstBytes()); + statisticsArray[StatisticType.last_value.ordinal()] = ByteBuffer + .wrap(statistics.getLastBytes()); + statisticsArray[StatisticType.sum_value.ordinal()] = ByteBuffer.wrap(statistics.getSumBytes()); TsDigest tsDigest = new TsDigest(); - tsDigest.setStatistics(statisticsMap); + tsDigest.setStatistics(statisticsArray); currentChunkMetaData.setDigest(tsDigest); @@ -338,7 +354,6 @@ public class TsFileIOWriter { } - /** * get chunkGroupMetaDataList. * @@ -361,7 +376,12 @@ public class TsFileIOWriter { } /** +<<<<<<< HEAD * close the outputStream or file channel without writing FileMetadata. +======= + * close the outputStream or file channel without writing FileMetadata. This is just used for + * Testing. +>>>>>>> master */ public void close() throws IOException { canWrite = false; @@ -371,6 +391,7 @@ public class TsFileIOWriter { void writeSeparatorMaskForTest() throws IOException { out.write(new byte[]{MetaMarker.SEPARATOR}); } + void writeChunkMaskForTest() throws IOException { out.write(new byte[]{MetaMarker.CHUNK_HEADER}); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java index 29863fee5d1bfdc447cedd974b3100143139741a..b6910952914269b2af5648055f03ad96f95bf01b 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkGroupMetaDataTest.java @@ -18,11 +18,13 @@ */ package org.apache.iotdb.tsfile.file.metadata; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; - import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.MetaMarker; import org.apache.iotdb.tsfile.file.header.ChunkHeader; @@ -87,7 +89,7 @@ public class ChunkGroupMetaDataTest { } int offsetListIndex = 0; List deviceMetadataIndexList = metaData.getDeviceMap().values().stream() - .sorted((x, y) -> (int) (x.getOffset() - y.getOffset())).collect(Collectors.toList()); + .sorted((x, y) -> (int) (x.getOffset() - y.getOffset())).collect(Collectors.toList()); for (TsDeviceMetadataIndex index : deviceMetadataIndexList) { TsDeviceMetadata deviceMetadata = reader.readTsDeviceMetaData(index); List chunkGroupMetaDataList = deviceMetadata.getChunkGroupMetaDataList(); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java index 39e85e082caea5b2b955bb97cbe403e62c10cb21..c2e9bd778b301294d69360afe27e200ebf797888 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetaDataTest.java @@ -52,7 +52,7 @@ public class ChunkMetaDataTest { } @Test - public void testWriteIntoFile() throws IOException { + public void testWriteIntoFile() { ChunkMetaData metaData = TestHelper.createSimpleTimeSeriesChunkMetaData(); serialized(metaData); ChunkMetaData readMetaData = deSerialized(); @@ -60,6 +60,15 @@ public class ChunkMetaDataTest { serialized(readMetaData); } + @Test + public void testWriteIntoFile2() throws IOException { + ChunkMetaData metaData = TestHelper.createNotCompleteSimpleTimeSeriesChunkMetaData(); + serialized(metaData); + ChunkMetaData readMetaData = deSerialized(); + Utils.isTimeSeriesChunkMetadataEqual(metaData, readMetaData); + serialized(readMetaData); + } + private ChunkMetaData deSerialized() { FileInputStream fis = null; ChunkMetaData metaData = null; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java index e76f522a37bc9e6babd791f73eff5b8943f31433..33aa6a3f9442b058eb2904935ce8d40a16953c51 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/TestHelper.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.file.metadata.utils; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -34,6 +35,7 @@ import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataTest; import org.apache.iotdb.tsfile.file.metadata.TsDigest; +import org.apache.iotdb.tsfile.file.metadata.TsDigest.StatisticType; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaDataTest; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -116,19 +118,49 @@ public class TestHelper { return metaData; } + public static ChunkMetaData createNotCompleteSimpleTimeSeriesChunkMetaData() throws IOException { + ChunkMetaData metaData = new ChunkMetaData(ChunkMetaDataTest.MEASUREMENT_UID, + ChunkMetaDataTest.DATA_TYPE, + ChunkMetaDataTest.FILE_OFFSET, ChunkMetaDataTest.START_TIME, ChunkMetaDataTest.END_TIME + ); + metaData.setNumOfPoints(ChunkMetaDataTest.NUM_OF_POINTS); + metaData.setDigest(createNotCompleteSimpleTsDigest()); + return metaData; + } + public static MeasurementSchema createSimpleMeasurementSchema() { return new MeasurementSchema(TimeSeriesMetadataTest.measurementUID, TSDataType.INT64, TSEncoding.RLE); } - public static TsDigest createSimpleTsDigest() { + public static TsDigest createSimpleTsDigest() throws IOException { + TsDigest digest = new TsDigest(); + ByteBuffer[] statisticsArray = new ByteBuffer[StatisticType.getTotalTypeNum()]; + statisticsArray[StatisticType.min_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(MIN_VALUE)); + statisticsArray[StatisticType.max_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(MAX_VALUE)); + statisticsArray[StatisticType.first_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(FIRST_VALUE)); + statisticsArray[StatisticType.last_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(LAST_VALUE)); + statisticsArray[StatisticType.sum_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(SUM_VALUE)); + digest.setStatistics(statisticsArray); + return digest; + } + + public static TsDigest createNotCompleteSimpleTsDigest() throws IOException { TsDigest digest = new TsDigest(); - digest.addStatistics("max", ByteBuffer.wrap(BytesUtils.stringToBytes(MAX_VALUE))); - digest.addStatistics("min", ByteBuffer.wrap(BytesUtils.stringToBytes(MIN_VALUE))); - digest.addStatistics("sum", ByteBuffer.wrap(BytesUtils.stringToBytes(SUM_VALUE))); - digest.addStatistics("first", ByteBuffer.wrap(BytesUtils.stringToBytes(FIRST_VALUE))); - digest.addStatistics("last", ByteBuffer.wrap(BytesUtils.stringToBytes(LAST_VALUE))); + ByteBuffer[] statisticsArray = new ByteBuffer[StatisticType.getTotalTypeNum()]; + statisticsArray[StatisticType.first_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(FIRST_VALUE)); + statisticsArray[StatisticType.last_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(LAST_VALUE)); + statisticsArray[StatisticType.sum_value.ordinal()] = ByteBuffer + .wrap(BytesUtils.stringToBytes(SUM_VALUE)); + digest.setStatistics(statisticsArray); return digest; } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java index a645c2e26e9f93c484b32d988ff103c439413dba..e217d8bc569ef3b46f204964c69c71bd88bf6443 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java @@ -19,10 +19,10 @@ package org.apache.iotdb.tsfile.file.metadata.utils; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import org.apache.iotdb.tsfile.file.header.PageHeader; @@ -30,14 +30,16 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData; import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata; import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadataIndex; +import org.apache.iotdb.tsfile.file.metadata.TsDigest; import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; public class Utils { private static final double maxError = 0.0001d; - + public static void isListEqual(List listA, List listB, String name) { if ((listA == null) ^ (listB == null)) { @@ -70,32 +72,21 @@ public class Utils { } } - public static void isMapBufferEqual(Map mapA, Map mapB, - String name) { - if ((mapA == null) ^ (mapB == null)) { + public static void isTwoTsDigestEqual(TsDigest digestA, TsDigest digestB, String name) { + if ((digestA == null) ^ (digestB == null)) { System.out.println("error"); fail(String.format("one of %s is null", name)); } - if ((mapA != null) && (mapB != null)) { - if (mapA.size() != mapB.size()) { - fail(String.format("%s size is different", name)); - } - for (String key : mapB.keySet()) { - ByteBuffer b = mapB.get(key); - ByteBuffer a = mapA.get(key); - assertTrue(b.equals(a)); - } + if (digestA != null) { + Assert.assertEquals(digestA, digestB); } } /** * when one of A and B is Null, A != B, so test case fails. * - * @param objectA - * @param objectB - * @param name - * @return false - A and B both are NULL, so we do not need to check whether their members are equal true - A and B - * both are not NULL, so we need to check their members + * @return false - A and B both are NULL, so we do not need to check whether their members are + * equal true - A and B both are not NULL, so we need to check their members */ public static boolean isTwoObjectsNotNULL(Object objectA, Object objectB, String name) { if ((objectA == null) && (objectB == null)) { @@ -128,11 +119,9 @@ public class Utils { assertTrue(metadata1.getNumOfPoints() == metadata2.getNumOfPoints()); assertTrue(metadata1.getStartTime() == metadata2.getStartTime()); assertTrue(metadata1.getEndTime() == metadata2.getEndTime()); - if (Utils.isTwoObjectsNotNULL(metadata1.getDigest(), metadata2.getDigest(), "digest")) { - Utils.isMapBufferEqual(metadata1.getDigest().getStatistics(), - metadata2.getDigest().getStatistics(), - "statistics"); - } + assertNotNull(metadata1.getDigest()); + assertNotNull(metadata2.getDigest()); + Utils.isTwoTsDigestEqual(metadata1.getDigest(), metadata2.getDigest(), "TsDigest"); } } @@ -142,9 +131,11 @@ public class Utils { assertEquals(metadata1.getStartTime(), metadata2.getStartTime()); assertEquals(metadata1.getEndTime(), metadata2.getEndTime()); - if (Utils.isTwoObjectsNotNULL(metadata1.getChunkGroupMetaDataList(), metadata2.getChunkGroupMetaDataList(), + if (Utils.isTwoObjectsNotNULL(metadata1.getChunkGroupMetaDataList(), + metadata2.getChunkGroupMetaDataList(), "Rowgroup metadata list")) { - assertEquals(metadata1.getChunkGroupMetaDataList().size(), metadata2.getChunkGroupMetaDataList().size()); + assertEquals(metadata1.getChunkGroupMetaDataList().size(), + metadata2.getChunkGroupMetaDataList().size()); for (int i = 0; i < metadata1.getChunkGroupMetaDataList().size(); i++) { Utils.isChunkGroupMetaDataEqual(metadata1.getChunkGroupMetaDataList().get(i), metadata1.getChunkGroupMetaDataList().get(i)); @@ -219,7 +210,8 @@ public class Utils { assertTrue(header1.getNumOfValues() == header2.getNumOfValues()); assertTrue(header1.getMaxTimestamp() == header2.getMaxTimestamp()); assertTrue(header1.getMinTimestamp() == header2.getMinTimestamp()); - if (Utils.isTwoObjectsNotNULL(header1.getStatistics(), header2.getStatistics(), "statistics")) { + if (Utils + .isTwoObjectsNotNULL(header1.getStatistics(), header2.getStatistics(), "statistics")) { Utils.isStatisticsEqual(header1.getStatistics(), header2.getStatistics()); } } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java index e66aa1c42359a93c3262d3d1e97be3aae4e825c4..e0686c806f641c0f28ea64b81ea2ca3fd58430af 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/DigestFilterTest.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.tsfile.read.filter; +import java.nio.ByteBuffer; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; @@ -31,16 +32,20 @@ public class DigestFilterTest { BytesUtils.intToBytes(100), TSDataType.INT32); private DigestForFilter digest2 = new DigestForFilter(101L, 200L, BytesUtils.intToBytes(101), BytesUtils.intToBytes(200), TSDataType.INT32); + private DigestForFilter digest3 = new DigestForFilter(101L, 200L, (ByteBuffer) null, null, + TSDataType.INT32); @Test public void testEq() { Filter timeEq = TimeFilter.eq(10L); Assert.assertEquals(true, timeEq.satisfy(digest1)); Assert.assertEquals(false, timeEq.satisfy(digest2)); + Assert.assertEquals(false, timeEq.satisfy(digest3)); Filter valueEq = ValueFilter.eq(100); Assert.assertEquals(true, valueEq.satisfy(digest1)); Assert.assertEquals(false, valueEq.satisfy(digest2)); + Assert.assertEquals(true, valueEq.satisfy(digest3)); } @Test @@ -48,10 +53,12 @@ public class DigestFilterTest { Filter timeGt = TimeFilter.gt(100L); Assert.assertEquals(false, timeGt.satisfy(digest1)); Assert.assertEquals(true, timeGt.satisfy(digest2)); + Assert.assertEquals(true, timeGt.satisfy(digest3)); Filter valueGt = ValueFilter.gt(100); Assert.assertEquals(false, valueGt.satisfy(digest1)); Assert.assertEquals(true, valueGt.satisfy(digest2)); + Assert.assertEquals(true, valueGt.satisfy(digest3)); } @Test @@ -59,10 +66,12 @@ public class DigestFilterTest { Filter timeGtEq = TimeFilter.gtEq(100L); Assert.assertEquals(true, timeGtEq.satisfy(digest1)); Assert.assertEquals(true, timeGtEq.satisfy(digest2)); + Assert.assertEquals(true, timeGtEq.satisfy(digest3)); Filter valueGtEq = ValueFilter.gtEq(100); Assert.assertEquals(true, valueGtEq.satisfy(digest1)); - Assert.assertEquals(true, valueGtEq.satisfy(digest2)); + Assert.assertEquals(true, valueGtEq.satisfy(digest3)); + Assert.assertEquals(true, valueGtEq.satisfy(digest3)); } @Test @@ -70,10 +79,12 @@ public class DigestFilterTest { Filter timeLt = TimeFilter.lt(101L); Assert.assertEquals(true, timeLt.satisfy(digest1)); Assert.assertEquals(false, timeLt.satisfy(digest2)); + Assert.assertEquals(false, timeLt.satisfy(digest3)); Filter valueLt = ValueFilter.lt(101); Assert.assertEquals(true, valueLt.satisfy(digest1)); Assert.assertEquals(false, valueLt.satisfy(digest2)); + Assert.assertEquals(true, valueLt.satisfy(digest3)); } @Test @@ -81,10 +92,12 @@ public class DigestFilterTest { Filter timeLtEq = TimeFilter.ltEq(101L); Assert.assertEquals(true, timeLtEq.satisfy(digest1)); Assert.assertEquals(true, timeLtEq.satisfy(digest2)); + Assert.assertEquals(true, timeLtEq.satisfy(digest3)); Filter valueLtEq = ValueFilter.ltEq(101); Assert.assertEquals(true, valueLtEq.satisfy(digest1)); Assert.assertEquals(true, valueLtEq.satisfy(digest2)); + Assert.assertEquals(true, valueLtEq.satisfy(digest3)); } @Test @@ -92,10 +105,12 @@ public class DigestFilterTest { Filter andFilter = FilterFactory.and(TimeFilter.gt(10L), ValueFilter.lt(50)); Assert.assertEquals(true, andFilter.satisfy(digest1)); Assert.assertEquals(false, andFilter.satisfy(digest2)); + Assert.assertEquals(true, andFilter.satisfy(digest3)); Filter orFilter = FilterFactory.or(andFilter, TimeFilter.eq(200L)); Assert.assertEquals(true, orFilter.satisfy(digest1)); Assert.assertEquals(true, orFilter.satisfy(digest2)); + Assert.assertEquals(true, orFilter.satisfy(digest3)); } }
Member DescriptionMember Type