Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
apache
Iotdb
提交
d1522e7d
I
Iotdb
项目概览
apache
/
Iotdb
11 个月 前同步成功
通知
25
Star
3344
Fork
916
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
I
Iotdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d1522e7d
编写于
7月 16, 2019
作者:
J
jack870131
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Code reformat
上级
43271bb8
变更
2
显示空白变更内容
内联
并排
Showing
2 changed file
with
103 addition
and
180 deletion
+103
-180
docs/Documentation/UserGuideV0.7.0/5-IoTDB SQL Documentation/1-IoTDB Query Statement.md
....7.0/5-IoTDB SQL Documentation/1-IoTDB Query Statement.md
+2
-2
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
...che/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
+101
-178
未找到文件。
docs/Documentation/UserGuideV0.7.0/5-IoTDB SQL Documentation/1-IoTDB Query Statement.md
浏览文件 @
d1522e7d
tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
浏览文件 @
d1522e7d
...
...
@@ -46,6 +46,7 @@ public abstract class RegularDataEncoder extends Encoder {
private
static
final
Logger
LOGGER
=
LoggerFactory
.
getLogger
(
RegularDataEncoder
.
class
);
protected
ByteArrayOutputStream
out
;
protected
int
blockSize
;
protected
boolean
isMissingPoint
;
protected
int
writeIndex
=
-
1
;
...
...
@@ -63,13 +64,36 @@ public abstract class RegularDataEncoder extends Encoder {
protected
abstract
void
reset
();
protected
abstract
void
flushBlockBuffer
(
ByteArrayOutputStream
out
)
throws
IOException
;
protected
abstract
void
checkMissingPoint
(
ByteArrayOutputStream
out
)
throws
IOException
;
protected
abstract
void
writeBitmap
(
ByteArrayOutputStream
out
)
throws
IOException
;
protected
void
writeHeaderToBytes
()
throws
IOException
{
out
.
write
(
BytesUtils
.
intToBytes
(
writeIndex
));
writeHeader
();
}
protected
void
flushBlockBuffer
(
ByteArrayOutputStream
out
)
throws
IOException
{
if
(
writeIndex
==
-
1
)
{
return
;
}
this
.
out
=
out
;
// check if the missing point exists
checkMissingPoint
(
out
);
// write identifier
out
.
write
(
BytesUtils
.
boolToBytes
(
isMissingPoint
));
// write bitmap if missing points exist
if
(
isMissingPoint
)
{
writeBitmap
(
out
);
}
// write header
writeHeaderToBytes
();
reset
();
writeIndex
=
-
1
;
}
/**
* calling this method to flush all values which haven't encoded to result byte array.
*/
...
...
@@ -85,15 +109,17 @@ public abstract class RegularDataEncoder extends Encoder {
public
static
class
IntRegularEncoder
extends
RegularDataEncoder
{
private
int
[]
data
;
private
int
[]
missingPointData
;
private
int
[]
regularData
;
private
int
firstValue
;
private
int
previousValue
;
private
int
minDeltaBase
;
private
boolean
isMissingPoint
;
private
boolean
isLastPack
;
private
int
newBlockSize
;
private
BitSet
bitmap
;
/**
* constructor of IntRegularEncoder which is a sub-class of RegularDataEncoder.
*
* @param size - the number how many numbers to be packed into a block.
*/
public
IntRegularEncoder
()
{
this
(
BLOCK_DEFAULT_SIZE
);
}
...
...
@@ -108,37 +134,10 @@ public abstract class RegularDataEncoder extends Encoder {
reset
();
}
@Override
protected
void
flushBlockBuffer
(
ByteArrayOutputStream
out
)
throws
IOException
{
if
(
writeIndex
==
-
1
)
{
return
;
}
this
.
out
=
out
;
// write last pack
if
(
writeIndex
<
blockSize
)
{
isLastPack
=
true
;
checkMissingPoint
(
out
);
}
// write identifier
out
.
write
(
BytesUtils
.
boolToBytes
(
isMissingPoint
));
// write bitmap if missing points exist
if
(
isMissingPoint
)
{
writeBitmap
(
out
);
}
// write header
writeHeaderToBytes
();
reset
();
writeIndex
=
-
1
;
}
@Override
protected
void
reset
()
{
blockSize
=
BLOCK_DEFAULT_SIZE
;
minDeltaBase
=
Integer
.
MAX_VALUE
;
isMissingPoint
=
false
;
isLastPack
=
false
;
firstValue
=
0
;
previousValue
=
0
;
}
...
...
@@ -169,86 +168,63 @@ public abstract class RegularDataEncoder extends Encoder {
return
(
long
)
20
+
(
writeIndex
*
2
/
8
)
+
(
writeIndex
*
4
);
}
/**
* input a integer or long value.
*
* @param value value to encode
* @param out - the ByteArrayOutputStream which data encode into
*/
public
void
encodeValue
(
int
value
,
ByteArrayOutputStream
out
)
throws
IOException
{
if
(
writeIndex
==
-
1
)
{
data
=
new
int
[
blockSize
];
writeIndex
=
0
;
}
data
[
writeIndex
++]
=
value
;
if
(
writeIndex
==
blockSize
)
{
checkMissingPoint
(
out
);
}
}
private
void
checkMissingPoint
(
ByteArrayOutputStream
out
)
throws
IOException
{
@Override
protected
void
checkMissingPoint
(
ByteArrayOutputStream
out
)
throws
IOException
{
// get the new regular data if the missing point exists in the original data
data
=
getRegularData
(
data
);
firstValue
=
data
[
0
];
if
(
isMissingPoint
)
{
writeIndex
=
data
.
length
;
}
if
(!
isLastPack
)
{
flush
(
out
);
}
}
private
void
writeBitmap
(
ByteArrayOutputStream
out
)
throws
IOException
{
// generate bitmap
data2Diff
(
missingPointData
);
byte
[]
bsArr
=
bitmap
.
toByteArray
();
out
.
write
(
BytesUtils
.
intToBytes
(
bsArr
.
length
));
out
.
write
(
bsArr
);
}
private
int
[]
getRegularData
(
int
[]
data
)
{
if
(
writeIndex
>
1
)
{
previousValue
=
data
[
0
];
minDeltaBase
=
data
[
1
]
-
data
[
0
];
// calculate minimum elapsed of the data and check whether the missing point exists
for
(
int
i
=
1
;
i
<
writeIndex
;
i
++)
{
calcDelta
(
data
[
i
]);
int
delta
=
data
[
i
]
-
previousValue
;
// calculate delta
if
(
delta
!=
minDeltaBase
)
{
isMissingPoint
=
true
;
}
if
(
delta
<
minDeltaBase
)
{
minDeltaBase
=
delta
;
}
previousValue
=
data
[
i
];
}
}
// generate the continious data when the missing point exists
firstValue
=
data
[
0
];
if
(
isMissingPoint
)
{
generateRegularData
(
data
)
;
return
regularData
;
newBlockSize
=
((
data
[
writeIndex
-
1
]
-
data
[
0
])
/
minDeltaBase
)
+
1
;
writeIndex
=
newBlockSize
;
}
return
data
;
}
private
void
generateRegularData
(
int
[]
data
)
{
missingPointData
=
data
;
blockSize
=
(
int
)
(((
data
[
writeIndex
-
1
]
-
data
[
0
])
/
minDeltaBase
)
+
1
);
regularData
=
new
int
[
blockSize
]
;
for
(
int
i
=
0
;
i
<
blockSize
;
i
++)
{
regularData
[
i
]
=
(
data
[
0
]
+
minDeltaBase
*
i
);
}
@Override
protected
void
writeBitmap
(
ByteArrayOutputStream
out
)
throws
IOException
{
// generate bitmap
data2Diff
(
data
)
;
byte
[]
bsArr
=
bitmap
.
toByteArray
();
out
.
write
(
BytesUtils
.
intToBytes
(
bsArr
.
length
)
);
out
.
write
(
bsArr
);
}
private
void
calcDelta
(
int
value
)
{
int
delta
=
value
-
previousValue
;
// calculate delta
if
(
delta
!=
minDeltaBase
)
{
isMissingPoint
=
true
;
/**
* input a integer or long value.
*
* @param value value to encode
* @param out - the ByteArrayOutputStream which data encode into
*/
public
void
encodeValue
(
int
value
,
ByteArrayOutputStream
out
)
throws
IOException
{
if
(
writeIndex
==
-
1
)
{
data
=
new
int
[
blockSize
];
writeIndex
=
0
;
}
if
(
delta
<
minDeltaBase
)
{
minDeltaBase
=
delta
;
data
[
writeIndex
++]
=
value
;
if
(
writeIndex
==
blockSize
)
{
flush
(
out
);
}
}
private
void
data2Diff
(
int
[]
missingPointData
)
{
bitmap
=
new
BitSet
(
regularData
.
length
);
bitmap
.
flip
(
0
,
regularData
.
length
);
bitmap
=
new
BitSet
(
newBlockSize
);
bitmap
.
flip
(
0
,
newBlockSize
);
int
offset
=
0
;
for
(
int
i
=
1
;
i
<
missingPointData
.
length
;
i
++)
{
long
delta
=
missingPointData
[
i
]
-
missingPointData
[
i
-
1
];
int
delta
=
missingPointData
[
i
]
-
missingPointData
[
i
-
1
];
if
(
delta
!=
minDeltaBase
)
{
int
missingPointNum
=
(
int
)
(
delta
/
minDeltaBase
)
-
1
;
for
(
int
j
=
0
;
j
<
missingPointNum
;
j
++)
{
...
...
@@ -262,13 +238,10 @@ public abstract class RegularDataEncoder extends Encoder {
public
static
class
LongRegularEncoder
extends
RegularDataEncoder
{
private
long
[]
data
;
private
long
[]
missingPointData
;
private
long
[]
regularData
;
private
long
firstValue
;
private
long
previousValue
;
private
long
minDeltaBase
;
private
boolean
isMissingPoint
;
private
boolean
isLastPack
;
private
int
newBlockSize
;
private
BitSet
bitmap
;
public
LongRegularEncoder
()
{
...
...
@@ -285,37 +258,10 @@ public abstract class RegularDataEncoder extends Encoder {
reset
();
}
@Override
protected
void
flushBlockBuffer
(
ByteArrayOutputStream
out
)
throws
IOException
{
if
(
writeIndex
==
-
1
)
{
return
;
}
this
.
out
=
out
;
// write last pack
if
(
writeIndex
<
blockSize
)
{
isLastPack
=
true
;
checkMissingPoint
(
out
);
}
// write identifier
out
.
write
(
BytesUtils
.
boolToBytes
(
isMissingPoint
));
// write bitmap if missing points exist
if
(
isMissingPoint
)
{
writeBitmap
(
out
);
}
// write header
writeHeaderToBytes
();
reset
();
writeIndex
=
-
1
;
}
@Override
protected
void
reset
()
{
blockSize
=
BLOCK_DEFAULT_SIZE
;
minDeltaBase
=
Long
.
MAX_VALUE
;
isMissingPoint
=
false
;
isLastPack
=
false
;
firstValue
=
0L
;
previousValue
=
0L
;
}
...
...
@@ -346,83 +292,60 @@ public abstract class RegularDataEncoder extends Encoder {
return
(
long
)
28
+
(
writeIndex
*
2
/
8
)
+
(
writeIndex
*
8
);
}
/**
* input a integer or long value.
*
* @param value value to encode
* @param out - the ByteArrayOutputStream which data encode into
*/
public
void
encodeValue
(
long
value
,
ByteArrayOutputStream
out
)
throws
IOException
{
if
(
writeIndex
==
-
1
)
{
data
=
new
long
[
blockSize
];
writeIndex
=
0
;
}
data
[
writeIndex
++]
=
value
;
if
(
writeIndex
==
blockSize
)
{
checkMissingPoint
(
out
);
}
}
private
void
checkMissingPoint
(
ByteArrayOutputStream
out
)
throws
IOException
{
@Override
protected
void
checkMissingPoint
(
ByteArrayOutputStream
out
)
throws
IOException
{
// get the new regular data if the missing point exists in the original data
data
=
getRegularData
(
data
);
firstValue
=
data
[
0
];
if
(
isMissingPoint
)
{
writeIndex
=
data
.
length
;
}
if
(!
isLastPack
)
{
flush
(
out
);
}
}
private
void
writeBitmap
(
ByteArrayOutputStream
out
)
throws
IOException
{
// generate bitmap
data2Diff
(
missingPointData
);
byte
[]
bsArr
=
bitmap
.
toByteArray
();
out
.
write
(
BytesUtils
.
intToBytes
(
bsArr
.
length
));
out
.
write
(
bsArr
);
}
private
long
[]
getRegularData
(
long
[]
data
)
{
if
(
writeIndex
>
1
)
{
previousValue
=
data
[
0
];
minDeltaBase
=
data
[
1
]
-
data
[
0
];
// calculate minimum elapsed of the data and check whether the missing point exists
for
(
int
i
=
1
;
i
<
writeIndex
;
i
++)
{
calcDelta
(
data
[
i
]);
long
delta
=
data
[
i
]
-
previousValue
;
// calculate delta
if
(
delta
!=
minDeltaBase
)
{
isMissingPoint
=
true
;
}
if
(
delta
<
minDeltaBase
)
{
minDeltaBase
=
delta
;
}
previousValue
=
data
[
i
];
}
}
// generate the continious data when the missing point exists
firstValue
=
data
[
0
];
if
(
isMissingPoint
)
{
generateRegularData
(
data
);
return
regularData
;
newBlockSize
=
(
int
)
(((
data
[
writeIndex
-
1
]
-
data
[
0
])
/
minDeltaBase
)
+
1
);
writeIndex
=
newBlockSize
;
}
return
data
;
}
private
void
generateRegularData
(
long
[]
data
)
{
missingPointData
=
data
;
blockSize
=
(
int
)
(((
data
[
writeIndex
-
1
]
-
data
[
0
])
/
minDeltaBase
)
+
1
);
regularData
=
new
long
[
blockSize
]
;
for
(
int
i
=
0
;
i
<
blockSize
;
i
++)
{
regularData
[
i
]
=
(
data
[
0
]
+
minDeltaBase
*
i
);
}
@Override
protected
void
writeBitmap
(
ByteArrayOutputStream
out
)
throws
IOException
{
// generate bitmap
data2Diff
(
data
)
;
byte
[]
bsArr
=
bitmap
.
toByteArray
();
out
.
write
(
BytesUtils
.
intToBytes
(
bsArr
.
length
)
);
out
.
write
(
bsArr
);
}
private
void
calcDelta
(
long
value
)
{
long
delta
=
value
-
previousValue
;
// calculate delta
if
(
delta
!=
minDeltaBase
)
{
isMissingPoint
=
true
;
/**
* input a integer or long value.
*
* @param value value to encode
* @param out - the ByteArrayOutputStream which data encode into
*/
public
void
encodeValue
(
long
value
,
ByteArrayOutputStream
out
)
throws
IOException
{
if
(
writeIndex
==
-
1
)
{
data
=
new
long
[
blockSize
];
writeIndex
=
0
;
}
if
(
delta
<
minDeltaBase
)
{
minDeltaBase
=
delta
;
data
[
writeIndex
++]
=
value
;
if
(
writeIndex
==
blockSize
)
{
flush
(
out
);
}
}
private
void
data2Diff
(
long
[]
missingPointData
)
{
bitmap
=
new
BitSet
(
regularData
.
length
);
bitmap
.
flip
(
0
,
regularData
.
length
);
bitmap
=
new
BitSet
(
newBlockSize
);
bitmap
.
flip
(
0
,
newBlockSize
);
int
offset
=
0
;
for
(
int
i
=
1
;
i
<
missingPointData
.
length
;
i
++)
{
long
delta
=
missingPointData
[
i
]
-
missingPointData
[
i
-
1
];
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录