Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
apache
Iotdb
提交
11f747af
I
Iotdb
项目概览
apache
/
Iotdb
大约 1 年 前同步成功
通知
25
Star
3344
Fork
916
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
I
Iotdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
11f747af
编写于
7月 08, 2020
作者:
J
JackieTien97
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix bug
上级
a7e90302
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
46 addition
and
30 deletion
+46
-30
server/src/main/java/org/apache/iotdb/db/engine/flush/VmMergeUtils.java
...n/java/org/apache/iotdb/db/engine/flush/VmMergeUtils.java
+5
-2
server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
.../apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+40
-28
tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
.../org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+1
-0
未找到文件。
server/src/main/java/org/apache/iotdb/db/engine/flush/VmMergeUtils.java
浏览文件 @
11f747af
...
...
@@ -87,7 +87,7 @@ public class VmMergeUtils {
.
computeIfAbsent
(
vmWriter
.
getFile
().
getAbsolutePath
(),
path
->
{
try
{
return
new
TsFileSequenceReader
(
path
);
return
new
TsFileSequenceReader
(
path
,
false
);
}
catch
(
IOException
e
)
{
logger
.
error
(
"Storage group {} tsfile {}, flush recover meets error. reader create failed."
,
...
...
@@ -140,7 +140,7 @@ public class VmMergeUtils {
.
computeIfAbsent
(
vmWriter
.
getFile
().
getAbsolutePath
(),
path
->
{
try
{
return
new
TsFileSequenceReader
(
path
);
return
new
TsFileSequenceReader
(
path
,
false
);
}
catch
(
IOException
e
)
{
logger
.
error
(
"Storage group {} tsfile {}, flush recover meets error. reader create failed."
,
...
...
@@ -152,6 +152,9 @@ public class VmMergeUtils {
if
(
reader
==
null
)
{
continue
;
}
if
(!
vmWriter
.
getMetadatasForQuery
().
containsKey
(
deviceId
))
{
continue
;
}
List
<
ChunkMetadata
>
chunkMetadataList
=
vmWriter
.
getMetadatasForQuery
()
.
get
(
deviceId
).
get
(
measurementId
);
if
(
chunkMetadataList
==
null
)
{
...
...
server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
浏览文件 @
11f747af
...
...
@@ -34,10 +34,9 @@ import java.util.List;
import
java.util.Map
;
import
java.util.Map.Entry
;
import
java.util.Set
;
import
java.util.concurrent.Callable
;
import
java.util.concurrent.ConcurrentLinkedDeque
;
import
java.util.concurrent.CopyOnWriteArrayList
;
import
java.util.concurrent.
Future
;
import
java.util.concurrent.
TimeUnit
;
import
java.util.concurrent.locks.ReadWriteLock
;
import
java.util.concurrent.locks.ReentrantReadWriteLock
;
import
org.apache.iotdb.db.conf.IoTDBConfig
;
...
...
@@ -79,7 +78,6 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import
org.apache.iotdb.tsfile.file.metadata.enums.TSDataType
;
import
org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding
;
import
org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer
;
import
org.apache.iotdb.tsfile.read.TsFileSequenceReader
;
import
org.apache.iotdb.tsfile.read.common.Path
;
import
org.apache.iotdb.tsfile.utils.Pair
;
import
org.apache.iotdb.tsfile.write.schema.MeasurementSchema
;
...
...
@@ -133,6 +131,8 @@ public class TsFileProcessor {
private
static
final
String
FLUSH_QUERY_WRITE_LOCKED
=
"{}: {} get flushQueryLock write lock"
;
private
static
final
String
FLUSH_QUERY_WRITE_RELEASE
=
"{}: {} get flushQueryLock write lock released"
;
private
volatile
boolean
mergeWorking
=
false
;
TsFileProcessor
(
String
storageGroupName
,
File
tsfile
,
List
<
File
>
vmFiles
,
VersionController
versionController
,
CloseTsFileCallBack
closeTsFileCallback
,
...
...
@@ -664,16 +664,18 @@ public class TsFileProcessor {
flushVmTimes
++;
try
{
MemTableFlushTask
flushTask
;
RestorableTsFileIOWriter
curWriter
;
if
(
config
.
isEnableVm
())
{
logger
.
info
(
"[Flush] flush a vm"
);
File
newVmFile
=
createNewVMFile
(
tsFileResource
);
vmTsFileResources
.
add
(
new
TsFileResource
(
newVmFile
));
RestorableTsFileIOWriter
currVmWriter
=
new
RestorableTsFileIOWriter
(
newVmFile
);
vmWriters
.
add
(
currVmWriter
);
flushTask
=
new
MemTableFlushTask
(
memTableToFlush
,
currVmWriter
,
storageGroupName
);
curWriter
=
new
RestorableTsFileIOWriter
(
newVmFile
);
vmWriters
.
add
(
curWriter
);
}
else
{
flushTask
=
new
MemTableFlushTask
(
memTableToFlush
,
writer
,
storageGroupName
)
;
curWriter
=
writer
;
}
curWriter
.
mark
();
flushTask
=
new
MemTableFlushTask
(
memTableToFlush
,
curWriter
,
storageGroupName
);
flushTask
.
syncFlushMemTable
();
}
catch
(
Exception
e
)
{
logger
.
error
(
"{}: {} meet error when flushing a memtable, change system mode to read-only"
,
...
...
@@ -708,13 +710,25 @@ public class TsFileProcessor {
memTableToFlush
.
isSignalMemTable
(),
flushingMemTables
.
size
());
}
}
Future
<
Void
>
vmFuture
=
VmMergeTaskPoolManager
.
getInstance
()
.
submit
(
new
VmMergeTask
(
new
ArrayList
<>(
vmTsFileResources
),
new
ArrayList
<>(
vmWriters
)));
if
(
shouldClose
&&
flushingMemTables
.
isEmpty
())
{
try
{
// confirm that all vm file is flushed to tsfile when closed
vmFuture
.
get
();
// merge vm to tsfile
while
(
true
)
{
if
(!
mergeWorking
)
{
break
;
}
try
{
TimeUnit
.
MILLISECONDS
.
sleep
(
10
);
}
catch
(
@SuppressWarnings
(
"squid:S2142"
)
InterruptedException
e
)
{
logger
.
error
(
"{}: {}, closing task is interrupted."
,
storageGroupName
,
tsFileResource
.
getFile
().
getName
(),
e
);
// generally it is because the thread pool is shutdown so the task should be aborted
break
;
}
}
logger
.
info
(
"[Flush] merge all {} vms to TsFile"
,
vmTsFileResources
.
size
()
+
1
);
flushAllVmToTsFile
(
vmWriters
,
vmTsFileResources
);
writer
.
mark
();
try
{
double
compressionRatio
=
((
double
)
totalMemTableSize
)
/
writer
.
getPos
();
...
...
@@ -766,6 +780,13 @@ public class TsFileProcessor {
synchronized
(
flushingMemTables
)
{
flushingMemTables
.
notifyAll
();
}
}
else
{
// on other merge task working now, it's safe to submit one.
if
(!
mergeWorking
)
{
mergeWorking
=
true
;
VmMergeTaskPoolManager
.
getInstance
()
.
submit
(
new
VmMergeTask
(
new
ArrayList
<>(
vmTsFileResources
),
new
ArrayList
<>(
vmWriters
)));
}
}
}
...
...
@@ -979,10 +1000,10 @@ public class TsFileProcessor {
logFile
.
delete
();
}
class
VmMergeTask
implements
Callable
<
Void
>
{
class
VmMergeTask
implements
Runnable
{
private
List
<
TsFileResource
>
vmMergeTsFiles
;
private
List
<
RestorableTsFileIOWriter
>
vmMergeWriters
;
private
final
List
<
TsFileResource
>
vmMergeTsFiles
;
private
final
List
<
RestorableTsFileIOWriter
>
vmMergeWriters
;
public
VmMergeTask
(
List
<
TsFileResource
>
vmMergeTsFiles
,
...
...
@@ -991,15 +1012,9 @@ public class TsFileProcessor {
this
.
vmMergeWriters
=
vmMergeWriters
;
}
public
void
merge
()
{
@Override
public
void
run
()
{
try
{
if
(
shouldClose
&&
flushingMemTables
.
isEmpty
())
{
// merge vm to tsfile
logger
.
info
(
"[Flush] merge all {} vms to TsFile"
,
vmMergeTsFiles
.
size
()
+
1
);
flushAllVmToTsFile
(
vmMergeWriters
,
vmMergeTsFiles
);
return
;
}
long
vmPointNum
=
0
;
// all flush to target file
Map
<
Path
,
MeasurementSchema
>
pathMeasurementSchemaMap
=
new
HashMap
<>();
...
...
@@ -1048,13 +1063,10 @@ public class TsFileProcessor {
}
}
catch
(
Exception
e
)
{
logger
.
error
(
"Error occurred in Vm Merge thread"
,
e
);
}
finally
{
// reset the merge working state to false
mergeWorking
=
false
;
}
}
@Override
public
Void
call
()
{
merge
();
return
null
;
}
}
}
tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
浏览文件 @
11f747af
...
...
@@ -163,6 +163,7 @@ public class TsFileIOWriter {
.
add
(
new
ChunkGroupMetadata
(
currentChunkGroupDeviceId
,
chunkMetadataList
));
currentChunkGroupDeviceId
=
null
;
chunkMetadataList
=
null
;
out
.
flush
();
}
/**
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录