Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
apache
Iotdb
提交
2aefcb31
I
Iotdb
项目概览
apache
/
Iotdb
11 个月 前同步成功
通知
25
Star
3344
Fork
916
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
I
Iotdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
2aefcb31
编写于
8月 12, 2022
作者:
Z
ZhangHongYin
提交者:
GitHub
8月 12, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[IOTDB-4000] Add the control of the size of wal folder in multiLeader consensus. (#6836)
上级
ea70aea2
变更
11
隐藏空白更改
内联
并排
Showing
11 changed file
with
121 addition
and
5 deletion
+121
-5
consensus/src/main/java/org/apache/iotdb/consensus/config/MultiLeaderConfig.java
.../org/apache/iotdb/consensus/config/MultiLeaderConfig.java
+44
-2
consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
...he/iotdb/consensus/multileader/MultiLeaderServerImpl.java
+24
-3
consensus/src/main/java/org/apache/iotdb/consensus/multileader/logdispatcher/LogDispatcher.java
...db/consensus/multileader/logdispatcher/LogDispatcher.java
+6
-0
consensus/src/main/java/org/apache/iotdb/consensus/multileader/wal/ConsensusReqReader.java
...e/iotdb/consensus/multileader/wal/ConsensusReqReader.java
+3
-0
consensus/src/test/java/org/apache/iotdb/consensus/multileader/util/FakeConsensusReqReader.java
...db/consensus/multileader/util/FakeConsensusReqReader.java
+5
-0
server/src/assembly/resources/conf/iotdb-datanode.properties
server/src/assembly/resources/conf/iotdb-datanode.properties
+5
-0
server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
...r/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+11
-0
server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
...c/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+9
-0
server/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
...rg/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
+4
-0
server/src/main/java/org/apache/iotdb/db/wal/node/WALFakeNode.java
...c/main/java/org/apache/iotdb/db/wal/node/WALFakeNode.java
+5
-0
server/src/main/java/org/apache/iotdb/db/wal/node/WALNode.java
...r/src/main/java/org/apache/iotdb/db/wal/node/WALNode.java
+5
-0
未找到文件。
consensus/src/main/java/org/apache/iotdb/consensus/config/MultiLeaderConfig.java
浏览文件 @
2aefcb31
...
...
@@ -202,6 +202,9 @@ public class MultiLeaderConfig {
private
final
int
maxWaitingTimeForAccumulatingBatchInMs
;
private
final
long
basicRetryWaitTimeMs
;
private
final
long
maxRetryWaitTimeMs
;
private
final
long
throttleDownThreshold
;
private
final
long
throttleUpThreshold
;
private
final
long
throttleTimeOutMs
;
private
Replication
(
int
maxPendingRequestNumPerNode
,
...
...
@@ -209,13 +212,19 @@ public class MultiLeaderConfig {
int
maxPendingBatch
,
int
maxWaitingTimeForAccumulatingBatchInMs
,
long
basicRetryWaitTimeMs
,
long
maxRetryWaitTimeMs
)
{
long
maxRetryWaitTimeMs
,
long
throttleDownThreshold
,
long
throttleUpThreshold
,
long
throttleTimeOutMs
)
{
this
.
maxPendingRequestNumPerNode
=
maxPendingRequestNumPerNode
;
this
.
maxRequestPerBatch
=
maxRequestPerBatch
;
this
.
maxPendingBatch
=
maxPendingBatch
;
this
.
maxWaitingTimeForAccumulatingBatchInMs
=
maxWaitingTimeForAccumulatingBatchInMs
;
this
.
basicRetryWaitTimeMs
=
basicRetryWaitTimeMs
;
this
.
maxRetryWaitTimeMs
=
maxRetryWaitTimeMs
;
this
.
throttleDownThreshold
=
throttleDownThreshold
;
this
.
throttleUpThreshold
=
throttleUpThreshold
;
this
.
throttleTimeOutMs
=
throttleTimeOutMs
;
}
public
int
getMaxPendingRequestNumPerNode
()
{
...
...
@@ -242,6 +251,18 @@ public class MultiLeaderConfig {
return
maxRetryWaitTimeMs
;
}
public
long
getThrottleDownThreshold
()
{
return
throttleDownThreshold
;
}
public
long
getThrottleUpThreshold
()
{
return
throttleUpThreshold
;
}
public
long
getThrottleTimeOutMs
()
{
return
throttleTimeOutMs
;
}
public
static
Replication
.
Builder
newBuilder
()
{
return
new
Replication
.
Builder
();
}
...
...
@@ -255,6 +276,9 @@ public class MultiLeaderConfig {
private
int
maxWaitingTimeForAccumulatingBatchInMs
=
500
;
private
long
basicRetryWaitTimeMs
=
TimeUnit
.
MILLISECONDS
.
toMillis
(
100
);
private
long
maxRetryWaitTimeMs
=
TimeUnit
.
SECONDS
.
toMillis
(
20
);
private
long
throttleDownThreshold
=
50
*
1024
*
1024
*
1024L
;
private
long
throttleUpThreshold
=
1024
*
1024
*
1024L
;
private
long
throttleTimeOutMs
=
TimeUnit
.
SECONDS
.
toMillis
(
30
);
public
Replication
.
Builder
setMaxPendingRequestNumPerNode
(
int
maxPendingRequestNumPerNode
)
{
this
.
maxPendingRequestNumPerNode
=
maxPendingRequestNumPerNode
;
...
...
@@ -287,6 +311,21 @@ public class MultiLeaderConfig {
return
this
;
}
public
Replication
.
Builder
setThrottleDownThreshold
(
long
throttleDownThreshold
)
{
this
.
throttleDownThreshold
=
throttleDownThreshold
;
return
this
;
}
public
Replication
.
Builder
setThrottleUpThreshold
(
long
throttleUpThreshold
)
{
this
.
throttleUpThreshold
=
throttleUpThreshold
;
return
this
;
}
public
Replication
.
Builder
setThrottleTimeOutMs
(
long
throttleTimeOutMs
)
{
this
.
throttleTimeOutMs
=
throttleTimeOutMs
;
return
this
;
}
public
Replication
build
()
{
return
new
Replication
(
maxPendingRequestNumPerNode
,
...
...
@@ -294,7 +333,10 @@ public class MultiLeaderConfig {
maxPendingBatch
,
maxWaitingTimeForAccumulatingBatchInMs
,
basicRetryWaitTimeMs
,
maxRetryWaitTimeMs
);
maxRetryWaitTimeMs
,
throttleDownThreshold
,
throttleUpThreshold
,
throttleTimeOutMs
);
}
}
}
...
...
consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
浏览文件 @
2aefcb31
...
...
@@ -61,6 +61,7 @@ public class MultiLeaderServerImpl {
private
final
AtomicLong
index
;
private
final
LogDispatcher
logDispatcher
;
private
final
MultiLeaderConfig
config
;
private
final
ConsensusReqReader
reader
;
public
MultiLeaderServerImpl
(
String
storageDir
,
...
...
@@ -80,9 +81,7 @@ public class MultiLeaderServerImpl {
}
this
.
config
=
config
;
this
.
logDispatcher
=
new
LogDispatcher
(
this
,
clientManager
);
// restart
ConsensusReqReader
reader
=
(
ConsensusReqReader
)
stateMachine
.
read
(
new
GetConsensusReqReaderPlan
());
reader
=
(
ConsensusReqReader
)
stateMachine
.
read
(
new
GetConsensusReqReaderPlan
());
long
currentSearchIndex
=
reader
.
getCurrentSearchIndex
();
if
(
1
==
configuration
.
size
())
{
// only one configuration means single replica.
...
...
@@ -110,6 +109,18 @@ public class MultiLeaderServerImpl {
*/
public
TSStatus
write
(
IConsensusRequest
request
)
{
synchronized
(
stateMachine
)
{
if
(
needToThrottleDown
())
{
logger
.
info
(
"[Throttle Down] index:{}, safeIndex:{}"
,
getIndex
(),
getCurrentSafelyDeletedSearchIndex
());
try
{
stateMachine
.
wait
(
config
.
getReplication
().
getThrottleTimeOutMs
());
}
catch
(
InterruptedException
e
)
{
logger
.
error
(
"Failed to throttle down because "
,
e
);
Thread
.
currentThread
().
interrupt
();
}
}
IndexedConsensusRequest
indexedConsensusRequest
=
buildIndexedConsensusRequestForLocalRequest
(
request
);
if
(
indexedConsensusRequest
.
getSearchIndex
()
%
1000
==
0
)
{
...
...
@@ -226,6 +237,16 @@ public class MultiLeaderServerImpl {
return
config
;
}
public
boolean
needToThrottleDown
()
{
return
reader
.
getTotalSize
()
>
config
.
getReplication
().
getThrottleDownThreshold
();
}
public
boolean
needToThrottleUp
()
{
return
reader
.
getTotalSize
()
<
config
.
getReplication
().
getThrottleDownThreshold
()
-
config
.
getReplication
().
getThrottleUpThreshold
();
}
public
AtomicLong
getIndexObject
()
{
return
index
;
}
...
...
consensus/src/main/java/org/apache/iotdb/consensus/multileader/logdispatcher/LogDispatcher.java
浏览文件 @
2aefcb31
...
...
@@ -222,6 +222,12 @@ public class LogDispatcher {
// indicating that insert nodes whose search index are before this value can be deleted
// safely
reader
.
setSafelyDeletedSearchIndex
(
impl
.
getCurrentSafelyDeletedSearchIndex
());
// notify
if
(
impl
.
needToThrottleUp
())
{
synchronized
(
impl
.
getStateMachine
())
{
impl
.
getStateMachine
().
notifyAll
();
}
}
}
public
PendingBatch
getBatch
()
{
...
...
consensus/src/main/java/org/apache/iotdb/consensus/multileader/wal/ConsensusReqReader.java
浏览文件 @
2aefcb31
...
...
@@ -79,4 +79,7 @@ public interface ConsensusReqReader {
/** Get current search index */
long
getCurrentSearchIndex
();
/** Get total size of wal files */
long
getTotalSize
();
}
consensus/src/test/java/org/apache/iotdb/consensus/multileader/util/FakeConsensusReqReader.java
浏览文件 @
2aefcb31
...
...
@@ -47,6 +47,11 @@ public class FakeConsensusReqReader implements ConsensusReqReader, DataSet {
return
requestSets
.
getLocalRequestNumber
();
}
@Override
public
long
getTotalSize
()
{
return
0
;
}
private
class
FakeConsensusReqIterator
implements
ConsensusReqReader
.
ReqIterator
{
private
long
nextSearchIndex
;
...
...
server/src/assembly/resources/conf/iotdb-datanode.properties
浏览文件 @
2aefcb31
...
...
@@ -178,6 +178,11 @@ target_config_nodes=127.0.0.1:22277
# Datatype: long
# delete_wal_files_period_in_ms=20000
# The minimum size of wal files when throttle down in MultiLeader consensus
# If it's a value smaller than 0, use the default value 50 * 1024 * 1024 * 1024 bytes (50GB).
# Datatype: long
# multi_leader_throttle_down_threshold_in_byte=53687091200
####################
### Directory Configuration
####################
...
...
server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
浏览文件 @
2aefcb31
...
...
@@ -948,6 +948,9 @@ public class IoTDBConfig {
/** Maximum execution time of a DriverTask */
private
int
driverTaskExecutionTimeSliceInMs
=
100
;
/** Maximum size of wal buffer used in MultiLeader consensus. Unit: byte */
private
long
throttleDownThreshold
=
50
*
1024
*
1024
*
1024L
;
IoTDBConfig
()
{}
public
float
getUdfMemoryBudgetInMB
()
{
...
...
@@ -3016,4 +3019,12 @@ public class IoTDBConfig {
public
void
setDriverTaskExecutionTimeSliceInMs
(
int
driverTaskExecutionTimeSliceInMs
)
{
this
.
driverTaskExecutionTimeSliceInMs
=
driverTaskExecutionTimeSliceInMs
;
}
public
long
getThrottleDownThreshold
()
{
return
throttleDownThreshold
;
}
public
void
setThrottleDownThreshold
(
long
throttleDownThreshold
)
{
this
.
throttleDownThreshold
=
throttleDownThreshold
;
}
}
server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
浏览文件 @
2aefcb31
...
...
@@ -1073,6 +1073,15 @@ public class IoTDBDescriptor {
if
(
deleteWalFilesPeriod
>
0
)
{
conf
.
setDeleteWalFilesPeriodInMs
(
deleteWalFilesPeriod
);
}
long
throttleDownThresholdInByte
=
Long
.
parseLong
(
properties
.
getProperty
(
"multi_leader_throttle_down_threshold_in_byte"
,
Long
.
toString
(
conf
.
getThrottleDownThreshold
())));
if
(
throttleDownThresholdInByte
>
0
)
{
conf
.
setThrottleDownThreshold
(
throttleDownThresholdInByte
);
}
}
private
void
loadAutoCreateSchemaProps
(
Properties
properties
)
{
...
...
server/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
浏览文件 @
2aefcb31
...
...
@@ -77,6 +77,10 @@ public class DataRegionConsensusImpl {
conf
.
getThriftServerAwaitTimeForStopService
())
.
setThriftMaxFrameSize
(
conf
.
getThriftMaxFrameSize
())
.
build
())
.
setReplication
(
MultiLeaderConfig
.
Replication
.
newBuilder
()
.
setThrottleDownThreshold
(
conf
.
getThrottleDownThreshold
())
.
build
())
.
build
())
.
setRatisConfig
(
RatisConfig
.
newBuilder
()
...
...
server/src/main/java/org/apache/iotdb/db/wal/node/WALFakeNode.java
浏览文件 @
2aefcb31
...
...
@@ -116,6 +116,11 @@ public class WALFakeNode implements IWALNode {
throw
new
UnsupportedOperationException
();
}
@Override
public
long
getTotalSize
()
{
return
0
;
}
public
static
WALFakeNode
getFailureInstance
(
Exception
e
)
{
return
new
WALFakeNode
(
WALFlushListener
.
Status
.
FAILURE
,
...
...
server/src/main/java/org/apache/iotdb/db/wal/node/WALNode.java
浏览文件 @
2aefcb31
...
...
@@ -749,6 +749,11 @@ public class WALNode implements IWALNode {
return
buffer
.
getCurrentSearchIndex
();
}
@Override
public
long
getTotalSize
()
{
return
WALManager
.
getInstance
().
getTotalDiskUsage
();
}
// endregion
@Override
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录