Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
indiff7643
Terarkdb
提交
c0077294
T
Terarkdb
项目概览
indiff7643
/
Terarkdb
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
Terarkdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c0077294
编写于
6月 05, 2021
作者:
Z
ZhaoMing
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix target_blob_file_size usage
上级
4e4a04d4
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
38 addition
and
22 deletion
+38
-22
db/builder.cc
db/builder.cc
+4
-1
db/compaction_job.cc
db/compaction_job.cc
+12
-10
db/compaction_picker.cc
db/compaction_picker.cc
+8
-11
options/cf_options.cc
options/cf_options.cc
+10
-0
options/cf_options.h
options/cf_options.h
+4
-0
未找到文件。
db/builder.cc
浏览文件 @
c0077294
...
...
@@ -232,13 +232,16 @@ Status BuildTable(
return
s
;
};
size_t
target_blob_file_size
=
MaxBlobSize
(
mutable_cf_options
,
ioptions
.
num_levels
,
ioptions
.
compaction_style
);
auto
trans_to_separate
=
[
&
](
const
Slice
&
key
,
LazyBuffer
&
value
)
{
assert
(
value
.
file_number
()
==
uint64_t
(
-
1
));
Status
status
;
TableBuilder
*
blob_builder
=
separate_helper
.
builder
.
get
();
FileMetaData
*
blob_meta
=
separate_helper
.
current_output
;
if
(
blob_builder
!=
nullptr
&&
blob_builder
->
FileSize
()
>
mutable_cf_options
.
target_file_size_bas
e
)
{
blob_builder
->
FileSize
()
>
target_blob_file_siz
e
)
{
status
=
finish_output_blob_sst
();
blob_builder
=
nullptr
;
}
...
...
db/compaction_job.cc
浏览文件 @
c0077294
...
...
@@ -417,15 +417,13 @@ struct CompactionJob::SubcompactionState {
// pop smaller scores if total_compaction_bytes > max_compaction_bytes
// remain one blob al least.
size_t
max_file_size
=
mutable_cf_options
->
target_blob_file_size
;
if
(
max_file_size
==
0
)
{
max_file_size
=
MaxFileSizeForLevel
(
*
mutable_cf_options
,
compaction
->
immutable_cf_options
()
->
num_levels
-
1
,
compaction
->
immutable_cf_options
()
->
compaction_style
);
}
uint64_t
target_compaction_bytes
=
std
::
max
(
max_compaction_bytes
,
input_blob_info
.
input_bytes
+
max_file_size
);
size_t
target_blob_file_size
=
MaxBlobSize
(
*
mutable_cf_options
,
compaction
->
immutable_cf_options
()
->
num_levels
,
compaction
->
immutable_cf_options
()
->
compaction_style
);
uint64_t
target_compaction_bytes
=
std
::
max
(
max_compaction_bytes
,
input_blob_info
.
input_bytes
+
target_blob_file_size
);
while
(
total_compaction_bytes
>
target_compaction_bytes
&&
target_range_blob_heap
.
size
()
>
1
)
{
total_compaction_bytes
-=
target_range_blob_heap
.
top
().
ref_bytes
;
...
...
@@ -1614,12 +1612,16 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
->
value_meta_extractor_factory
->
CreateValueExtractor
(
context
);
}
size_t
target_blob_file_size
=
MaxBlobSize
(
*
mutable_cf_options
,
cfd
->
ioptions
()
->
num_levels
,
cfd
->
ioptions
()
->
compaction_style
);
auto
trans_to_separate
=
[
&
](
const
Slice
&
key
,
LazyBuffer
&
value
)
{
Status
s
;
TableBuilder
*
blob_builder
=
sub_compact
->
blob_builder
.
get
();
FileMetaData
*
blob_meta
=
&
sub_compact
->
current_blob_output
()
->
meta
;
if
(
blob_builder
!=
nullptr
&&
blob_builder
->
FileSize
()
>
mutable_cf_options
->
target_file_size_bas
e
)
{
blob_builder
->
FileSize
()
>
target_blob_file_siz
e
)
{
s
=
FinishCompactionOutputBlob
(
s
,
sub_compact
,
{});
blob_builder
=
nullptr
;
}
...
...
db/compaction_picker.cc
浏览文件 @
c0077294
...
...
@@ -795,19 +795,16 @@ void CompactionPicker::GetGrandparents(
Compaction
*
CompactionPicker
::
PickGarbageCollection
(
const
std
::
string
&
/*cf_name*/
,
const
MutableCFOptions
&
mutable_cf_options
,
VersionStorageInfo
*
vstorage
,
LogBuffer
*
/*log_buffer*/
)
{
// Setting fragment_size as one eighth max_file_size prevents selecting
// massive files to single compaction which would pin down the maximum
// deletable file number for a long time resulting possible storage leakage.
size_t
max_file_size
=
mutable_cf_options
.
target_blob_file_size
;
if
(
max_file_size
==
0
)
{
max_file_size
=
MaxFileSizeForLevel
(
mutable_cf_options
,
ioptions_
.
num_levels
-
1
,
ioptions_
.
compaction_style
);
}
// Setting fragment_size as one eighth target_blob_file_size prevents
// selecting massive files to single compaction which would pin down the
// maximum deletable file number for a long time resulting possible storage
// leakage.
size_t
target_blob_file_size
=
MaxBlobSize
(
mutable_cf_options
,
ioptions_
.
num_levels
,
ioptions_
.
compaction_style
);
size_t
fragment_size
=
mutable_cf_options
.
blob_file_defragment_size
;
if
(
fragment_size
==
0
)
{
fragment_size
=
max
_file_size
/
8
;
fragment_size
=
target_blob
_file_size
/
8
;
}
auto
&
hidden_files
=
vstorage
->
LevelFiles
(
-
1
);
...
...
@@ -905,7 +902,7 @@ Compaction* CompactionPicker::PickGarbageCollection(
while
(
!
candidate_blob_vec
.
empty
()
&&
input
.
files
.
size
()
<
8
)
{
auto
f
=
candidate_blob_vec
.
front
().
f
;
uint64_t
estimate_size
=
candidate_blob_vec
.
front
().
estimate_size
;
if
(
total_estimate_size
+
estimate_size
<
max
_file_size
)
{
if
(
total_estimate_size
+
estimate_size
<
target_blob
_file_size
)
{
total_estimate_size
+=
estimate_size
;
num_antiquation
+=
f
->
num_antiquation
;
f
->
set_gc_candidate
();
...
...
options/cf_options.cc
浏览文件 @
c0077294
...
...
@@ -126,6 +126,16 @@ uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options, int level,
}
}
uint64_t
MaxBlobSize
(
const
MutableCFOptions
&
cf_options
,
int
num_levels
,
CompactionStyle
compaction_style
)
{
size_t
target_blob_file_size
=
cf_options
.
target_blob_file_size
;
if
(
target_blob_file_size
==
0
)
{
target_blob_file_size
=
MaxFileSizeForLevel
(
cf_options
,
num_levels
-
1
,
compaction_style
);
}
return
target_blob_file_size
;
}
void
MutableCFOptions
::
RefreshDerivedOptions
(
int
num_levels
)
{
max_file_size
.
resize
(
num_levels
);
max_file_size
[
0
]
=
0
;
// unlimited
...
...
options/cf_options.h
浏览文件 @
c0077294
...
...
@@ -260,4 +260,8 @@ uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options, int level,
CompactionStyle
compaction_style
,
int
base_level
=
1
,
bool
level_compaction_dynamic_level_bytes
=
false
);
uint64_t
MaxBlobSize
(
const
MutableCFOptions
&
cf_options
,
int
num_levels
,
CompactionStyle
compaction_style
);
}
// namespace TERARKDB_NAMESPACE
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录