Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
indiff7643
Terarkdb
提交
33c36fcb
T
Terarkdb
项目概览
indiff7643
/
Terarkdb
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
Terarkdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
33c36fcb
编写于
4月 21, 2021
作者:
Z
ZhaoMing
提交者:
奏之章
6月 02, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Refactor inheritance_chain to inheritance_tree
上级
e8655842
变更
28
隐藏空白更改
内联
并排
Showing
28 changed file
with
218 addition
and
94 deletion
+218
-94
.gitignore
.gitignore
+1
-0
db/compaction.cc
db/compaction.cc
+91
-0
db/compaction.h
db/compaction.h
+9
-0
db/compaction_dispatcher.cc
db/compaction_dispatcher.cc
+1
-1
db/compaction_job.cc
db/compaction_job.cc
+19
-27
db/compaction_job.h
db/compaction_job.h
+1
-1
db/repair.cc
db/repair.cc
+1
-1
db/version_builder.cc
db/version_builder.cc
+2
-2
db/version_builder_test.cc
db/version_builder_test.cc
+9
-7
db/version_edit.cc
db/version_edit.cc
+4
-4
db/version_edit.h
db/version_edit.h
+10
-10
db/version_edit_test.cc
db/version_edit_test.cc
+2
-3
db/version_set.cc
db/version_set.cc
+2
-2
include/rocksdb/table_properties.h
include/rocksdb/table_properties.h
+3
-2
table/block_based_table_builder.cc
table/block_based_table_builder.cc
+5
-2
table/block_based_table_builder.h
table/block_based_table_builder.h
+2
-1
table/cuckoo_table_builder.cc
table/cuckoo_table_builder.cc
+5
-2
table/cuckoo_table_builder.h
table/cuckoo_table_builder.h
+2
-1
table/cuckoo_table_builder_test.cc
table/cuckoo_table_builder_test.cc
+11
-11
table/cuckoo_table_reader_test.cc
table/cuckoo_table_reader_test.cc
+2
-2
table/meta_blocks.cc
table/meta_blocks.cc
+10
-3
table/mock_table.h
table/mock_table.h
+5
-3
table/plain_table_builder.cc
table/plain_table_builder.cc
+7
-3
table/plain_table_builder.h
table/plain_table_builder.h
+2
-1
table/table_builder.h
table/table_builder.h
+3
-2
table/table_properties.cc
table/table_properties.cc
+2
-0
table/terark_zip_table_builder.cc
table/terark_zip_table_builder.cc
+5
-2
table/terark_zip_table_builder.h
table/terark_zip_table_builder.h
+2
-1
未找到文件。
.gitignore
浏览文件 @
33c36fcb
...
...
@@ -94,3 +94,4 @@ zlib-1.2.11.tar.gz
zlib-1.2.11/
zstd-1.3.3.tar.gz
.vscode
include/rocksdb/terark_namespace.h
db/compaction.cc
浏览文件 @
33c36fcb
...
...
@@ -85,6 +85,97 @@ const char* CompactionTypeName(CompactionType type) {
}
}
Status
BuildInheritanceTree
(
const
std
::
vector
<
CompactionInputFiles
>&
inputs
,
const
DependenceMap
&
dependence_map
,
const
Version
*
version
,
std
::
vector
<
uint64_t
>*
inheritance_tree
,
size_t
*
pruge_count_ptr
)
{
// Purging the leaf nodes recursively with inheritance tree encoded by DFS
// sequence.
//
// GC job:
// input sst = [10,9,11]
// sst 10 = [3,1,1,2,2,3,7,4,4,5,5,6,6,7]
// sst 9 = [8,8]
// sst 11 = []
// output sst = 12
// dependence_map = {5,6,8,9,10,11}
//
// 1 2 4 5 6
// \ / \ /
// 3 7 8
// \ / |
// sst 10 + sst 9 + sst 11
//
// After purge & merge, we got output inheritance tree
//
// 5 6
// \ /
// 7 8
// | |
// 10 9 11
// \ | /
// sst 12
//
// sst 12 = [10,7,5,5,6,6,7,10,9,8,8,9,11,11]
Status
s
;
size_t
pruge_count
=
0
;
inheritance_tree
->
clear
();
for
(
auto
&
level
:
inputs
)
{
for
(
auto
f
:
level
.
files
)
{
std
::
shared_ptr
<
const
TableProperties
>
tp
;
s
=
version
->
GetTableProperties
(
&
tp
,
f
);
if
(
!
s
.
ok
())
{
return
s
;
}
inheritance_tree
->
emplace_back
(
f
->
fd
.
GetNumber
());
// 1. verify input inheritance tree
// 2. purge unnecessary node
// 3. merge input inheritance tree
size_t
size
=
inheritance_tree
->
size
();
size_t
count
=
0
;
for
(
auto
file_number
:
tp
->
inheritance_tree
)
{
if
(
inheritance_tree
->
back
()
!=
file_number
)
{
inheritance_tree
->
emplace_back
(
file_number
);
}
else
if
(
dependence_map
.
count
(
file_number
)
!=
0
)
{
++
count
;
inheritance_tree
->
emplace_back
(
file_number
);
}
else
{
++
pruge_count
;
inheritance_tree
->
pop_back
();
if
(
inheritance_tree
->
size
()
<
size
)
{
break
;
}
}
}
if
(
size
+
count
*
2
!=
inheritance_tree
->
size
())
{
// input inheritance tree invalid !
return
Status
::
Corruption
(
"BuildInheritanceTree: bad inheritance_tree, file_number "
,
ToString
(
f
->
fd
.
GetNumber
()));
}
inheritance_tree
->
emplace_back
(
f
->
fd
.
GetNumber
());
}
}
if
(
pruge_count_ptr
!=
nullptr
)
{
*
pruge_count_ptr
=
pruge_count
;
}
return
s
;
}
std
::
vector
<
uint64_t
>
InheritanceTreeToSet
(
const
std
::
vector
<
uint64_t
>&
tree
)
{
std
::
vector
<
uint64_t
>
set
=
tree
;
std
::
sort
(
set
.
begin
(),
set
.
end
());
auto
it
=
std
::
unique
(
set
.
begin
(),
set
.
end
());
assert
(
set
.
end
()
-
it
==
it
-
set
.
begin
());
set
.
resize
(
it
-
set
.
begin
());
set
.
shrink_to_fit
();
return
set
;
}
void
Compaction
::
SetInputVersion
(
Version
*
_input_version
)
{
input_version_
=
_input_version
;
cfd_
=
input_version_
->
cfd
();
...
...
db/compaction.h
浏览文件 @
33c36fcb
...
...
@@ -17,6 +17,7 @@
#include "rocksdb/terark_namespace.h"
#include "util/arena.h"
#include "util/autovector.h"
#include "util/iterator_cache.h"
namespace
TERARKDB_NAMESPACE
{
...
...
@@ -531,4 +532,12 @@ extern uint64_t TotalFileSize(const std::vector<FileMetaData*>& files);
extern
const
char
*
CompactionTypeName
(
CompactionType
type
);
extern
Status
BuildInheritanceTree
(
const
std
::
vector
<
CompactionInputFiles
>&
inputs
,
const
DependenceMap
&
dependence_map
,
const
Version
*
version
,
std
::
vector
<
uint64_t
>*
tree
,
size_t
*
pruge_count
);
extern
std
::
vector
<
uint64_t
>
InheritanceTreeToSet
(
const
std
::
vector
<
uint64_t
>&
tree
);
}
// namespace TERARKDB_NAMESPACE
db/compaction_dispatcher.cc
浏览文件 @
33c36fcb
...
...
@@ -219,7 +219,7 @@ AJSON(FileDescriptor, packed_number_and_path_id, file_size, smallest_seqno,
AJSON
(
TablePropertyCache
,
num_entries
,
num_deletions
,
raw_key_size
,
raw_value_size
,
flags
,
purpose
,
max_read_amp
,
read_amp
,
dependence
,
inheritance
_chain
);
inheritance
);
AJSON
(
FileMetaData
,
fd
,
smallest
,
largest
,
prop
);
...
...
db/compaction_job.cc
浏览文件 @
33c36fcb
...
...
@@ -880,7 +880,8 @@ Status CompactionJob::Run() {
output
.
meta
.
prop
.
max_read_amp
=
tp
->
max_read_amp
;
output
.
meta
.
prop
.
read_amp
=
tp
->
read_amp
;
output
.
meta
.
prop
.
dependence
=
tp
->
dependence
;
output
.
meta
.
prop
.
inheritance_chain
=
tp
->
inheritance_chain
;
output
.
meta
.
prop
.
inheritance
=
InheritanceTreeToSet
(
tp
->
inheritance_tree
);
if
(
iopt
->
ttl_extractor_factory
!=
nullptr
)
{
GetCompactionTimePoint
(
tp
->
user_collected_properties
,
...
...
@@ -1854,26 +1855,14 @@ void CompactionJob::ProcessGarbageCollection(SubcompactionState* sub_compact) {
if
(
status
.
ok
())
{
status
=
input
->
status
();
}
std
::
vector
<
uint64_t
>
inheritance_chain
;
size_t
raw_chain_length
=
0
;
for
(
auto
&
level
:
*
sub_compact
->
compaction
->
inputs
())
{
for
(
auto
f
:
level
.
files
)
{
raw_chain_length
+=
f
->
prop
.
inheritance_chain
.
size
()
+
1
;
inheritance_chain
.
push_back
(
f
->
fd
.
GetNumber
());
for
(
size_t
i
=
0
;
i
<
f
->
prop
.
inheritance_chain
.
size
();
++
i
)
{
if
(
dependence_map
.
count
(
f
->
prop
.
inheritance_chain
[
i
])
>
0
)
{
inheritance_chain
.
insert
(
inheritance_chain
.
end
(),
f
->
prop
.
inheritance_chain
.
begin
()
+
i
,
f
->
prop
.
inheritance_chain
.
end
());
break
;
}
}
}
std
::
vector
<
uint64_t
>
inheritance_tree
;
size_t
inheritance_tree_pruge_count
=
0
;
if
(
status
.
ok
())
{
status
=
BuildInheritanceTree
(
*
sub_compact
->
compaction
->
inputs
(),
dependence_map
,
input_version
,
&
inheritance_tree
,
&
inheritance_tree_pruge_count
);
}
std
::
sort
(
inheritance_chain
.
begin
(),
inheritance_chain
.
end
());
assert
(
std
::
unique
(
inheritance_chain
.
begin
(),
inheritance_chain
.
end
())
==
inheritance_chain
.
end
());
Status
s
=
FinishCompactionOutputBlob
(
status
,
sub_compact
,
inheritance_chain
);
Status
s
=
FinishCompactionOutputBlob
(
status
,
sub_compact
,
inheritance_tree
);
if
(
status
.
ok
())
{
status
=
s
;
}
...
...
@@ -1888,13 +1877,14 @@ void CompactionJob::ProcessGarbageCollection(SubcompactionState* sub_compact) {
" inputs from %zd files. %"
PRIu64
" clear, %.2f%% estimation: [ %"
PRIu64
" garbage type, %"
PRIu64
" get not found, %"
PRIu64
" file number mismatch ], inheritance
chain
: %"
PRIu64
" -> %"
PRIu64
,
" file number mismatch ], inheritance
tree
: %"
PRIu64
" -> %"
PRIu64
,
cfd
->
GetName
().
c_str
(),
job_id_
,
meta
.
fd
.
GetNumber
(),
counter
.
input
,
files
.
size
(),
counter
.
input
-
meta
.
prop
.
num_entries
,
sub_compact
->
compaction
->
num_antiquation
()
*
100.
/
counter
.
input
,
counter
.
garbage_type
,
counter
.
get_not_found
,
counter
.
file_number_mismatch
,
raw_chain_length
,
inheritance_chain
.
size
());
counter
.
file_number_mismatch
,
meta
.
prop
.
inheritance
.
size
()
+
inheritance_tree_pruge_count
,
meta
.
prop
.
inheritance
.
size
());
if
((
std
::
find_if
(
files
.
begin
(),
files
.
end
(),
[](
FileMetaData
*
f
)
{
return
f
->
marked_for_compaction
;
...
...
@@ -2210,7 +2200,7 @@ Status CompactionJob::FinishCompactionOutputFile(
Status
CompactionJob
::
FinishCompactionOutputBlob
(
const
Status
&
input_status
,
SubcompactionState
*
sub_compact
,
const
std
::
vector
<
uint64_t
>&
inheritance_
chain
)
{
const
std
::
vector
<
uint64_t
>&
inheritance_
tree
)
{
AutoThreadOperationStageUpdater
stage_updater
(
ThreadStatus
::
STAGE_COMPACTION_SYNC_FILE
);
assert
(
sub_compact
!=
nullptr
);
...
...
@@ -2231,9 +2221,11 @@ Status CompactionJob::FinishCompactionOutputBlob(
if
(
s
.
ok
())
{
meta
->
marked_for_compaction
=
sub_compact
->
blob_builder
->
NeedCompact
();
meta
->
prop
.
num_entries
=
sub_compact
->
blob_builder
->
NumEntries
();
assert
(
std
::
is_sorted
(
inheritance_chain
.
begin
(),
inheritance_chain
.
end
()));
meta
->
prop
.
inheritance_chain
=
inheritance_chain
;
s
=
sub_compact
->
blob_builder
->
Finish
(
&
meta
->
prop
,
nullptr
);
meta
->
prop
.
inheritance
=
InheritanceTreeToSet
(
inheritance_tree
);
assert
(
std
::
is_sorted
(
meta
->
prop
.
inheritance
.
begin
(),
meta
->
prop
.
inheritance
.
end
()));
s
=
sub_compact
->
blob_builder
->
Finish
(
&
meta
->
prop
,
nullptr
,
&
inheritance_tree
);
}
else
{
sub_compact
->
blob_builder
->
Abandon
();
}
...
...
db/compaction_job.h
浏览文件 @
33c36fcb
...
...
@@ -131,7 +131,7 @@ class CompactionJob {
const
Slice
*
next_table_min_key
=
nullptr
);
Status
FinishCompactionOutputBlob
(
const
Status
&
input_status
,
SubcompactionState
*
sub_compact
,
const
std
::
vector
<
uint64_t
>&
inheritance_
chain
);
const
std
::
vector
<
uint64_t
>&
inheritance_
tree
);
Status
InstallCompactionResults
(
const
MutableCFOptions
&
mutable_cf_options
);
void
RecordCompactionIOStats
();
Status
OpenCompactionOutputFile
(
SubcompactionState
*
sub_compact
);
...
...
db/repair.cc
浏览文件 @
33c36fcb
...
...
@@ -674,7 +674,7 @@ class Repairer {
t
->
meta
.
prop
.
max_read_amp
=
props
->
max_read_amp
;
t
->
meta
.
prop
.
read_amp
=
props
->
read_amp
;
t
->
meta
.
prop
.
dependence
=
props
->
dependence
;
t
->
meta
.
prop
.
inheritance
_chain
=
props
->
inheritance_chain
;
t
->
meta
.
prop
.
inheritance
=
InheritanceTreeToSet
(
props
->
inheritance_tree
)
;
}
return
status
;
}
...
...
db/version_builder.cc
浏览文件 @
33c36fcb
...
...
@@ -216,7 +216,7 @@ class VersionBuilder::Rep {
}
}
};
for
(
auto
file_number
:
item
->
f
->
prop
.
inheritance
_chain
)
{
for
(
auto
file_number
:
item
->
f
->
prop
.
inheritance
)
{
emplace
(
file_number
);
}
emplace
(
item
->
f
->
fd
.
GetNumber
());
...
...
@@ -230,7 +230,7 @@ class VersionBuilder::Rep {
inheritance_counter_
.
erase
(
find
);
}
};
for
(
auto
file_number
:
f
->
prop
.
inheritance
_chain
)
{
for
(
auto
file_number
:
f
->
prop
.
inheritance
)
{
erase
(
file_number
);
}
erase
(
f
->
fd
.
GetNumber
());
...
...
db/version_builder_test.cc
浏览文件 @
33c36fcb
...
...
@@ -4,6 +4,7 @@
// (found in the LICENSE.Apache file in the root directory).
#include <inttypes.h>
#include <string>
#include "db/version_edit.h"
...
...
@@ -115,13 +116,13 @@ bool VerifyDependFiles(VersionStorageInfo* new_vstorage,
TablePropertyCache
GetPropCache
(
uint8_t
purpose
,
std
::
initializer_list
<
uint64_t
>
dependence
=
{},
std
::
initializer_list
<
uint64_t
>
inheritance
_chain
=
{})
{
std
::
initializer_list
<
uint64_t
>
inheritance
=
{})
{
std
::
vector
<
Dependence
>
dep
;
for
(
auto
&
d
:
dependence
)
dep
.
emplace_back
(
Dependence
{
d
,
1
});
TablePropertyCache
ret
;
ret
.
purpose
=
purpose
;
ret
.
dependence
=
dep
;
ret
.
inheritance
_chain
=
inheritance_chain
;
ret
.
inheritance
=
inheritance
;
return
ret
;
}
...
...
@@ -434,7 +435,7 @@ TEST_F(VersionBuilderTest, HugeLSM) {
TablePropertyCache
prop
;
for
(
uint32_t
j
=
i
*
32
+
1
,
je
=
j
+
kFilesBlobInheritance
-
1
;
j
<
je
;
++
j
)
{
prop
.
inheritance
_chain
.
emplace_back
(
j
);
prop
.
inheritance
.
emplace_back
(
j
);
}
Add
(
-
1
,
(
i
+
1
)
*
32
,
"0"
,
"1"
,
100
,
0
,
0
,
100
,
100000
,
0
,
0
,
100
,
prop
);
}
...
...
@@ -465,8 +466,9 @@ TEST_F(VersionBuilderTest, HugeLSM) {
for
(
uint32_t
level
=
1
;
level
<
kNumLevels
;
++
level
)
{
for
(
uint32_t
i
=
0
;
i
<
level_file_count
;
++
i
)
{
Add
(
level
,
fn
++
,
to_fix_string
(
i
*
2
).
c_str
(),
to_fix_string
(
i
*
2
+
1
).
c_str
(),
100
,
0
,
0
,
100
,
kFilesBlobDependence
,
0
,
0
,
100
,
make_prop
());
Add
(
level
,
fn
++
,
to_fix_string
(
i
*
2
).
c_str
(),
to_fix_string
(
i
*
2
+
1
).
c_str
(),
100
,
0
,
0
,
100
,
kFilesBlobDependence
,
0
,
0
,
100
,
make_prop
());
}
level_file_count
*=
kFilesPerLevelMultiplier
;
}
...
...
@@ -481,8 +483,8 @@ TEST_F(VersionBuilderTest, HugeLSM) {
version_edit
.
AddFile
(
1
,
fn
,
0
,
100
,
GetInternalKey
(
to_fix_string
(
kFilesPerLevel
*
2
).
c_str
(),
0
),
GetInternalKey
(
to_fix_string
(
kFilesPerLevel
*
2
+
1
).
c_str
(),
100
),
0
,
100
,
false
,
make_prop
());
GetInternalKey
(
to_fix_string
(
kFilesPerLevel
*
2
+
1
).
c_str
(),
100
),
0
,
100
,
false
,
make_prop
());
version_edit
.
DeleteFile
(
1
,
kFilesBlobCount
*
kFilesBlobInheritance
+
1
);
version_builder
.
Apply
(
&
version_edit
);
...
...
db/version_edit.cc
浏览文件 @
33c36fcb
...
...
@@ -196,8 +196,8 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
PutVarint64
(
&
encode_property_cache
,
f
.
prop
.
num_entries
);
PutVarint32Varint64
(
&
encode_property_cache
,
f
.
prop
.
max_read_amp
,
DoubleToU64
(
f
.
prop
.
read_amp
));
PutVarint64
(
&
encode_property_cache
,
f
.
prop
.
inheritance
_chain
.
size
());
for
(
auto
file_number
:
f
.
prop
.
inheritance
_chain
)
{
PutVarint64
(
&
encode_property_cache
,
f
.
prop
.
inheritance
.
size
());
for
(
auto
file_number
:
f
.
prop
.
inheritance
)
{
PutVarint64
(
&
encode_property_cache
,
file_number
);
}
for
(
auto
&
dependence
:
f
.
prop
.
dependence
)
{
...
...
@@ -350,13 +350,13 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
}
f
.
prop
.
max_read_amp
=
uint16_t
(
max_read_amp
);
f
.
prop
.
read_amp
=
U64ToDouble
(
read_amp
);
f
.
prop
.
inheritance
_chain
.
reserve
(
size
);
f
.
prop
.
inheritance
.
reserve
(
size
);
for
(
size_t
i
=
0
;
i
<
size
;
++
i
)
{
uint64_t
file_number
;
if
(
!
GetVarint64
(
&
field
,
&
file_number
))
{
return
error_msg
;
}
f
.
prop
.
inheritance
_chain
.
emplace_back
(
file_number
);
f
.
prop
.
inheritance
.
emplace_back
(
file_number
);
}
}
if
(
!
field
.
empty
())
{
...
...
db/version_edit.h
浏览文件 @
33c36fcb
...
...
@@ -92,16 +92,16 @@ struct TablePropertyCache {
kHasSnapshots
=
1ULL
<<
1
,
kNoRangeDeletions
=
1ULL
<<
2
,
};
uint64_t
num_entries
=
0
;
// the number of entries.
uint64_t
num_deletions
=
0
;
// the number of deletion entries.
uint64_t
raw_key_size
=
0
;
// total uncompressed key size.
uint64_t
raw_value_size
=
0
;
// total uncompressed value size.
uint8_t
flags
=
0
;
// save flags
uint8_t
purpose
=
0
;
// zero for essence sst
uint16_t
max_read_amp
=
1
;
// max read amp from sst
float
read_amp
=
1
;
// expt read amp from sst
std
::
vector
<
Dependence
>
dependence
;
// make these sst hidden
std
::
vector
<
uint64_t
>
inheritance
_chain
;
// inheritance chain
uint64_t
num_entries
=
0
;
// the number of entries.
uint64_t
num_deletions
=
0
;
// the number of deletion entries.
uint64_t
raw_key_size
=
0
;
// total uncompressed key size.
uint64_t
raw_value_size
=
0
;
// total uncompressed value size.
uint8_t
flags
=
0
;
// save flags
uint8_t
purpose
=
0
;
// zero for essence sst
uint16_t
max_read_amp
=
1
;
// max read amp from sst
float
read_amp
=
1
;
// expt read amp from sst
std
::
vector
<
Dependence
>
dependence
;
// make these sst hidden
std
::
vector
<
uint64_t
>
inheritance
;
// inheritance set
uint64_t
earliest_time_begin_compact
=
port
::
kMaxUint64
;
uint64_t
latest_time_end_compact
=
port
::
kMaxUint64
;
...
...
db/version_edit_test.cc
浏览文件 @
33c36fcb
...
...
@@ -18,11 +18,10 @@ namespace TERARKDB_NAMESPACE {
namespace
{
TablePropertyCache
GetPropCache
(
uint8_t
purpose
,
std
::
initializer_list
<
uint64_t
>
dependence
=
{},
std
::
initializer_list
<
uint64_t
>
inheritance
_chain
=
{})
{
std
::
initializer_list
<
uint64_t
>
inheritance
=
{})
{
std
::
vector
<
Dependence
>
dep
;
for
(
auto
&
d
:
dependence
)
dep
.
emplace_back
(
Dependence
{
d
,
1
});
return
TablePropertyCache
{
0
,
0
,
1
,
1
,
0
,
purpose
,
0
,
0
,
dep
,
inheritance_chain
};
return
TablePropertyCache
{
0
,
0
,
1
,
1
,
0
,
purpose
,
0
,
0
,
dep
,
inheritance
};
}
}
// namespace
...
...
db/version_set.cc
浏览文件 @
33c36fcb
...
...
@@ -1893,12 +1893,12 @@ void VersionStorageInfo::AddFile(int level, FileMetaData* f,
// When this function is not set, dependence_map_ will update with subject
// file's property.
if
(
exists
==
nullptr
)
{
for
(
auto
file_number
:
f
->
prop
.
inheritance
_chain
)
{
for
(
auto
file_number
:
f
->
prop
.
inheritance
)
{
assert
(
dependence_map_
.
count
(
file_number
)
==
0
);
dependence_map_
.
emplace
(
file_number
,
f
);
}
}
else
{
for
(
auto
file_number
:
f
->
prop
.
inheritance
_chain
)
{
for
(
auto
file_number
:
f
->
prop
.
inheritance
)
{
assert
(
dependence_map_
.
count
(
file_number
)
==
0
);
if
(
exists
(
exists_args
,
file_number
))
{
dependence_map_
.
emplace
(
file_number
,
f
);
...
...
include/rocksdb/table_properties.h
浏览文件 @
33c36fcb
...
...
@@ -68,6 +68,7 @@ struct TablePropertiesNames {
static
const
std
::
string
kDependence
;
static
const
std
::
string
kDependenceEntryCount
;
static
const
std
::
string
kInheritanceChain
;
static
const
std
::
string
kInheritanceTree
;
static
const
std
::
string
kEarliestTimeBeginCompact
;
static
const
std
::
string
kLatestTimeEndCompact
;
};
...
...
@@ -253,8 +254,8 @@ struct TablePropertiesBase {
// Make these sst hidden
std
::
vector
<
Dependence
>
dependence
;
// Inheritance
chain
std
::
vector
<
uint64_t
>
inheritance_
chain
;
// Inheritance
tree
std
::
vector
<
uint64_t
>
inheritance_
tree
;
// convert this object to a human readable form
// @prop_delim: delimiter for each property.
...
...
table/block_based_table_builder.cc
浏览文件 @
33c36fcb
...
...
@@ -876,7 +876,8 @@ void BlockBasedTableBuilder::WriteRangeDelBlock(
Status
BlockBasedTableBuilder
::
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
SequenceNumber
>*
snapshots
)
{
const
std
::
vector
<
SequenceNumber
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
{
Rep
*
r
=
rep_
;
assert
(
r
->
status
.
ok
());
bool
empty_data_block
=
r
->
data_block
.
empty
();
...
...
@@ -889,11 +890,13 @@ Status BlockBasedTableBuilder::Finish(
r
->
props
.
max_read_amp
=
prop
->
max_read_amp
;
r
->
props
.
read_amp
=
prop
->
read_amp
;
r
->
props
.
dependence
=
prop
->
dependence
;
r
->
props
.
inheritance_chain
=
prop
->
inheritance_chain
;
}
if
(
snapshots
!=
nullptr
)
{
r
->
props
.
snapshots
=
*
snapshots
;
}
if
(
inheritance_tree
!=
nullptr
)
{
r
->
props
.
inheritance_tree
=
*
inheritance_tree
;
}
// To make sure properties block is able to keep the accurate size of index
// block, we will finish writing all index entries first.
...
...
table/block_based_table_builder.h
浏览文件 @
33c36fcb
...
...
@@ -63,7 +63,8 @@ class BlockBasedTableBuilder : public TableBuilder {
// constructor after this function returns.
// REQUIRES: Finish(), Abandon() have not been called
Status
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
uint64_t
>*
snapshots
)
override
;
const
std
::
vector
<
uint64_t
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
override
;
// Indicate that the contents of this builder should be abandoned. Stops
// using the file passed to the constructor after this function returns.
...
...
table/cuckoo_table_builder.cc
浏览文件 @
33c36fcb
...
...
@@ -256,7 +256,8 @@ Status CuckooTableBuilder::MakeHashTable(std::vector<CuckooBucket>* buckets) {
Status
CuckooTableBuilder
::
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
SequenceNumber
>*
snapshots
)
{
const
std
::
vector
<
SequenceNumber
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
{
assert
(
!
closed_
);
closed_
=
true
;
if
(
prop
!=
nullptr
)
{
...
...
@@ -264,11 +265,13 @@ Status CuckooTableBuilder::Finish(
properties_
.
max_read_amp
=
prop
->
max_read_amp
;
properties_
.
read_amp
=
prop
->
read_amp
;
properties_
.
dependence
=
prop
->
dependence
;
properties_
.
inheritance_chain
=
prop
->
inheritance_chain
;
}
if
(
snapshots
!=
nullptr
)
{
properties_
.
snapshots
=
*
snapshots
;
}
if
(
inheritance_tree
!=
nullptr
)
{
properties_
.
inheritance_tree
=
*
inheritance_tree
;
}
std
::
vector
<
CuckooBucket
>
buckets
;
Status
s
;
std
::
string
unused_bucket
;
...
...
table/cuckoo_table_builder.h
浏览文件 @
33c36fcb
...
...
@@ -44,7 +44,8 @@ class CuckooTableBuilder : public TableBuilder {
// constructor after this function returns.
// REQUIRES: Finish(), Abandon() have not been called
Status
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
SequenceNumber
>*
snapshots
)
override
;
const
std
::
vector
<
SequenceNumber
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
override
;
// Indicate that the contents of this builder should be abandoned. Stops
// using the file passed to the constructor after this function returns.
...
...
table/cuckoo_table_builder_test.cc
浏览文件 @
33c36fcb
...
...
@@ -179,7 +179,7 @@ TEST_F(CuckooBuilderTest, SuccessWithEmptyFile) {
GetSliceHash
,
0
/* column_family_id */
,
kDefaultColumnFamilyName
);
ASSERT_EQ
(
0UL
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
CheckFileContents
({},
{},
{},
""
,
2
,
2
,
false
);
}
...
...
@@ -225,7 +225,7 @@ TEST_F(CuckooBuilderTest, WriteSuccessNoCollisionFullKey) {
}
size_t
bucket_size
=
keys
[
0
].
size
()
+
values
[
0
].
size
();
ASSERT_EQ
(
expected_table_size
*
bucket_size
-
1
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
ASSERT_LE
(
expected_table_size
*
bucket_size
,
builder
.
FileSize
());
...
...
@@ -272,7 +272,7 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionFullKey) {
}
size_t
bucket_size
=
keys
[
0
].
size
()
+
values
[
0
].
size
();
ASSERT_EQ
(
expected_table_size
*
bucket_size
-
1
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
ASSERT_LE
(
expected_table_size
*
bucket_size
,
builder
.
FileSize
());
...
...
@@ -319,7 +319,7 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionAndCuckooBlock) {
}
size_t
bucket_size
=
keys
[
0
].
size
()
+
values
[
0
].
size
();
ASSERT_EQ
(
expected_table_size
*
bucket_size
-
1
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
ASSERT_LE
(
expected_table_size
*
bucket_size
,
builder
.
FileSize
());
...
...
@@ -367,7 +367,7 @@ TEST_F(CuckooBuilderTest, WithCollisionPathFullKey) {
}
size_t
bucket_size
=
keys
[
0
].
size
()
+
values
[
0
].
size
();
ASSERT_EQ
(
expected_table_size
*
bucket_size
-
1
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
ASSERT_LE
(
expected_table_size
*
bucket_size
,
builder
.
FileSize
());
...
...
@@ -412,7 +412,7 @@ TEST_F(CuckooBuilderTest, WithCollisionPathFullKeyAndCuckooBlock) {
}
size_t
bucket_size
=
keys
[
0
].
size
()
+
values
[
0
].
size
();
ASSERT_EQ
(
expected_table_size
*
bucket_size
-
1
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
ASSERT_LE
(
expected_table_size
*
bucket_size
,
builder
.
FileSize
());
...
...
@@ -454,7 +454,7 @@ TEST_F(CuckooBuilderTest, WriteSuccessNoCollisionUserKey) {
}
size_t
bucket_size
=
user_keys
[
0
].
size
()
+
values
[
0
].
size
();
ASSERT_EQ
(
expected_table_size
*
bucket_size
-
1
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
ASSERT_LE
(
expected_table_size
*
bucket_size
,
builder
.
FileSize
());
...
...
@@ -497,7 +497,7 @@ TEST_F(CuckooBuilderTest, WriteSuccessWithCollisionUserKey) {
}
size_t
bucket_size
=
user_keys
[
0
].
size
()
+
values
[
0
].
size
();
ASSERT_EQ
(
expected_table_size
*
bucket_size
-
1
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
ASSERT_LE
(
expected_table_size
*
bucket_size
,
builder
.
FileSize
());
...
...
@@ -539,7 +539,7 @@ TEST_F(CuckooBuilderTest, WithCollisionPathUserKey) {
}
size_t
bucket_size
=
user_keys
[
0
].
size
()
+
values
[
0
].
size
();
ASSERT_EQ
(
expected_table_size
*
bucket_size
-
1
,
builder
.
FileSize
());
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_OK
(
file_writer
->
Close
());
ASSERT_LE
(
expected_table_size
*
bucket_size
,
builder
.
FileSize
());
...
...
@@ -578,7 +578,7 @@ TEST_F(CuckooBuilderTest, FailWhenCollisionPathTooLong) {
LazyBuffer
(
"value"
)));
ASSERT_EQ
(
builder
.
NumEntries
(),
i
+
1
);
}
ASSERT_TRUE
(
builder
.
Finish
(
nullptr
,
nullptr
).
IsNotSupported
());
ASSERT_TRUE
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
).
IsNotSupported
());
ASSERT_OK
(
file_writer
->
Close
());
}
...
...
@@ -608,7 +608,7 @@ TEST_F(CuckooBuilderTest, FailWhenSameKeyInserted) {
builder
.
Add
(
Slice
(
GetInternalKey
(
user_key
,
true
)),
LazyBuffer
(
"value2"
)));
ASSERT_EQ
(
builder
.
NumEntries
(),
2u
);
ASSERT_TRUE
(
builder
.
Finish
(
nullptr
,
nullptr
).
IsNotSupported
());
ASSERT_TRUE
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
).
IsNotSupported
());
ASSERT_OK
(
file_writer
->
Close
());
}
}
// namespace TERARKDB_NAMESPACE
...
...
table/cuckoo_table_reader_test.cc
浏览文件 @
33c36fcb
...
...
@@ -108,7 +108,7 @@ class CuckooReaderTest : public testing::Test {
ASSERT_OK
(
builder
.
Add
(
Slice
(
keys
[
key_idx
]),
LazyBuffer
(
values
[
key_idx
])));
ASSERT_EQ
(
builder
.
NumEntries
(),
key_idx
+
1
);
}
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_EQ
(
num_items
,
builder
.
NumEntries
());
file_size
=
builder
.
FileSize
();
ASSERT_OK
(
file_writer
->
Close
());
...
...
@@ -433,7 +433,7 @@ void WriteFile(const std::vector<std::string>& keys, const uint64_t num,
builder
.
Add
(
Slice
(
keys
[
key_idx
]),
LazyBuffer
(
&
keys
[
key_idx
][
0
],
4
)));
ASSERT_EQ
(
builder
.
NumEntries
(),
key_idx
+
1
);
}
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
));
ASSERT_OK
(
builder
.
Finish
(
nullptr
,
nullptr
,
nullptr
));
ASSERT_EQ
(
num
,
builder
.
NumEntries
());
ASSERT_OK
(
file_writer
->
Close
());
...
...
table/meta_blocks.cc
浏览文件 @
33c36fcb
...
...
@@ -124,8 +124,8 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) {
}
Add
(
TablePropertiesNames
::
kDependenceEntryCount
,
val
);
}
if
(
!
props
.
inheritance_
chain
.
empty
())
{
Add
(
TablePropertiesNames
::
kInheritance
Chain
,
props
.
inheritance_chain
);
if
(
!
props
.
inheritance_
tree
.
empty
())
{
Add
(
TablePropertiesNames
::
kInheritance
Tree
,
props
.
inheritance_tree
);
}
if
(
!
props
.
filter_policy_name
.
empty
())
{
...
...
@@ -415,7 +415,14 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
new_table_properties
->
dependence
[
i
].
entry_count
=
val
[
i
];
}
}
else
if
(
key
==
TablePropertiesNames
::
kInheritanceChain
)
{
GetUint64Vector
(
key
,
&
raw_val
,
new_table_properties
->
inheritance_chain
);
std
::
vector
<
uint64_t
>
val
;
GetUint64Vector
(
key
,
&
raw_val
,
val
);
auto
&
tree
=
new_table_properties
->
inheritance_tree
;
tree
.
reserve
(
val
.
size
()
*
2
);
tree
.
insert
(
tree
.
end
(),
val
.
rbegin
(),
val
.
rend
());
tree
.
insert
(
tree
.
end
(),
val
.
begin
(),
val
.
end
());
}
else
if
(
key
==
TablePropertiesNames
::
kInheritanceTree
)
{
GetUint64Vector
(
key
,
&
raw_val
,
new_table_properties
->
inheritance_tree
);
}
else
{
// handle user-collected properties
new_table_properties
->
user_collected_properties
.
insert
(
...
...
table/mock_table.h
浏览文件 @
33c36fcb
...
...
@@ -156,8 +156,8 @@ class MockTableBuilder : public TableBuilder {
return
AddToTable
(
key
,
value
,
file_data_
.
tombstone
);
}
Status
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
uint64_t
>*
)
override
{
Status
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
uint64_t
>*
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
override
{
prop_
.
num_entries
=
file_data_
.
table
.
size
();
prop_
.
raw_key_size
=
file_data_
.
table
.
size
();
prop_
.
raw_value_size
=
file_data_
.
table
.
size
();
...
...
@@ -166,7 +166,9 @@ class MockTableBuilder : public TableBuilder {
prop_
.
max_read_amp
=
prop
->
max_read_amp
;
prop_
.
read_amp
=
prop
->
read_amp
;
prop_
.
dependence
=
prop
->
dependence
;
prop_
.
inheritance_chain
=
prop
->
inheritance_chain
;
}
if
(
inheritance_tree
!=
nullptr
)
{
prop_
.
inheritance_tree
=
*
inheritance_tree
;
}
file_data_
.
prop
=
std
::
make_shared
<
const
TableProperties
>
(
prop_
);
MutexLock
lock_guard
(
&
file_system_
->
mutex
);
...
...
table/plain_table_builder.cc
浏览文件 @
33c36fcb
...
...
@@ -177,8 +177,10 @@ Status PlainTableBuilder::Add(const Slice& key, const LazyBuffer& lazy_value) {
return
Status
::
OK
();
}
Status
PlainTableBuilder
::
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
SequenceNumber
>*
snapshots
)
{
Status
PlainTableBuilder
::
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
SequenceNumber
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
{
assert
(
!
closed_
);
closed_
=
true
;
...
...
@@ -187,11 +189,13 @@ Status PlainTableBuilder::Finish(const TablePropertyCache* prop,
properties_
.
max_read_amp
=
prop
->
max_read_amp
;
properties_
.
read_amp
=
prop
->
read_amp
;
properties_
.
dependence
=
prop
->
dependence
;
properties_
.
inheritance_chain
=
prop
->
inheritance_chain
;
}
if
(
snapshots
!=
nullptr
)
{
properties_
.
snapshots
=
*
snapshots
;
}
if
(
inheritance_tree
!=
nullptr
)
{
properties_
.
inheritance_tree
=
*
inheritance_tree
;
}
properties_
.
data_size
=
offset_
;
...
...
table/plain_table_builder.h
浏览文件 @
33c36fcb
...
...
@@ -53,7 +53,8 @@ class PlainTableBuilder : public TableBuilder {
// constructor after this function returns.
// REQUIRES: Finish(), Abandon() have not been called
Status
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
SequenceNumber
>*
snapshots
)
override
;
const
std
::
vector
<
SequenceNumber
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
override
;
// Indicate that the contents of this builder should be abandoned. Stops
// using the file passed to the constructor after this function returns.
...
...
table/table_builder.h
浏览文件 @
33c36fcb
...
...
@@ -166,8 +166,9 @@ class TableBuilder {
// Finish building the table.
// REQUIRES: Finish(), Abandon() have not been called
virtual
Status
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
uint64_t
>*
snapshots
)
=
0
;
virtual
Status
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
uint64_t
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
=
nullptr
)
=
0
;
// Indicate that the contents of this builder should be abandoned.
// If the caller is not going to call Finish(), it must call Abandon()
...
...
table/table_properties.cc
浏览文件 @
33c36fcb
...
...
@@ -229,6 +229,8 @@ const std::string TablePropertiesNames::kDependenceEntryCount =
"rocksdb.sst.dependence.entry-count"
;
const
std
::
string
TablePropertiesNames
::
kInheritanceChain
=
"rocksdb.sst.inheritance-chain"
;
const
std
::
string
TablePropertiesNames
::
kInheritanceTree
=
"rocksdb.sst.inheritance-tree"
;
const
std
::
string
TablePropertiesNames
::
kEarliestTimeBeginCompact
=
"rocksdb.compact.earliest-time-begin"
;
const
std
::
string
TablePropertiesNames
::
kLatestTimeEndCompact
=
...
...
table/terark_zip_table_builder.cc
浏览文件 @
33c36fcb
...
...
@@ -720,7 +720,8 @@ Status TerarkZipTableBuilder::EmptyTableFinish() {
Status
TerarkZipTableBuilder
::
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
SequenceNumber
>*
snapshots
)
try
{
const
std
::
vector
<
SequenceNumber
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
try
{
assert
(
!
closed_
);
closed_
=
true
;
...
...
@@ -728,11 +729,13 @@ Status TerarkZipTableBuilder::Finish(
properties_
.
purpose
=
prop
->
purpose
;
properties_
.
read_amp
=
prop
->
read_amp
;
properties_
.
dependence
=
prop
->
dependence
;
properties_
.
inheritance_chain
=
prop
->
inheritance_chain
;
}
if
(
snapshots
!=
nullptr
)
{
properties_
.
snapshots
=
*
snapshots
;
}
if
(
inheritance_tree
!=
nullptr
)
{
properties_
.
inheritance_tree
=
*
inheritance_tree
;
}
if
(
!
r00_
)
{
return
EmptyTableFinish
();
...
...
table/terark_zip_table_builder.h
浏览文件 @
33c36fcb
...
...
@@ -73,7 +73,8 @@ class TerarkZipTableBuilder : public TableBuilder, boost::noncopyable {
Status
Add
(
const
Slice
&
key
,
const
LazyBuffer
&
value
)
override
;
Status
AddTombstone
(
const
Slice
&
key
,
const
LazyBuffer
&
value
)
override
;
Status
Finish
(
const
TablePropertyCache
*
prop
,
const
std
::
vector
<
SequenceNumber
>*
snapshots
)
override
;
const
std
::
vector
<
SequenceNumber
>*
snapshots
,
const
std
::
vector
<
uint64_t
>*
inheritance_tree
)
override
;
Status
AbortFinish
(
const
std
::
exception
&
ex
);
void
Abandon
()
override
;
uint64_t
NumEntries
()
const
override
{
return
properties_
.
num_entries
;
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录