Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
indiff7643
Terarkdb
提交
92291666
T
Terarkdb
项目概览
indiff7643
/
Terarkdb
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
Terarkdb
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
92291666
编写于
8月 03, 2018
作者:
奏
奏之章
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[WIP] rename hidden to depend , fix VersionBuilder
上级
390cb870
变更
14
隐藏空白更改
内联
并排
Showing
14 changed file
with
141 addition
and
111 deletion
+141
-111
db/compaction_job.cc
db/compaction_job.cc
+5
-5
db/db_impl_compaction_flush.cc
db/db_impl_compaction_flush.cc
+2
-2
db/db_impl_experimental.cc
db/db_impl_experimental.cc
+1
-1
db/db_impl_open.cc
db/db_impl_open.cc
+1
-1
db/flush_job.cc
db/flush_job.cc
+1
-1
db/internal_stats.h
db/internal_stats.h
+1
-1
db/repair.cc
db/repair.cc
+8
-7
db/table_properties_collector.cc
db/table_properties_collector.cc
+13
-13
db/table_properties_collector.h
db/table_properties_collector.h
+11
-11
db/version_builder.cc
db/version_builder.cc
+83
-54
db/version_edit.cc
db/version_edit.cc
+4
-4
db/version_edit.h
db/version_edit.h
+4
-4
db/version_set.cc
db/version_set.cc
+5
-5
include/rocksdb/table_properties.h
include/rocksdb/table_properties.h
+2
-2
未找到文件。
db/compaction_job.cc
浏览文件 @
92291666
...
...
@@ -1206,17 +1206,17 @@ void CompactionJob::ProcessLinkCompaction(SubcompactionState* sub_compact) {
std
::
unique_ptr
<
InternalIterator
>
input
(
versions_
->
MakeInputIterator
(
sub_compact
->
compaction
,
range_del_agg
.
get
(),
env_optiosn_for_read_
));
std
::
vector
<
uint64_t
>
sst_
takeover
;
std
::
vector
<
uint64_t
>
sst_
depend
;
for
(
auto
level_files
:
*
sub_compact
->
compaction
->
inputs
())
{
for
(
auto
file
:
level_files
.
files
)
{
sst_
takeover
.
emplace_back
(
file
->
fd
.
GetNumber
());
sst_
depend
.
emplace_back
(
file
->
fd
.
GetNumber
());
}
}
std
::
sort
(
sst_
takeover
.
begin
(),
sst_takeover
.
end
());
std
::
sort
(
sst_
depend
.
begin
(),
sst_depend
.
end
());
std
::
vector
<
std
::
unique_ptr
<
IntTblPropCollectorFactory
>>
collectors
;
collectors
.
emplace_back
(
new
SSTLinkPropertiesCollectorFactory
((
uint8_t
)
kLinkSst
,
&
sst_
takeover
));
new
SSTLinkPropertiesCollectorFactory
((
uint8_t
)
kLinkSst
,
&
sst_
depend
));
auto
status
=
OpenCompactionOutputFile
(
sub_compact
,
&
collectors
);
if
(
!
status
.
ok
())
{
...
...
@@ -1257,7 +1257,7 @@ void CompactionJob::ProcessLinkCompaction(SubcompactionState* sub_compact) {
status
=
FinishCompactionOutputFile
(
status
,
sub_compact
,
range_del_agg
.
get
(),
&
range_del_out_stats
);
sub_compact
->
current_output
()
->
meta
.
sst_variety
=
kLinkSst
;
sub_compact
->
current_output
()
->
meta
.
sst_
takeover
=
std
::
move
(
sst_takeover
);
sub_compact
->
current_output
()
->
meta
.
sst_
depend
=
std
::
move
(
sst_depend
);
sub_compact
->
actual_start
=
sub_compact
->
current_output
()
->
meta
.
smallest
;
sub_compact
->
actual_end
=
sub_compact
->
current_output
()
->
meta
.
largest
;
if
(
!
status
.
ok
())
{
...
...
db/db_impl_compaction_flush.cc
浏览文件 @
92291666
...
...
@@ -887,7 +887,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
f
->
fd
.
GetFileSize
(),
f
->
smallest
,
f
->
largest
,
f
->
fd
.
smallest_seqno
,
f
->
fd
.
largest_seqno
,
f
->
marked_for_compaction
,
f
->
sst_variety
,
f
->
sst_
takeover
);
f
->
sst_
depend
);
}
ROCKS_LOG_DEBUG
(
immutable_db_options_
.
info_log
,
"[%s] Apply version edit:
\n
%s"
,
cfd
->
GetName
().
c_str
(),
...
...
@@ -1807,7 +1807,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
f
->
fd
.
GetPathId
(),
f
->
fd
.
GetFileSize
(),
f
->
smallest
,
f
->
largest
,
f
->
fd
.
smallest_seqno
,
f
->
fd
.
largest_seqno
,
f
->
marked_for_compaction
,
f
->
sst_variety
,
f
->
sst_
takeover
);
f
->
sst_variety
,
f
->
sst_
depend
);
ROCKS_LOG_BUFFER
(
log_buffer
,
...
...
db/db_impl_experimental.cc
浏览文件 @
92291666
...
...
@@ -133,7 +133,7 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
f
->
fd
.
GetFileSize
(),
f
->
smallest
,
f
->
largest
,
f
->
fd
.
smallest_seqno
,
f
->
fd
.
largest_seqno
,
f
->
marked_for_compaction
,
f
->
sst_variety
,
f
->
sst_
takeover
);
f
->
sst_
depend
);
}
status
=
versions_
->
LogAndApply
(
cfd
,
*
cfd
->
GetLatestMutableCFOptions
(),
...
...
db/db_impl_open.cc
浏览文件 @
92291666
...
...
@@ -971,7 +971,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
meta
.
fd
.
GetFileSize
(),
meta
.
smallest
,
meta
.
largest
,
meta
.
fd
.
smallest_seqno
,
meta
.
fd
.
largest_seqno
,
meta
.
marked_for_compaction
,
meta
.
sst_variety
,
meta
.
sst_
takeover
);
meta
.
sst_
depend
);
}
InternalStats
::
CompactionStats
stats
(
CompactionReason
::
kFlush
,
1
);
...
...
db/flush_job.cc
浏览文件 @
92291666
...
...
@@ -391,7 +391,7 @@ Status FlushJob::WriteLevel0Table() {
meta_
.
fd
.
GetFileSize
(),
meta_
.
smallest
,
meta_
.
largest
,
meta_
.
fd
.
smallest_seqno
,
meta_
.
fd
.
largest_seqno
,
meta_
.
marked_for_compaction
,
meta_
.
sst_variety
,
meta_
.
sst_
takeover
);
meta_
.
sst_
depend
);
}
// Note that here we treat flush as level 0 compaction in internal stats
...
...
db/internal_stats.h
浏览文件 @
92291666
...
...
@@ -124,7 +124,7 @@ class InternalStats {
cf_stats_value_
{},
cf_stats_count_
{},
comp_stats_
(
num_levels
),
file_read_latency_
(
num_levels
),
file_read_latency_
(
num_levels
+
1
),
bg_error_count_
(
0
),
number_levels_
(
num_levels
),
env_
(
env
),
...
...
db/repair.cc
浏览文件 @
92291666
...
...
@@ -535,8 +535,8 @@ class Repairer {
}
delete
iter
;
t
->
meta
.
sst_variety
=
GetSst
Gene
(
props
->
user_collected_properties
);
t
->
meta
.
sst_
takeover
=
GetSstTakeover
(
props
->
user_collected_properties
);
t
->
meta
.
sst_variety
=
GetSst
Variety
(
props
->
user_collected_properties
);
t
->
meta
.
sst_
depend
=
GetSstDepend
(
props
->
user_collected_properties
);
ROCKS_LOG_INFO
(
db_options_
.
info_log
,
"Table #%"
PRIu64
": %d entries %s"
,
t
->
meta
.
fd
.
GetNumber
(),
counter
,
...
...
@@ -567,24 +567,25 @@ class Repairer {
edit
.
SetNextFile
(
next_file_number_
);
edit
.
SetColumnFamily
(
cfd
->
GetID
());
std
::
set
<
uint64_t
>
hidden_id
;
std
::
set
<
uint64_t
>
depend_set
;
for
(
const
auto
*
table
:
cf_id_and_tables
.
second
)
{
if
(
table
->
meta
.
sst_variety
!=
0
)
{
auto
&
sst_
takeover
=
table
->
meta
.
sst_takeover
;
hidden_id
.
insert
(
sst_takeover
.
begin
(),
sst_takeover
.
end
());
auto
&
sst_
depend
=
table
->
meta
.
sst_depend
;
depend_set
.
insert
(
sst_depend
.
begin
(),
sst_depend
.
end
());
}
}
// TODO(opt): separate out into multiple levels
for
(
const
auto
*
table
:
cf_id_and_tables
.
second
)
{
int
level
=
0
;
if
(
hidden_id
.
count
(
table
->
meta
.
fd
.
GetNumber
())
>
0
)
{
if
(
depend_set
.
count
(
table
->
meta
.
fd
.
GetNumber
())
>
0
)
{
// This sst should insert into depend level
level
=
default_cf_iopts_
.
num_levels
;
}
edit
.
AddFile
(
level
,
table
->
meta
.
fd
.
GetNumber
(),
table
->
meta
.
fd
.
GetPathId
(),
table
->
meta
.
fd
.
GetFileSize
(),
table
->
meta
.
smallest
,
table
->
meta
.
largest
,
table
->
min_sequence
,
table
->
max_sequence
,
table
->
meta
.
marked_for_compaction
,
table
->
meta
.
sst_variety
,
table
->
meta
.
sst_
takeover
);
table
->
meta
.
sst_variety
,
table
->
meta
.
sst_
depend
);
}
assert
(
next_file_number_
>
0
);
vset_
.
MarkFileNumberUsed
(
next_file_number_
-
1
);
...
...
db/table_properties_collector.cc
浏览文件 @
92291666
...
...
@@ -58,12 +58,12 @@ InternalKeyPropertiesCollector::GetReadableProperties() const {
{
"kMergeOperands"
,
ToString
(
merge_operands_
)}};
}
Status
Sst
Gene
PropertiesCollector
::
Finish
(
Status
Sst
Variety
PropertiesCollector
::
Finish
(
UserCollectedProperties
*
properties
)
{
assert
(
properties
);
assert
(
properties
->
find
(
SSTVarietiesTablePropertiesNames
::
kSstVariety
)
==
properties
->
end
());
assert
(
properties
->
find
(
SSTVarietiesTablePropertiesNames
::
kSst
Takeover
)
==
assert
(
properties
->
find
(
SSTVarietiesTablePropertiesNames
::
kSst
Depend
)
==
properties
->
end
());
auto
sst_variety_value
=
std
::
string
((
const
char
*
)
&
sst_variety_
,
1
);
...
...
@@ -71,31 +71,31 @@ Status SstGenePropertiesCollector::Finish(
{
SSTVarietiesTablePropertiesNames
::
kSstVariety
,
sst_variety_value
});
std
::
string
sst_takeover_value
;
PutVarint64
(
&
sst_takeover_value
,
sst_
takeover
_
->
size
());
for
(
auto
sst_id
:
*
sst_
takeover
_
)
{
PutVarint64
(
&
sst_takeover_value
,
sst_
depend
_
->
size
());
for
(
auto
sst_id
:
*
sst_
depend
_
)
{
PutVarint64
(
&
sst_takeover_value
,
sst_id
);
}
properties
->
insert
(
{
SSTVarietiesTablePropertiesNames
::
kSst
Takeover
,
sst_takeover_value
});
{
SSTVarietiesTablePropertiesNames
::
kSst
Depend
,
sst_takeover_value
});
return
Status
::
OK
();
}
UserCollectedProperties
Sst
Gene
PropertiesCollector
::
GetReadableProperties
()
const
{
Sst
Variety
PropertiesCollector
::
GetReadableProperties
()
const
{
std
::
string
sst_takeover_value
;
if
(
sst_
takeover
_
->
empty
())
{
if
(
sst_
depend
_
->
empty
())
{
sst_takeover_value
+=
"[]"
;
}
else
{
sst_takeover_value
+=
'['
;
for
(
auto
sst_id
:
*
sst_
takeover
_
)
{
for
(
auto
sst_id
:
*
sst_
depend
_
)
{
sst_takeover_value
+=
ToString
(
sst_id
);
sst_takeover_value
+=
','
;
}
sst_takeover_value
.
back
()
=
']'
;
}
return
{{
"kSstVariety"
,
ToString
((
int
)
sst_variety_
)},
{
"kSst
Takeover
"
,
sst_takeover_value
}};
{
"kSst
Depend
"
,
sst_takeover_value
}};
}
namespace
{
...
...
@@ -145,7 +145,7 @@ const std::string InternalKeyTablePropertiesNames::kMergeOperands =
"rocksdb.merge.operands"
;
const
std
::
string
SSTVarietiesTablePropertiesNames
::
kSstVariety
=
"rocksdb.sst.gene"
;
const
std
::
string
SSTVarietiesTablePropertiesNames
::
kSst
Takeover
=
const
std
::
string
SSTVarietiesTablePropertiesNames
::
kSst
Depend
=
"rocksdb.sst.takeover"
;
uint64_t
GetDeletedKeys
(
...
...
@@ -161,7 +161,7 @@ uint64_t GetMergeOperands(const UserCollectedProperties& props,
props
,
InternalKeyTablePropertiesNames
::
kMergeOperands
,
property_present
);
}
uint8_t
GetSst
Gene
(
uint8_t
GetSst
Variety
(
const
UserCollectedProperties
&
props
)
{
auto
pos
=
props
.
find
(
SSTVarietiesTablePropertiesNames
::
kSstVariety
);
if
(
pos
==
props
.
end
())
{
...
...
@@ -171,10 +171,10 @@ uint8_t GetSstGene(
return
raw
[
0
];
}
std
::
vector
<
uint64_t
>
GetSst
Takeover
(
std
::
vector
<
uint64_t
>
GetSst
Depend
(
const
UserCollectedProperties
&
props
)
{
std
::
vector
<
uint64_t
>
result
;
auto
pos
=
props
.
find
(
SSTVarietiesTablePropertiesNames
::
kSst
Takeover
);
auto
pos
=
props
.
find
(
SSTVarietiesTablePropertiesNames
::
kSst
Depend
);
if
(
pos
==
props
.
end
())
{
return
result
;
}
...
...
db/table_properties_collector.h
浏览文件 @
92291666
...
...
@@ -21,7 +21,7 @@ struct InternalKeyTablePropertiesNames {
struct
SSTVarietiesTablePropertiesNames
{
static
const
std
::
string
kSstVariety
;
static
const
std
::
string
kSst
Takeover
;
static
const
std
::
string
kSst
Depend
;
};
// Base class for internal table properties collector.
...
...
@@ -89,12 +89,12 @@ class InternalKeyPropertiesCollectorFactory
// Write link or map info
// Used for repair. E.g missing manifest
class
Sst
Gene
PropertiesCollector
final
:
public
IntTblPropCollector
{
class
Sst
Variety
PropertiesCollector
final
:
public
IntTblPropCollector
{
public:
Sst
Gene
PropertiesCollector
(
uint8_t
_sst_variety
,
std
::
vector
<
uint64_t
>*
_sst_
takeover
)
Sst
Variety
PropertiesCollector
(
uint8_t
_sst_variety
,
std
::
vector
<
uint64_t
>*
_sst_
depend
)
:
sst_variety_
(
_sst_variety
),
sst_
takeover_
(
_sst_takeover
)
{}
sst_
depend_
(
_sst_depend
)
{}
virtual
Status
InternalAdd
(
const
Slice
&
key
,
const
Slice
&
value
,
uint64_t
file_size
)
override
{
...
...
@@ -104,27 +104,27 @@ class SstGenePropertiesCollector final : public IntTblPropCollector {
virtual
Status
Finish
(
UserCollectedProperties
*
properties
)
override
;
virtual
const
char
*
Name
()
const
override
{
return
"SST
Gene
PropertiesCollector"
;
return
"SST
Variety
PropertiesCollector"
;
}
UserCollectedProperties
GetReadableProperties
()
const
override
;
private:
uint8_t
sst_variety_
;
std
::
vector
<
uint64_t
>*
sst_
takeover
_
;
std
::
vector
<
uint64_t
>*
sst_
depend
_
;
};
class
SSTLinkPropertiesCollectorFactory
final
:
public
IntTblPropCollectorFactory
{
public:
SSTLinkPropertiesCollectorFactory
(
uint8_t
_sst_variety
,
std
::
vector
<
uint64_t
>*
_sst_
takeover
)
uint8_t
_sst_variety
,
std
::
vector
<
uint64_t
>*
_sst_
depend
)
:
sst_variety_
(
_sst_variety
),
sst_
takeover_
(
_sst_takeover
)
{}
sst_
depend_
(
_sst_depend
)
{}
virtual
IntTblPropCollector
*
CreateIntTblPropCollector
(
uint32_t
/*column_family_id*/
)
override
{
return
new
Sst
GenePropertiesCollector
(
sst_variety_
,
sst_takeover
_
);
return
new
Sst
VarietyPropertiesCollector
(
sst_variety_
,
sst_depend
_
);
}
virtual
const
char
*
Name
()
const
override
{
...
...
@@ -132,7 +132,7 @@ class SSTLinkPropertiesCollectorFactory final
}
private:
uint8_t
sst_variety_
;
std
::
vector
<
uint64_t
>*
sst_
takeover
_
;
std
::
vector
<
uint64_t
>*
sst_
depend
_
;
};
// When rocksdb creates a new table, it will encode all "user keys" into
...
...
db/version_builder.cc
浏览文件 @
92291666
...
...
@@ -56,23 +56,23 @@ bool BySmallestKey(FileMetaData* a, FileMetaData* b,
return
(
a
->
fd
.
GetNumber
()
<
b
->
fd
.
GetNumber
());
}
void
AttachSstTakeover
(
FileMetaData
*
f
,
std
::
map
<
uint64_t
,
size_t
>&
hidden_id
)
{
for
(
auto
sst_id
:
f
->
sst_
takeover
)
{
auto
ib
=
hidden_id
.
emplace
(
sst_id
,
1
);
void
LoadSstDepend
(
FileMetaData
*
f
,
std
::
map
<
uint64_t
,
size_t
>&
depend_map
)
{
for
(
auto
sst_id
:
f
->
sst_
depend
)
{
auto
ib
=
depend_map
.
emplace
(
sst_id
,
1
);
if
(
!
ib
.
second
)
{
++
ib
.
first
->
second
;
}
}
}
void
DetachSstTakever
(
FileMetaData
*
f
,
std
::
map
<
uint64_t
,
size_t
>&
hidden_id
)
{
for
(
auto
sst_id
:
f
->
sst_
takeover
)
{
auto
find
=
hidden_id
.
find
(
sst_id
);
assert
(
find
!=
hidden_id
.
end
());
void
UnloadSstDepend
(
FileMetaData
*
f
,
std
::
map
<
uint64_t
,
size_t
>&
depend_map
)
{
for
(
auto
sst_id
:
f
->
sst_
depend
)
{
auto
find
=
depend_map
.
find
(
sst_id
);
assert
(
find
!=
depend_map
.
end
());
if
(
--
find
->
second
==
0
)
{
hidden_id
.
erase
(
find
);
depend_map
.
erase
(
find
);
}
}
}
...
...
@@ -113,8 +113,8 @@ class VersionBuilder::Rep {
VersionStorageInfo
*
base_vstorage_
;
int
num_levels_
;
LevelState
*
levels_
;
std
::
map
<
uint64_t
,
size_t
>
hidden_id
_
;
std
::
vector
<
FileMetaData
*>
hidden
_files_
;
std
::
map
<
uint64_t
,
size_t
>
depend_map
_
;
std
::
vector
<
FileMetaData
*>
depend
_files_
;
// Store states of levels larger than num_levels_. We do this instead of
// storing them in levels_ to avoid regression in case there are no files
// on invalid levels. The version is not consistent if in the end the files
...
...
@@ -149,7 +149,7 @@ class VersionBuilder::Rep {
UnrefFile
(
pair
.
second
);
}
}
for
(
auto
f
:
hidden
_files_
)
{
for
(
auto
f
:
depend
_files_
)
{
UnrefFile
(
f
);
}
delete
[]
levels_
;
...
...
@@ -291,6 +291,9 @@ class VersionBuilder::Rep {
void
Apply
(
VersionEdit
*
edit
)
{
CheckConsistency
(
base_vstorage_
);
size_t
depend_file_count
=
depend_files_
.
size
();
bool
depend_changed
=
false
;
// Delete files
const
VersionEdit
::
DeletedFileSet
&
del
=
edit
->
GetDeletedFiles
();
for
(
const
auto
&
del_file
:
del
)
{
...
...
@@ -303,8 +306,11 @@ class VersionBuilder::Rep {
auto
exising
=
levels_
[
level
].
added_files
.
find
(
number
);
if
(
exising
!=
levels_
[
level
].
added_files
.
end
())
{
auto
f
=
exising
->
second
;
DetachSstTakever
(
f
,
hidden_id_
);
hidden_files_
.
push_back
(
f
);
if
(
f
->
sst_variety
!=
0
)
{
UnloadSstDepend
(
f
,
depend_map_
);
depend_changed
=
true
;
}
depend_files_
.
push_back
(
f
);
levels_
[
level
].
added_files
.
erase
(
exising
);
}
}
else
{
...
...
@@ -317,6 +323,26 @@ class VersionBuilder::Rep {
}
}
}
// Remove recursive depend
if
(
depend_changed
&&
depend_file_count
>
0
)
{
do
{
depend_changed
=
false
;
// depend files <- mid -> deleted files
size_t
mid
=
std
::
partition
(
depend_files_
.
begin
(),
depend_files_
.
begin
()
+
depend_file_count
,
[
&
](
FileMetaData
*
f
)
{
return
depend_map_
.
count
(
f
->
fd
.
GetNumber
())
>
0
;
})
-
depend_files_
.
begin
();
while
(
depend_file_count
>
mid
)
{
auto
f
=
depend_files_
[
--
depend_file_count
];
if
(
f
->
sst_variety
!=
0
)
{
UnloadSstDepend
(
f
,
depend_map_
);
depend_changed
=
true
;
}
}
}
while
(
depend_changed
);
}
// Add new files
for
(
const
auto
&
new_file
:
edit
->
GetNewFiles
())
{
...
...
@@ -327,10 +353,13 @@ class VersionBuilder::Rep {
assert
(
levels_
[
level
].
added_files
.
find
(
f
->
fd
.
GetNumber
())
==
levels_
[
level
].
added_files
.
end
());
assert
(
hidden_id
_
.
count
(
f
->
fd
.
GetNumber
())
==
0
);
assert
(
depend_map
_
.
count
(
f
->
fd
.
GetNumber
())
==
0
);
levels_
[
level
].
deleted_files
.
erase
(
f
->
fd
.
GetNumber
());
levels_
[
level
].
added_files
[
f
->
fd
.
GetNumber
()]
=
f
;
AttachSstTakeover
(
f
,
hidden_id_
);
if
(
f
->
sst_variety
!=
0
)
{
LoadSstDepend
(
f
,
depend_map_
);
depend_changed
=
true
;
}
}
else
{
uint64_t
number
=
new_file
.
second
.
fd
.
GetNumber
();
if
(
invalid_levels_
[
level
].
count
(
number
)
==
0
)
{
...
...
@@ -342,38 +371,35 @@ class VersionBuilder::Rep {
}
}
// TODO(zouzhizhang): Fix loading bug
// Shrink hidden_files_
if
(
hidden_id_
.
empty
())
{
for
(
auto
f
:
hidden_files_
)
{
UnrefFile
(
f
);
}
hidden_files_
.
clear
();
}
else
if
(
!
hidden_files_
.
empty
())
{
size_t
size
=
hidden_files_
.
size
();
bool
hidden_changed
;
// Reclaim depend files
if
(
depend_map_
.
empty
())
{
depend_file_count
=
0
;
}
else
if
(
depend_changed
&&
depend_files_
.
size
()
>
depend_file_count
)
{
do
{
hidden
_changed
=
false
;
//
hidden
files <- mid -> deleted files
depend
_changed
=
false
;
//
depend
files <- mid -> deleted files
size_t
mid
=
std
::
partition
(
hidden_files_
.
begin
()
,
hidden_files_
.
begin
()
+
size
,
depend_files_
.
begin
()
+
depend_file_count
,
depend_files_
.
end
()
,
[
&
](
FileMetaData
*
f
)
{
return
hidden_id_
.
count
(
f
->
fd
.
GetNumber
())
>
0
;
})
-
hidden_files_
.
begin
();
while
(
size
>
mid
)
{
--
size
;
auto
f
=
hidden_files_
[
size
];
return
depend_map_
.
count
(
f
->
fd
.
GetNumber
())
>
0
;
})
-
depend_files_
.
begin
();
for
(;
depend_file_count
<
mid
;
++
depend_file_count
)
{
auto
f
=
depend_files_
[
depend_file_count
];
if
(
f
->
sst_variety
!=
0
)
{
hidden_changed
=
true
;
DetachSstTakever
(
f
,
hidden_id_
)
;
LoadSstDepend
(
f
,
depend_map_
)
;
depend_changed
=
true
;
}
UnrefFile
(
f
);
}
}
while
(
hidden_changed
);
hidden_files_
.
resize
(
size
);
}
while
(
depend_changed
);
}
// Actual remove files
for
(
size_t
i
=
depend_file_count
;
i
<
depend_files_
.
size
();
++
i
)
{
UnrefFile
(
depend_files_
[
i
]);
}
depend_files_
.
resize
(
depend_file_count
);
}
// Save the current state in *v.
...
...
@@ -381,14 +407,14 @@ class VersionBuilder::Rep {
CheckConsistency
(
base_vstorage_
);
CheckConsistency
(
vstorage
);
// Apply new
hidden
files
for
(
auto
f
:
hidden
_files_
)
{
// Apply new
depend
files
for
(
auto
f
:
depend
_files_
)
{
vstorage
->
AddFile
(
num_levels_
,
f
,
info_log_
);
UnrefFile
(
f
);
}
hidden
_files_
.
clear
();
depend
_files_
.
clear
();
// Deep copy
// Deep copy
base depend files to deleted files
auto
deleted_files
=
base_vstorage_
->
LevelFiles
(
num_levels_
);
for
(
int
level
=
0
;
level
<
num_levels_
;
level
++
)
{
...
...
@@ -418,7 +444,7 @@ class VersionBuilder::Rep {
if
(
levels_
[
level
].
deleted_files
.
count
(
f
->
fd
.
GetNumber
())
>
0
)
{
deleted_files
.
push_back
(
f
);
}
else
{
AttachSstTakeover
(
*
base_iter
,
hidden_id
_
);
LoadSstDepend
(
*
base_iter
,
depend_map
_
);
vstorage
->
AddFile
(
level
,
f
,
info_log_
);
}
};
...
...
@@ -446,21 +472,21 @@ class VersionBuilder::Rep {
maybe_add_file
(
*
base_iter
);
}
}
// Reclaim
hidden
files form deleted files
// Reclaim
depend
files form deleted files
size_t
pos
=
0
;
while
(
!
hidden_id
_
.
empty
())
{
//
hidden
files <- mid -> deleted files
while
(
!
depend_map
_
.
empty
())
{
//
depend
files <- mid -> deleted files
size_t
mid
=
std
::
partition
(
deleted_files
.
begin
()
+
pos
,
deleted_files
.
end
(),
[
&
](
FileMetaData
*
f
)
{
return
hidden_id
_
.
count
(
f
->
fd
.
GetNumber
())
>
0
;
return
depend_map
_
.
count
(
f
->
fd
.
GetNumber
())
>
0
;
})
-
deleted_files
.
begin
();
hidden_id
_
.
clear
();
depend_map
_
.
clear
();
for
(;
pos
<
mid
;
++
pos
)
{
auto
f
=
deleted_files
[
pos
];
// a
hidden
file !
// a
depend
file !
vstorage
->
AddFile
(
num_levels_
,
f
,
info_log_
);
AttachSstTakeover
(
f
,
hidden_id
_
);
LoadSstDepend
(
f
,
depend_map
_
);
}
}
// Handle actual deleted files
...
...
@@ -486,6 +512,9 @@ class VersionBuilder::Rep {
files_meta
.
emplace_back
(
file_meta
,
level
);
}
}
for
(
auto
f
:
depend_files_
)
{
files_meta
.
emplace_back
(
f
,
num_levels_
);
}
std
::
atomic
<
size_t
>
next_file_meta_idx
(
0
);
std
::
function
<
void
()
>
load_handlers_func
=
[
&
]()
{
...
...
db/version_edit.cc
浏览文件 @
92291666
...
...
@@ -186,8 +186,8 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
PutVarint32
(
dst
,
CustomTag
::
kSstVariety
);
std
::
string
encode_buffer
;
encode_buffer
+=
(
char
)
f
.
sst_variety
;
PutVarint64
(
&
encode_buffer
,
f
.
sst_
takeover
.
size
());
for
(
auto
sst_id
:
f
.
sst_
takeover
)
{
PutVarint64
(
&
encode_buffer
,
f
.
sst_
depend
.
size
());
for
(
auto
sst_id
:
f
.
sst_
depend
)
{
PutVarint64
(
&
encode_buffer
,
sst_id
);
}
PutLengthPrefixedSlice
(
dst
,
Slice
(
encode_buffer
));
...
...
@@ -304,13 +304,13 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
if
(
!
GetVarint64
(
&
field
,
&
size
))
{
return
error_msg
;
}
f
.
sst_
takeover
.
reserve
(
size
);
f
.
sst_
depend
.
reserve
(
size
);
for
(
size_t
i
=
0
;
i
<
size
;
++
i
)
{
uint64_t
sst_id
;
if
(
!
GetVarint64
(
&
field
,
&
sst_id
))
{
return
error_msg
;
}
f
.
sst_
takeover
.
emplace_back
(
sst_id
);
f
.
sst_
depend
.
emplace_back
(
sst_id
);
}
}
while
(
false
);
break
;
...
...
db/version_edit.h
浏览文件 @
92291666
...
...
@@ -121,8 +121,8 @@ struct FileMetaData {
bool
marked_for_compaction
;
// True if client asked us nicely to compact this
// file.
uint8_t
sst_variety
;
// Zero for plain sst
std
::
vector
<
uint64_t
>
sst_
takeover
;
// Make these sst hidden
uint8_t
sst_variety
;
// Zero for plain sst
std
::
vector
<
uint64_t
>
sst_
depend
;
// Make these sst hidden
FileMetaData
()
:
table_reader_handle
(
nullptr
),
...
...
@@ -247,7 +247,7 @@ class VersionEdit {
const
InternalKey
&
largest
,
const
SequenceNumber
&
smallest_seqno
,
const
SequenceNumber
&
largest_seqno
,
bool
marked_for_compaction
,
uint8_t
sst_variety
,
const
std
::
vector
<
uint64_t
>&
sst_
takeover
)
{
const
std
::
vector
<
uint64_t
>&
sst_
depend
)
{
assert
(
smallest_seqno
<=
largest_seqno
);
FileMetaData
f
;
f
.
fd
=
FileDescriptor
(
file
,
file_path_id
,
file_size
,
smallest_seqno
,
...
...
@@ -258,7 +258,7 @@ class VersionEdit {
f
.
fd
.
largest_seqno
=
largest_seqno
;
f
.
marked_for_compaction
=
marked_for_compaction
;
f
.
sst_variety
=
sst_variety
;
f
.
sst_
takeover
=
sst_takeover
;
f
.
sst_
depend
=
sst_depend
;
new_files_
.
emplace_back
(
level
,
std
::
move
(
f
));
}
...
...
db/version_set.cc
浏览文件 @
92291666
...
...
@@ -352,7 +352,7 @@ Version::~Version() {
next_
->
prev_
=
prev_
;
// Drop references to files
// here use level less or EQUAL num_levels for clean
hidden
files
// here use level less or EQUAL num_levels for clean
depend
files
for
(
int
level
=
0
;
level
<=
storage_info_
.
num_levels_
;
level
++
)
{
for
(
size_t
i
=
0
;
i
<
storage_info_
.
files_
[
level
].
size
();
i
++
)
{
FileMetaData
*
f
=
storage_info_
.
files_
[
level
][
i
];
...
...
@@ -1373,7 +1373,7 @@ void Version::UpdateAccumulatedStats(bool update_stats) {
// compensated_file_size, making lower-level to higher-level compaction
// will be triggered, which creates higher-level files whose num_deletions
// will be updated here.
// here use level less or EQUAL num_levels for include
hidden
files
// here use level less or EQUAL num_levels for include
depend
files
for
(
int
level
=
0
;
level
<=
storage_info_
.
num_levels_
&&
init_count
<
kMaxInitCount
;
++
level
)
{
...
...
@@ -1399,7 +1399,7 @@ void Version::UpdateAccumulatedStats(bool update_stats) {
// In case all sampled-files contain only deletion entries, then we
// load the table-property of a file in higher-level to initialize
// that value.
// here use level start from num_levels for include
hidden
files
// here use level start from num_levels for include
depend
files
for
(
int
level
=
storage_info_
.
num_levels_
;
storage_info_
.
accumulated_raw_value_size_
==
0
&&
level
>=
0
;
--
level
)
{
...
...
@@ -3699,7 +3699,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
new_files_list
[
new_levels
-
1
]
=
vstorage
->
LevelFiles
(
first_nonempty_level
);
}
// Keep the
hidden
layer sst files
// Keep the
depend
layer sst files
new_files_list
[
new_levels
]
=
vstorage
->
LevelFiles
(
current_levels
);
delete
[]
vstorage
->
files_
;
...
...
@@ -3983,7 +3983,7 @@ Status VersionSet::WriteSnapshot(log::Writer* log) {
f
->
fd
.
GetFileSize
(),
f
->
smallest
,
f
->
largest
,
f
->
fd
.
smallest_seqno
,
f
->
fd
.
largest_seqno
,
f
->
marked_for_compaction
,
f
->
sst_variety
,
f
->
sst_
takeover
);
f
->
sst_
depend
);
}
}
edit
.
SetLogNumber
(
cfd
->
GetLogNumber
());
...
...
include/rocksdb/table_properties.h
浏览文件 @
92291666
...
...
@@ -217,8 +217,8 @@ struct TableProperties {
extern
uint64_t
GetDeletedKeys
(
const
UserCollectedProperties
&
props
);
extern
uint64_t
GetMergeOperands
(
const
UserCollectedProperties
&
props
,
bool
*
property_present
);
extern
uint8_t
GetSst
Gene
(
const
UserCollectedProperties
&
props
);
extern
std
::
vector
<
uint64_t
>
GetSst
Takeover
(
extern
uint8_t
GetSst
Variety
(
const
UserCollectedProperties
&
props
);
extern
std
::
vector
<
uint64_t
>
GetSst
Depend
(
const
UserCollectedProperties
&
props
);
}
// namespace rocksdb
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录