Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Metz
oceanbase
提交
5d3d5c7c
O
oceanbase
项目概览
Metz
/
oceanbase
与 Fork 源项目一致
Fork自
oceanbase / oceanbase
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
O
oceanbase
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
5d3d5c7c
编写于
8月 12, 2022
作者:
O
obdev
提交者:
wangzelin.wzl
8月 12, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[CP] Reduce tmp file block cache memory usage.
上级
c6ce3292
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
95 addition
and
56 deletion
+95
-56
src/sql/executor/ob_interm_result_pool.h
src/sql/executor/ob_interm_result_pool.h
+1
-1
src/storage/blocksstable/ob_tmp_file_cache.cpp
src/storage/blocksstable/ob_tmp_file_cache.cpp
+3
-3
src/storage/blocksstable/ob_tmp_file_store.cpp
src/storage/blocksstable/ob_tmp_file_store.cpp
+64
-48
src/storage/blocksstable/ob_tmp_file_store.h
src/storage/blocksstable/ob_tmp_file_store.h
+6
-3
unittest/storage/blocksstable/test_tmp_file.cpp
unittest/storage/blocksstable/test_tmp_file.cpp
+21
-1
未找到文件。
src/sql/executor/ob_interm_result_pool.h
浏览文件 @
5d3d5c7c
...
...
@@ -32,7 +32,7 @@ public:
static
const
int64_t
SCANNER_CAPACITY
=
256L
<<
10
;
// 256K
// adapt to block size in tmp file.
static
const
int64_t
SCANNER_MEM_LIMIT
=
(
8
<<
20
)
-
(
32
<<
10
);
// 8MB - 32
K
static
const
int64_t
SCANNER_MEM_LIMIT
=
(
8
<<
20
)
-
(
128
<<
10
);
// 8MB - 128
K
ObIntermResultPool
();
virtual
~
ObIntermResultPool
();
...
...
src/storage/blocksstable/ob_tmp_file_cache.cpp
浏览文件 @
5d3d5c7c
...
...
@@ -150,12 +150,12 @@ int ObTmpPageCache::prefetch(
callback
.
offset_
=
info
.
offset_
;
callback
.
buf_size_
=
info
.
size_
;
callback
.
allocator_
=
&
allocator_
;
void
*
buf
=
allocator_
.
alloc
(
sizeof
(
common
::
ObSEArray
<
ObTmpPageIOInfo
,
255
>
));
void
*
buf
=
allocator_
.
alloc
(
sizeof
(
common
::
ObSEArray
<
ObTmpPageIOInfo
,
ObTmpFilePageBuddy
::
MAX_PAGE_NUMS
>
));
if
(
NULL
==
buf
)
{
ret
=
OB_ALLOCATE_MEMORY_FAILED
;
STORAGE_LOG
(
WARN
,
"fail to alloc a buf"
,
K
(
ret
),
K
(
info
));
}
else
{
callback
.
page_io_infos_
=
new
(
buf
)
common
::
ObSEArray
<
ObTmpPageIOInfo
,
255
>
();
callback
.
page_io_infos_
=
new
(
buf
)
common
::
ObSEArray
<
ObTmpPageIOInfo
,
ObTmpFilePageBuddy
::
MAX_PAGE_NUMS
>
();
callback
.
page_io_infos_
->
assign
(
page_io_infos
);
if
(
OB_FAIL
(
read_io
(
info
,
callback
,
mb_handle
)))
{
if
(
mb_handle
.
get_io_handle
().
is_empty
())
{
...
...
@@ -798,7 +798,7 @@ int ObTmpTenantMemBlockManager::free_extent(const int64_t free_page_nums, const
STORAGE_LOG
(
WARN
,
"ObTmpBlockCache has not been inited"
,
K
(
ret
));
}
else
if
(
free_page_nums
<
0
||
free_page_nums
>
mblk_page_nums_
||
NULL
==
t_mblk
)
{
ret
=
OB_INVALID_ARGUMENT
;
STORAGE_LOG
(
WARN
,
"invalid argument"
,
K
(
ret
),
K
(
free_page_nums
),
K
(
*
t_mblk
));
STORAGE_LOG
(
WARN
,
"invalid argument"
,
K
(
ret
),
K
(
free_page_nums
),
K
PC
(
t_mblk
));
}
else
if
(
OB_FAIL
(
refresh_dir_to_blk_map
(
t_mblk
->
get_dir_id
(),
t_mblk
)))
{
STORAGE_LOG
(
WARN
,
"fail to refresh dir_to_blk_map"
,
K
(
ret
),
K
(
*
t_mblk
));
}
else
{
...
...
src/storage/blocksstable/ob_tmp_file_store.cpp
浏览文件 @
5d3d5c7c
...
...
@@ -39,13 +39,13 @@ int ObTmpFilePageBuddy::init(common::ObIAllocator& allocator)
STORAGE_LOG
(
WARN
,
"ObTmpFilePageBuddy has not been inited"
,
K
(
ret
));
}
else
{
allocator_
=
&
allocator
;
buf_
=
reinterpret_cast
<
ObTmpFileArea
*>
(
allocator_
->
alloc
(
sizeof
(
ObTmpFileArea
)
*
std
::
pow
(
2
,
ObTmpFilePageBuddy
::
MAX_ORDER
)
-
1
));
buf_
=
reinterpret_cast
<
ObTmpFileArea
*>
(
allocator_
->
alloc
(
sizeof
(
ObTmpFileArea
)
*
(
std
::
pow
(
2
,
MAX_ORDER
)
-
std
::
pow
(
2
,
MIN_ORDER
))
));
if
(
NULL
==
buf_
)
{
ret
=
OB_ALLOCATE_MEMORY_FAILED
;
STORAGE_LOG
(
WARN
,
"fail to alloc a buf"
,
K
(
ret
));
}
else
{
max_cont_page_nums_
=
std
::
pow
(
2
,
ObTmpFilePageBuddy
::
MAX_ORDER
-
1
);
max_cont_page_nums_
=
std
::
pow
(
2
,
MAX_ORDER
-
1
);
/**
* page buddy free_list for a new block:
* -------------- --------- --------- --------- --------- --------- --------- --------- -------
...
...
@@ -55,8 +55,11 @@ int ObTmpFilePageBuddy::init(common::ObIAllocator& allocator)
* -------------- --------- --------- --------- --------- --------- --------- --------- -------
*/
int64_t
nums
=
max_cont_page_nums_
;
for
(
int32_t
i
=
MAX_ORDER
-
1
;
i
>=
0
;
--
i
)
{
char
*
buf
=
reinterpret_cast
<
char
*>
(
&
(
buf_
[
start_id
]));
for
(
int32_t
i
=
MIN_ORDER
-
1
;
i
>=
0
;
--
i
)
{
free_area_
[
i
]
=
NULL
;
}
for
(
int32_t
i
=
MAX_ORDER
-
1
;
i
>=
MIN_ORDER
;
--
i
)
{
char
*
buf
=
reinterpret_cast
<
char
*>
(
&
(
buf_
[
start_id
]));
free_area_
[
i
]
=
new
(
buf
)
ObTmpFileArea
(
start_id
,
nums
);
start_id
+=
nums
;
nums
/=
2
;
...
...
@@ -89,13 +92,14 @@ int ObTmpFilePageBuddy::alloc_all_pages()
ret
=
OB_NOT_INIT
;
STORAGE_LOG
(
WARN
,
"ObTmpFilePageBuddy has not been inited"
,
K
(
ret
));
}
else
if
(
is_empty
())
{
for
(
int32_t
i
=
0
;
i
<
ObTmpFilePageBuddy
::
MAX_ORDER
;
++
i
)
{
for
(
int32_t
i
=
0
;
i
<
MAX_ORDER
;
++
i
)
{
while
(
NULL
!=
free_area_
[
i
])
{
tmp
=
free_area_
[
i
];
free_area_
[
i
]
=
tmp
->
next_
;
tmp
->~
ObTmpFileArea
();
}
}
max_cont_page_nums_
=
0
;
}
else
{
ret
=
OB_ERR_UNEXPECTED
;
...
...
@@ -118,7 +122,7 @@ int ObTmpFilePageBuddy::alloc(const int32_t page_nums,
}
else
{
int32_t
index
=
std
::
ceil
(
std
::
log
(
page_nums
)
/
std
::
log
(
2
));
bool
is_alloced
=
false
;
for
(
int32_t
i
=
index
;
i
<
ObTmpFilePageBuddy
::
MAX_ORDER
&&
!
is_alloced
;
++
i
)
{
for
(
int32_t
i
=
index
;
i
<
MAX_ORDER
&&
!
is_alloced
;
++
i
)
{
if
(
NULL
!=
free_area_
[
i
])
{
int64_t
num
=
i
-
index
;
ObTmpFileArea
*
tmp
=
free_area_
[
i
];
...
...
@@ -185,7 +189,13 @@ void ObTmpFilePageBuddy::free_align(const int32_t start_page_id, const int32_t p
bool
ObTmpFilePageBuddy
::
is_empty
()
const
{
bool
is_empty
=
true
;
for
(
int32_t
i
=
0
;
i
<
ObTmpFilePageBuddy
::
MAX_ORDER
;
++
i
)
{
for
(
int32_t
i
=
0
;
i
<
MIN_ORDER
&&
is_empty
;
++
i
)
{
if
(
NULL
!=
free_area_
[
i
])
{
is_empty
=
false
;
break
;
}
}
for
(
int32_t
i
=
MIN_ORDER
;
i
<
MAX_ORDER
&&
is_empty
;
++
i
)
{
if
(
NULL
==
free_area_
[
i
])
{
is_empty
=
false
;
break
;
...
...
@@ -200,7 +210,7 @@ int64_t ObTmpFilePageBuddy::to_string(char* buf, const int64_t buf_len) const
bool
first
=
true
;
ObTmpFileArea
*
area
=
NULL
;
common
::
databuff_printf
(
buf
,
buf_len
,
pos
,
"{"
);
for
(
int32_t
i
=
0
;
i
<
ObTmpFilePageBuddy
::
MAX_ORDER
;
++
i
)
{
for
(
int32_t
i
=
0
;
i
<
MAX_ORDER
;
++
i
)
{
area
=
free_area_
[
i
];
if
(
NULL
!=
area
)
{
common
::
databuff_print_kv
(
buf
,
buf_len
,
pos
,
"page_nums"
,
static_cast
<
int64_t
>
(
std
::
pow
(
2
,
i
)));
...
...
@@ -231,45 +241,51 @@ int64_t ObTmpFilePageBuddy::to_string(char* buf, const int64_t buf_len) const
void
ObTmpFilePageBuddy
::
free
(
const
int32_t
start_page_id
,
const
int32_t
page_nums
)
{
int32_t
start_id
=
start_page_id
;
int32_t
nums
=
page_nums
;
int32_t
length
=
0
;
while
(
nums
>
0
)
{
/**
* PURPOSE: align free area into power of 2.
*
* The probable value of alloc_start_id:
* page nums start page id
* 128 0 ---------------- 128 ------------------- 256
* 64 0 ------ 64 ------ 128 ------- 192 ------- 256
* 32 0 - 32 - 64 - 96 - 128 - 160 - 192 - 224 - 256
* ... ...
* So, the maximum number of consecutive pages from a start_page_id is the
* gcd(greatest common divisor) between it and 512, except 0. The maximum
* consecutive page nums of 0 is 256.
*
* The layout of free area in alocated area :
* |<---------------alloc_page_nums--------------->|
* <---- |<--free_page_nums-->|
* |==========================|====================|
* alloc_start free_page_id alloc_end
*
* So, free_end always equal to alloc_end.
*
* Based on two observations above, the algorithm is designed as follows:
*/
length
=
2
;
while
(
0
==
start_id
%
length
&&
length
<=
nums
)
{
length
*=
2
;
}
length
=
std
::
min
(
length
/
2
,
nums
);
if
(
OB_UNLIKELY
(
start_page_id
+
page_nums
>=
std
::
pow
(
2
,
MAX_ORDER
)))
{
STORAGE_LOG
(
ERROR
,
"page id more than max numbers in block"
,
K
(
start_page_id
),
K
(
page_nums
));
ob_abort
();
}
else
{
int32_t
start_id
=
start_page_id
;
int32_t
nums
=
page_nums
;
int32_t
length
=
0
;
while
(
nums
>
0
)
{
/**
* PURPOSE: align free area into power of 2.
*
* The probable value of alloc_start_id:
* page nums start page id
* 128 0 ---------------- 128 ------------------- 256
* 64 0 ------ 64 ------ 128 ------- 192 ------- 256
* 32 0 - 32 - 64 - 96 - 128 - 160 - 192 - 224 - 256
* ... ...
* So, the maximum number of consecutive pages from a start_page_id is the
* gcd(greatest common divisor) between it and 512, except 0. The maximum
* consecutive page nums of 0 is 256.
*
* The layout of free area in alocated area :
* |<---------------alloc_page_nums--------------->|
* <---- |<--free_page_nums-->|
* |==========================|====================|
* alloc_start free_page_id alloc_end
*
* So, free_end always equal to alloc_end.
*
* Based on two observations above, the algorithm is designed as follows:
*/
length
=
2
;
while
(
0
==
start_id
%
length
&&
length
<=
nums
)
{
length
*=
2
;
}
length
=
std
::
min
(
length
/
2
,
nums
);
char
*
buf
=
reinterpret_cast
<
char
*>
(
&
(
buf_
[
start_id
]));
ObTmpFileArea
*
area
=
new
(
buf
)
ObTmpFileArea
(
start_id
,
length
);
free_align
(
area
->
start_page_id_
,
area
->
page_nums_
,
area
);
start_id
+=
length
;
nums
-=
length
;
char
*
buf
=
reinterpret_cast
<
char
*>
(
&
(
buf_
[
start_id
]));
ObTmpFileArea
*
area
=
new
(
buf
)
ObTmpFileArea
(
start_id
,
length
);
free_align
(
area
->
start_page_id_
,
area
->
page_nums_
,
area
);
start_id
+=
length
;
nums
-=
length
;
}
}
}
ObTmpFileArea
*
ObTmpFilePageBuddy
::
find_buddy
(
const
int32_t
page_nums
,
const
int32_t
start_page_id
)
...
...
@@ -704,7 +720,7 @@ int64_t ObTmpTenantMacroBlockManager::get_next_blk_id()
}
ObTmpTenantFileStore
::
ObTmpTenantFileStore
()
:
tmp_block_manager_
(),
tmp_mem_block_manager_
(),
page_cache_
(
NULL
),
file_handle_
(),
allocator_
(),
is_inited_
(
false
)
:
tmp_block_manager_
(),
page_cache_
(
NULL
),
tmp_mem_block_manager_
(
),
file_handle_
(),
allocator_
(),
is_inited_
(
false
)
{}
ObTmpTenantFileStore
::~
ObTmpTenantFileStore
()
...
...
@@ -1036,7 +1052,7 @@ int ObTmpTenantFileStore::read_page(ObTmpMacroBlock* block, ObTmpBlockIOInfo& io
int64_t
remain_size
=
io_info
.
size_
;
int64_t
size
=
std
::
min
(
ObTmpMacroBlock
::
get_default_page_size
()
-
offset
,
remain_size
);
int32_t
page_nums
=
0
;
common
::
ObSEArray
<
ObTmpPageIOInfo
,
255
>
page_io_infos
;
common
::
ObSEArray
<
ObTmpPageIOInfo
,
ObTmpFilePageBuddy
::
MAX_PAGE_NUMS
>
page_io_infos
;
do
{
ObTmpPageCacheKey
key
(
io_info
.
block_id_
,
page_start_id
,
io_info
.
tenant_id_
);
ObTmpPageValueHandle
p_handle
;
...
...
src/storage/blocksstable/ob_tmp_file_store.h
浏览文件 @
5d3d5c7c
...
...
@@ -67,6 +67,8 @@ public:
bool
is_empty
()
const
;
int64_t
to_string
(
char
*
buf
,
const
int64_t
buf_len
)
const
;
static
const
int64_t
MAX_PAGE_NUMS
=
252
;
// 2^MAX_ORDER - 2^MIN_ORDER
private:
void
free_align
(
const
int32_t
start_page_id
,
const
int32_t
page_nums
,
ObTmpFileArea
*&
area
);
ObTmpFileArea
*
find_buddy
(
const
int32_t
page_nums
,
const
int32_t
start_page_id
);
...
...
@@ -74,6 +76,7 @@ private:
{
return
std
::
pow
(
2
,
ObTmpFilePageBuddy
::
MAX_ORDER
)
-
1
;
}
static
const
int
MIN_ORDER
=
2
;
static
const
int
MAX_ORDER
=
8
;
ObTmpFileArea
*
free_area_
[
ObTmpFilePageBuddy
::
MAX_ORDER
];
int64_t
max_cont_page_nums_
;
...
...
@@ -295,8 +298,8 @@ private:
static
constexpr
double
DEFAULT_PAGE_IO_MERGE_RATIO
=
0.5
;
ObTmpTenantMacroBlockManager
tmp_block_manager_
;
ObTmpTenantMemBlockManager
tmp_mem_block_manager_
;
ObTmpPageCache
*
page_cache_
;
ObTmpTenantMemBlockManager
tmp_mem_block_manager_
;
ObStorageFileHandle
file_handle_
;
common
::
ObConcurrentFIFOAllocator
allocator_
;
common
::
SpinRWLock
lock_
;
...
...
@@ -325,11 +328,11 @@ public:
OB_INLINE
int64_t
get_mblk_page_nums
()
const
{
return
OB_FILE_SYSTEM
.
get_macro_block_size
()
/
ObTmpMacroBlock
::
get_default_page_size
()
-
1
;
return
OB_FILE_SYSTEM
.
get_macro_block_size
()
/
ObTmpMacroBlock
::
get_default_page_size
()
-
4
;
}
OB_INLINE
int64_t
get_block_size
()
const
{
return
OB_FILE_SYSTEM
.
get_macro_block_size
()
-
ObTmpMacroBlock
::
get_default_page_size
();
return
OB_FILE_SYSTEM
.
get_macro_block_size
()
-
4
*
ObTmpMacroBlock
::
get_default_page_size
();
}
private:
...
...
unittest/storage/blocksstable/test_tmp_file.cpp
浏览文件 @
5d3d5c7c
...
...
@@ -1602,12 +1602,15 @@ TEST_F(TestTmpFile, test_page_buddy)
int32_t
page_nums
=
64
;
int32_t
alloced_page_nums
=
64
;
int32_t
start_page_id
=
-
1
;
ASSERT_EQ
(
true
,
page_buddy_1
.
is_empty
());
ret
=
page_buddy_1
.
alloc
(
page_nums
,
start_page_id
,
alloced_page_nums
);
ASSERT_EQ
(
OB_SUCCESS
,
ret
);
ASSERT_EQ
(
false
,
page_buddy_1
.
is_empty
());
int32_t
start_page_id_2
=
-
1
;
ret
=
page_buddy_1
.
alloc
(
page_nums
,
start_page_id_2
,
alloced_page_nums
);
ASSERT_EQ
(
OB_SUCCESS
,
ret
);
ASSERT_EQ
(
false
,
page_buddy_1
.
is_empty
());
page_buddy_1
.
free
(
start_page_id
+
63
,
page_nums
-
63
);
page_buddy_1
.
free
(
start_page_id_2
+
1
,
page_nums
-
1
);
...
...
@@ -1620,11 +1623,14 @@ TEST_F(TestTmpFile, test_page_buddy)
ObTmpFilePageBuddy
page_buddy_2
;
ret
=
page_buddy_2
.
init
(
allocator
);
ASSERT_EQ
(
OB_SUCCESS
,
ret
);
ASSERT_EQ
(
true
,
page_buddy_2
.
is_empty
());
start_page_id
=
0
;
ret
=
page_buddy_2
.
alloc_all_pages
();
ASSERT_EQ
(
OB_SUCCESS
,
ret
);
ASSERT_EQ
(
false
,
page_buddy_2
.
is_empty
());
int32_t
free_nums
=
511
-
129
;
int32_t
free_nums
=
252
-
129
;
page_buddy_2
.
free
(
start_page_id
+
129
,
free_nums
);
free_nums
=
127
;
page_buddy_2
.
free
(
start_page_id
+
2
,
free_nums
);
...
...
@@ -1645,6 +1651,20 @@ TEST_F(TestTmpFile, test_page_buddy)
ASSERT_EQ
(
true
,
page_buddy_3
.
is_empty
());
STORAGE_LOG
(
INFO
,
"page buddy"
,
K
(
page_buddy_3
));
}
ObTmpFilePageBuddy
page_buddy_4
;
ret
=
page_buddy_4
.
init
(
allocator
);
ASSERT_EQ
(
OB_SUCCESS
,
ret
);
ASSERT_EQ
(
true
,
page_buddy_4
.
is_empty
());
page_nums
=
2
;
alloced_page_nums
=
-
1
;
start_page_id
=
-
1
;
ASSERT_EQ
(
true
,
page_buddy_4
.
is_empty
());
ret
=
page_buddy_4
.
alloc
(
page_nums
,
start_page_id
,
alloced_page_nums
);
ASSERT_EQ
(
OB_SUCCESS
,
ret
);
ASSERT_EQ
(
alloced_page_nums
,
page_nums
);
ASSERT_EQ
(
false
,
page_buddy_4
.
is_empty
());
}
}
// end namespace unittest
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录