Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
知世而放
oceanbase
提交
f0bad67c
O
oceanbase
项目概览
知世而放
/
oceanbase
与 Fork 源项目一致
Fork自
oceanbase / oceanbase
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
O
oceanbase
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
f0bad67c
编写于
7月 19, 2021
作者:
O
obdev
提交者:
wangzelin.wzl
7月 19, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Patch fixs of estimating row count to 3.1_opensource_release
上级
af7df013
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
116 addition
and
59 deletion
+116
-59
src/storage/ob_sstable_estimator.cpp
src/storage/ob_sstable_estimator.cpp
+62
-57
src/storage/ob_sstable_estimator.h
src/storage/ob_sstable_estimator.h
+1
-2
unittest/storage/test_micro_block_row_scanner_with_special_uncom_row.cpp
...e/test_micro_block_row_scanner_with_special_uncom_row.cpp
+53
-0
未找到文件。
src/storage/ob_sstable_estimator.cpp
浏览文件 @
f0bad67c
...
...
@@ -18,6 +18,7 @@ namespace oceanbase {
using
namespace
common
;
using
namespace
blocksstable
;
namespace
storage
{
const
int64_t
MACRO_BLOCK_COUNT_THRESHOLD
=
1024
;
ObSSTableEstimateContext
::
ObSSTableEstimateContext
()
:
sstable_
(
NULL
),
rowkeys_
(
NULL
)
{}
...
...
@@ -118,50 +119,34 @@ ObStoreRowSingleScanEstimator::ObStoreRowSingleScanEstimator()
ObStoreRowSingleScanEstimator
::~
ObStoreRowSingleScanEstimator
()
{}
int
ObStoreRowSingleScanEstimator
::
set_context
(
ObSSTableEstimateContext
&
context
)
{
int
ret
=
OB_SUCCESS
;
if
(
OB_FAIL
(
ObISSTableEstimator
::
set_context
(
context
)))
{
STORAGE_LOG
(
WARN
,
"failed to set context"
,
K
(
ret
));
}
else
if
(
OB_FAIL
(
context_
.
sstable_
->
find_macros
(
*
context_
.
range_
,
context_
.
macro_blocks_
)))
{
STORAGE_LOG
(
WARN
,
"fail to find macros"
,
K
(
ret
));
}
return
ret
;
}
int
ObStoreRowSingleScanEstimator
::
open
()
int
ObStoreRowSingleScanEstimator
::
check_bf
(
ObMacroBlockCtx
&
macro_block_ctx
)
{
int
ret
=
OB_SUCCESS
;
int
tmp_ret
=
OB_SUCCESS
;
if
(
1
==
context_
.
macro_blocks_
.
count
())
{
// check bloom filter to identify if the scan is empty
const
MacroBlockId
&
macro_block_id
=
context_
.
macro_blocks_
.
at
(
0
).
get_macro_block_id
();
ObStoreRowkey
rowkey
;
ObStorageFileHandle
file_handle
;
ObStorageFile
*
file
=
nullptr
;
if
(
OB_FAIL
(
file_handle
.
assign
(
context_
.
sstable_
->
get_storage_file_handle
())))
{
STORAGE_LOG
(
WARN
,
"fail to get file handle"
,
K
(
ret
),
K
(
context_
.
sstable_
->
get_storage_file_handle
()));
}
else
if
(
OB_ISNULL
(
file
=
file_handle
.
get_storage_file
()))
{
ret
=
OB_ERR_UNEXPECTED
;
STORAGE_LOG
(
WARN
,
"fail to get pg file"
,
K
(
ret
),
K
(
file_handle
));
}
else
if
(
OB_FAIL
(
get_common_rowkey
(
context_
.
range_
->
get_range
(),
rowkey
)))
{
STORAGE_LOG
(
WARN
,
"failed to get common rowkey"
,
K
(
ret
),
K
(
context_
.
range_
));
}
else
if
(
rowkey
.
get_obj_cnt
()
>
0
)
{
// allow check contain fail, should not overwrite ret
bool
is_contain
=
false
;
if
(
OB_SUCCESS
!=
(
tmp_ret
=
context_
.
cache_context_
.
bf_cache_
->
may_contain
(
context_
.
sstable_
->
get_table_id
(),
macro_block_id
,
file
->
get_file_id
(),
rowkey
,
is_contain
)))
{
if
(
OB_ENTRY_NOT_EXIST
!=
tmp_ret
)
{
STORAGE_LOG
(
WARN
,
"failed to check may contain"
,
K
(
tmp_ret
),
K_
(
context
),
K
(
macro_block_id
),
K
(
rowkey
));
}
}
else
if
(
!
is_contain
)
{
is_empty_scan_
=
true
;
// check bloom filter to identify if the scan is empty
const
MacroBlockId
&
macro_block_id
=
macro_block_ctx
.
get_macro_block_id
();
ObStoreRowkey
rowkey
;
ObStorageFileHandle
file_handle
;
ObStorageFile
*
file
=
nullptr
;
if
(
OB_FAIL
(
file_handle
.
assign
(
context_
.
sstable_
->
get_storage_file_handle
())))
{
STORAGE_LOG
(
WARN
,
"fail to get file handle"
,
K
(
ret
),
K
(
context_
.
sstable_
->
get_storage_file_handle
()));
}
else
if
(
OB_ISNULL
(
file
=
file_handle
.
get_storage_file
()))
{
ret
=
OB_ERR_UNEXPECTED
;
STORAGE_LOG
(
WARN
,
"fail to get pg file"
,
K
(
ret
),
K
(
file_handle
));
}
else
if
(
OB_FAIL
(
get_common_rowkey
(
context_
.
range_
->
get_range
(),
rowkey
)))
{
STORAGE_LOG
(
WARN
,
"failed to get common rowkey"
,
K
(
ret
),
K
(
context_
.
range_
));
}
else
if
(
rowkey
.
get_obj_cnt
()
>
0
)
{
// allow check contain fail, should not overwrite ret
bool
is_contain
=
false
;
if
(
OB_SUCCESS
!=
(
tmp_ret
=
context_
.
cache_context_
.
bf_cache_
->
may_contain
(
context_
.
sstable_
->
get_table_id
(),
macro_block_id
,
file
->
get_file_id
(),
rowkey
,
is_contain
)))
{
if
(
OB_ENTRY_NOT_EXIST
!=
tmp_ret
)
{
STORAGE_LOG
(
WARN
,
"failed to check may contain"
,
K
(
tmp_ret
),
K_
(
context
),
K
(
macro_block_id
),
K
(
rowkey
));
}
}
else
if
(
!
is_contain
)
{
is_empty_scan_
=
true
;
}
}
else
if
(
0
==
context_
.
macro_blocks_
.
count
())
{
is_empty_scan_
=
true
;
}
return
ret
;
...
...
@@ -177,31 +162,51 @@ void ObStoreRowSingleScanEstimator::reset()
int
ObStoreRowSingleScanEstimator
::
estimate_row_count
(
ObPartitionEst
&
part_est
)
{
int
ret
=
OB_SUCCESS
;
int64_t
total_macro_block_count
=
0
;
ObMacroBlockIterator
macro_iter
;
if
(
OB_FAIL
(
open
()))
{
STORAGE_LOG
(
WARN
,
"failed to open single scan estimator"
,
K
(
ret
));
if
(
!
context_
.
range_
->
get_range
().
is_valid
())
{
}
else
if
(
context_
.
range_
->
get_range
().
is_whole_range
())
{
part_est
.
logical_row_count_
=
context_
.
sstable_
->
get_meta
().
row_count_
;
part_est
.
physical_row_count_
=
part_est
.
logical_row_count_
;
}
else
if
(
OB_FAIL
(
macro_iter
.
open
(
*
context_
.
sstable_
,
*
context_
.
range_
)))
{
STORAGE_LOG
(
WARN
,
"fail to open macro iter,"
,
K
(
ret
));
}
else
if
(
OB_FAIL
(
macro_iter
.
get_macro_block_count
(
total_macro_block_count
)))
{
STORAGE_LOG
(
WARN
,
"fail to get macro block count,"
,
K
(
ret
));
}
else
if
(
0
==
total_macro_block_count
)
{
}
else
if
(
total_macro_block_count
>
MACRO_BLOCK_COUNT_THRESHOLD
)
{
// there are too many block, do estimate in fast way
part_est
.
logical_row_count_
=
(
double
)
total_macro_block_count
/
context_
.
sstable_
->
get_meta
().
macro_block_count_
*
context_
.
sstable_
->
get_meta
().
row_count_
;
part_est
.
physical_row_count_
=
part_est
.
logical_row_count_
;
}
else
{
// do calculate cost metrics by macro block scan.
int64_t
total_macro_block_count
=
context_
.
macro_blocks_
.
count
();
if
(
context_
.
range_
->
get_range
().
is_whole_range
())
{
part_est
.
logical_row_count_
+=
context_
.
sstable_
->
get_meta
().
row_count_
;
}
else
if
(
!
is_empty_scan_
)
{
MacroBlockId
macro_block_id
;
for
(
int64_t
i
=
0
;
OB_SUCC
(
ret
)
&&
i
<
total_macro_block_count
;
++
i
)
{
const
bool
is_start_block
=
(
0
==
i
);
const
bool
is_last_block
=
(
total_macro_block_count
-
1
==
i
);
const
ObMacroBlockCtx
&
macro_block_ctx
=
context_
.
macro_blocks_
.
at
(
i
);
if
(
OB_FAIL
(
estimate_macro_row_count
(
macro_block_ctx
,
is_start_block
,
is_last_block
,
part_est
)))
{
// do estimate by macro block scan
ObMacroBlockCtx
macro_block_ctx
;
int64_t
idx
=
0
;
while
(
OB_SUCC
(
ret
))
{
bool
is_start_block
=
0
==
idx
;
bool
is_last_block
=
total_macro_block_count
-
1
==
idx
;
if
(
OB_FAIL
(
macro_iter
.
get_next_macro_block
(
macro_block_ctx
)))
{
if
(
OB_ITER_END
!=
ret
)
{
STORAGE_LOG
(
WARN
,
"fail to get next macro block, "
,
K
(
ret
));
}
else
{
ret
=
OB_SUCCESS
;
break
;
}
}
else
{
if
(
1
==
total_macro_block_count
&&
OB_FAIL
(
check_bf
(
macro_block_ctx
)))
{
STORAGE_LOG
(
WARN
,
"failed to open single scan estimator"
,
K
(
ret
));
}
else
if
(
is_empty_scan_
)
{
}
else
if
(
OB_FAIL
(
estimate_macro_row_count
(
macro_block_ctx
,
is_start_block
,
is_last_block
,
part_est
)))
{
STORAGE_LOG
(
WARN
,
"cannot estimate cost of macro block."
,
K
(
ret
),
K
(
macro_block_ctx
),
K
(
i
),
K
(
i
dx
),
K
(
total_macro_block_count
));
}
}
idx
++
;
}
part_est
.
physical_row_count_
=
part_est
.
logical_row_count_
;
}
...
...
@@ -626,7 +631,7 @@ int ObMultiVersionSingleScanEstimator::estimate_macro_row_count(const blocksstab
int64_t
logical_row_count
=
0
,
physical_row_count
=
0
;
if
(
OB_FAIL
(
context_
.
cache_context_
.
block_index_cache_
->
get_micro_infos
(
context_
.
sstable_
->
get_table_id
(),
macro_block_ctx
,
context_
.
range_
->
get_range
(),
context_
.
multi_version_range_
.
get_range
(),
is_left_border
,
is_right_border
,
micro_infos
)))
{
...
...
@@ -637,7 +642,7 @@ int ObMultiVersionSingleScanEstimator::estimate_macro_row_count(const blocksstab
}
}
else
if
(
1
!=
micro_infos
.
count
())
{
ret
=
OB_ERR_UNEXPECTED
;
STORAGE_LOG
(
WARN
,
"unexpected error, should only 1 micro block, "
,
K
(
ret
));
STORAGE_LOG
(
WARN
,
"unexpected error, should only 1 micro block, "
,
K
(
ret
)
,
K
(
micro_infos
.
count
())
);
}
else
if
(
OB_FAIL
(
estimate_border_row_count
(
micro_infos
.
at
(
0
),
macro_block_ctx
,
true
,
logical_row_count
,
physical_row_count
)))
{
STORAGE_LOG
(
WARN
,
"failed to estimate_border_row_count for multi version, "
,
K
(
ret
));
...
...
src/storage/ob_sstable_estimator.h
浏览文件 @
f0bad67c
...
...
@@ -150,8 +150,7 @@ class ObStoreRowSingleScanEstimator : public ObISSTableEstimator {
public:
ObStoreRowSingleScanEstimator
();
virtual
~
ObStoreRowSingleScanEstimator
();
int
set_context
(
ObSSTableEstimateContext
&
context
);
int
open
();
int
check_bf
(
blocksstable
::
ObMacroBlockCtx
&
macro_block_ctx
);
void
reset
();
virtual
int
estimate_row_count
(
ObPartitionEst
&
part_est
);
...
...
unittest/storage/test_micro_block_row_scanner_with_special_uncom_row.cpp
浏览文件 @
f0bad67c
...
...
@@ -1987,6 +1987,59 @@ TEST_F(TestMicroBlockRowScanner, test_magic_row)
scanner_iter
.
reset
();
}
TEST_F
(
TestMicroBlockRowScanner
,
test_estimate_with_magic_row
)
{
GCONF
.
_enable_sparse_row
=
false
;
const
int64_t
rowkey_cnt
=
4
;
const
int64_t
micro_cnt
=
2
;
const
char
*
micro_data
[
micro_cnt
];
int
index
=
0
;
micro_data
[
index
++
]
=
"bigint var bigint bigint bigint bigint flag multi_version_row_flag trans_id
\n
"
"1 var1 -1 -1 9 NOP EXIST U trans_id_1
\n
"
;
micro_data
[
index
++
]
=
"bigint var bigint bigint bigint bigint flag multi_version_row_flag trans_id
\n
"
"1 var1 MAGIC MAGIC NOP NOP EXIST LM trans_id_0
\n
"
;
prepare_data
(
micro_data
,
index
,
rowkey_cnt
,
9
,
"none"
,
FLAT_ROW_STORE
,
0
);
// minor
ObVersionRange
trans_version_range
;
trans_version_range
.
base_version_
=
0
;
trans_version_range
.
snapshot_version_
=
100
;
trans_version_range
.
multi_version_start_
=
1
;
prepare_query_param
(
trans_version_range
,
true
,
false
);
ObStoreRange
range
;
const
char
var1
[]
=
"var1"
;
ObObj
start_val
[
2
];
ObObj
end_val
[
2
];
start_val
[
0
].
set_int
(
1
);
start_val
[
1
].
set_varchar
(
var1
,
4
);
start_val
[
1
].
set_collation_type
(
CS_TYPE_UTF8MB4_GENERAL_CI
);
end_val
[
0
].
set_int
(
2
);
end_val
[
1
].
set_varchar
(
var1
,
4
);
end_val
[
1
].
set_collation_type
(
CS_TYPE_UTF8MB4_GENERAL_CI
);
ObStoreRowkey
start_key
(
start_val
,
2
);
ObStoreRowkey
end_key
(
end_val
,
2
);
range
.
table_id_
=
combine_id
(
TENANT_ID
,
TABLE_ID
);
range
.
start_key_
=
start_key
;
range
.
end_key_
=
end_key
;
common
::
ObQueryFlag
query_flag
;
ObExtStoreRange
ext_range
;
ext_range
.
reset
();
ext_range
.
get_range
()
=
range
;
ASSERT_EQ
(
OB_SUCCESS
,
ext_range
.
to_collation_free_range_on_demand_and_cutoff_range
(
allocator_
));
ObPartitionEst
cost_metrics
;
cost_metrics
.
reset
();
// (1 : 2) -> (1,-max : 2,max) would cover 1 micro block
// but if not converts to multiversion, (1 : 2) is same with (1,-max : 2,max) would cover 2 micro blocks
int
ret
=
sstable_
.
estimate_scan_row_count
(
context_
.
query_flag_
,
range
.
table_id_
,
ext_range
,
cost_metrics
);
ASSERT_EQ
(
OB_SUCCESS
,
ret
);
ASSERT_EQ
(
0
,
cost_metrics
.
logical_row_count_
);
ASSERT_EQ
(
0
,
cost_metrics
.
physical_row_count_
);
}
TEST_F
(
TestMicroBlockRowScanner
,
minor_merge_lob_reuse_allocator
)
{
const
int64_t
rowkey_cnt
=
4
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录