Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
753f5b84
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
753f5b84
编写于
2月 17, 2021
作者:
V
Vitaly Baranov
提交者:
GitHub
2月 17, 2021
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #20106 from vitlibar/fix-quota-consuming
Fix quota consuming
上级
2c560d8e
5f8a6ab9
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
69 addition
and
33 deletion
+69
-33
src/Access/EnabledQuota.cpp
src/Access/EnabledQuota.cpp
+28
-19
src/Interpreters/InterpreterShowProcesslistQuery.h
src/Interpreters/InterpreterShowProcesslistQuery.h
+5
-0
src/Interpreters/InterpreterShowTablesQuery.h
src/Interpreters/InterpreterShowTablesQuery.h
+5
-0
src/Interpreters/InterpreterSystemQuery.h
src/Interpreters/InterpreterSystemQuery.h
+0
-3
src/Storages/System/StorageSystemQuotaUsage.cpp
src/Storages/System/StorageSystemQuotaUsage.cpp
+3
-3
tests/integration/test_quota/configs/users.d/assign_myquota_to_default_user.xml
..._quota/configs/users.d/assign_myquota_to_default_user.xml
+0
-0
tests/integration/test_quota/configs/users.d/myquota.xml
tests/integration/test_quota/configs/users.d/myquota.xml
+0
-0
tests/integration/test_quota/configs/users.d/user_with_no_quota.xml
...gration/test_quota/configs/users.d/user_with_no_quota.xml
+10
-0
tests/integration/test_quota/test.py
tests/integration/test_quota/test.py
+18
-8
未找到文件。
src/Access/EnabledQuota.cpp
浏览文件 @
753f5b84
...
...
@@ -26,10 +26,6 @@ struct EnabledQuota::Impl
std
::
chrono
::
seconds
duration
,
std
::
chrono
::
system_clock
::
time_point
end_of_interval
)
{
std
::
function
<
String
(
UInt64
)
>
amount_to_string
=
[](
UInt64
amount
)
{
return
std
::
to_string
(
amount
);
};
if
(
resource_type
==
Quota
::
EXECUTION_TIME
)
amount_to_string
=
[
&
](
UInt64
amount
)
{
return
ext
::
to_string
(
std
::
chrono
::
nanoseconds
(
amount
));
};
const
auto
&
type_info
=
Quota
::
ResourceTypeInfo
::
get
(
resource_type
);
throw
Exception
(
"Quota for user "
+
backQuote
(
user_name
)
+
" for "
+
ext
::
to_string
(
duration
)
+
" has been exceeded: "
...
...
@@ -39,35 +35,47 @@ struct EnabledQuota::Impl
}
/// Returns the end of the current interval. If the passed `current_time` is greater than that end,
/// the function automatically recalculates the interval's end by adding the interval's duration
/// one or more times until the interval's end is greater than `current_time`.
/// If that recalculation occurs the function also resets amounts of resources used and sets the variable
/// `counters_were_reset`.
static
std
::
chrono
::
system_clock
::
time_point
getEndOfInterval
(
const
Interval
&
interval
,
std
::
chrono
::
system_clock
::
time_point
current_time
,
bool
*
counters_were_reset
=
nullptr
)
const
Interval
&
interval
,
std
::
chrono
::
system_clock
::
time_point
current_time
,
bool
&
counters_were_reset
)
{
auto
&
end_of_interval
=
interval
.
end_of_interval
;
auto
end_loaded
=
end_of_interval
.
load
();
auto
end
=
std
::
chrono
::
system_clock
::
time_point
{
end_loaded
};
if
(
current_time
<
end
)
{
if
(
counters_were_reset
)
*
counters_were_reset
=
false
;
counters_were_reset
=
false
;
return
end
;
}
const
auto
duration
=
interval
.
duration
;
/// We reset counters only if the interval's end has been calculated before.
/// If it hasn't we just calculate the interval's end for the first time and don't reset counters yet.
bool
need_reset_counters
=
(
end_loaded
.
count
()
!=
0
);
do
{
end
=
end
+
(
current_time
-
end
+
duration
)
/
duration
*
duration
;
/// Calculate the end of the next interval:
/// | X |
/// end current_time next_end = end + duration * n
/// where n is an integer number, n >= 1.
const
auto
duration
=
interval
.
duration
;
UInt64
n
=
static_cast
<
UInt64
>
((
current_time
-
end
+
duration
)
/
duration
);
end
=
end
+
duration
*
n
;
if
(
end_of_interval
.
compare_exchange_strong
(
end_loaded
,
end
.
time_since_epoch
()))
{
boost
::
range
::
fill
(
interval
.
used
,
0
);
break
;
}
end
=
std
::
chrono
::
system_clock
::
time_point
{
end_loaded
};
}
while
(
current_time
>=
end
);
if
(
counters_were_reset
)
*
counters_were_reset
=
true
;
if
(
need_reset_counters
)
{
boost
::
range
::
fill
(
interval
.
used
,
0
);
counters_were_reset
=
true
;
}
return
end
;
}
...
...
@@ -89,7 +97,7 @@ struct EnabledQuota::Impl
if
(
used
>
max
)
{
bool
counters_were_reset
=
false
;
auto
end_of_interval
=
getEndOfInterval
(
interval
,
current_time
,
&
counters_were_reset
);
auto
end_of_interval
=
getEndOfInterval
(
interval
,
current_time
,
counters_were_reset
);
if
(
counters_were_reset
)
{
used
=
(
interval
.
used
[
resource_type
]
+=
amount
);
...
...
@@ -116,9 +124,9 @@ struct EnabledQuota::Impl
continue
;
if
(
used
>
max
)
{
bool
used_counters
_reset
=
false
;
std
::
chrono
::
system_clock
::
time_point
end_of_interval
=
getEndOfInterval
(
interval
,
current_time
,
&
used_counters
_reset
);
if
(
!
used_counters
_reset
)
bool
counters_were
_reset
=
false
;
std
::
chrono
::
system_clock
::
time_point
end_of_interval
=
getEndOfInterval
(
interval
,
current_time
,
counters_were
_reset
);
if
(
!
counters_were
_reset
)
throwQuotaExceed
(
user_name
,
intervals
.
quota_name
,
resource_type
,
used
,
max
,
interval
.
duration
,
end_of_interval
);
}
}
...
...
@@ -177,7 +185,8 @@ std::optional<QuotaUsage> EnabledQuota::Intervals::getUsage(std::chrono::system_
auto
&
out
=
usage
.
intervals
.
back
();
out
.
duration
=
in
.
duration
;
out
.
randomize_interval
=
in
.
randomize_interval
;
out
.
end_of_interval
=
Impl
::
getEndOfInterval
(
in
,
current_time
);
bool
counters_were_reset
=
false
;
out
.
end_of_interval
=
Impl
::
getEndOfInterval
(
in
,
current_time
,
counters_were_reset
);
for
(
auto
resource_type
:
ext
::
range
(
MAX_RESOURCE_TYPE
))
{
if
(
in
.
max
[
resource_type
])
...
...
src/Interpreters/InterpreterShowProcesslistQuery.h
浏览文件 @
753f5b84
...
...
@@ -20,6 +20,11 @@ public:
BlockIO
execute
()
override
;
/// We ignore the quota and limits here because execute() will rewrite a show query as a SELECT query and then
/// the SELECT query will checks the quota and limits.
bool
ignoreQuota
()
const
override
{
return
true
;
}
bool
ignoreLimits
()
const
override
{
return
true
;
}
private:
ASTPtr
query_ptr
;
Context
&
context
;
...
...
src/Interpreters/InterpreterShowTablesQuery.h
浏览文件 @
753f5b84
...
...
@@ -20,6 +20,11 @@ public:
BlockIO
execute
()
override
;
/// We ignore the quota and limits here because execute() will rewrite a show query as a SELECT query and then
/// the SELECT query will checks the quota and limits.
bool
ignoreQuota
()
const
override
{
return
true
;
}
bool
ignoreLimits
()
const
override
{
return
true
;
}
private:
ASTPtr
query_ptr
;
Context
&
context
;
...
...
src/Interpreters/InterpreterSystemQuery.h
浏览文件 @
753f5b84
...
...
@@ -37,9 +37,6 @@ public:
BlockIO
execute
()
override
;
bool
ignoreQuota
()
const
override
{
return
true
;
}
bool
ignoreLimits
()
const
override
{
return
true
;
}
private:
ASTPtr
query_ptr
;
Context
&
context
;
...
...
src/Storages/System/StorageSystemQuotaUsage.cpp
浏览文件 @
753f5b84
...
...
@@ -137,6 +137,9 @@ void StorageSystemQuotaUsage::fillDataImpl(
column_quota_name
.
insertData
(
quota_name
.
data
(),
quota_name
.
length
());
column_quota_key
.
insertData
(
quota_key
.
data
(),
quota_key
.
length
());
if
(
add_column_is_current
)
column_is_current
->
push_back
(
quota_id
==
current_quota_id
);
if
(
!
interval
)
{
column_start_time
.
insertDefault
();
...
...
@@ -171,9 +174,6 @@ void StorageSystemQuotaUsage::fillDataImpl(
addValue
(
*
column_max
[
resource_type
],
*
column_max_null_map
[
resource_type
],
interval
->
max
[
resource_type
],
type_info
);
addValue
(
*
column_usage
[
resource_type
],
*
column_usage_null_map
[
resource_type
],
interval
->
used
[
resource_type
],
type_info
);
}
if
(
add_column_is_current
)
column_is_current
->
push_back
(
quota_id
==
current_quota_id
);
};
auto
add_rows
=
[
&
](
const
String
&
quota_name
,
const
UUID
&
quota_id
,
const
String
&
quota_key
,
const
std
::
vector
<
QuotaUsage
::
Interval
>
&
intervals
)
...
...
tests/integration/test_quota/configs/users.d/assign_myquota.xml
→
tests/integration/test_quota/configs/users.d/assign_myquota
_to_default_user
.xml
浏览文件 @
753f5b84
文件已移动
tests/integration/test_quota/configs/users.d/quota.xml
→
tests/integration/test_quota/configs/users.d/
my
quota.xml
浏览文件 @
753f5b84
文件已移动
tests/integration/test_quota/configs/users.d/user_with_no_quota.xml
0 → 100644
浏览文件 @
753f5b84
<yandex>
<users>
<user_with_no_quota>
<no_password/>
<networks>
<ip>
::/0
</ip>
</networks>
</user_with_no_quota>
</users>
</yandex>
tests/integration/test_quota/test.py
浏览文件 @
753f5b84
...
...
@@ -7,9 +7,10 @@ from helpers.cluster import ClickHouseCluster
from
helpers.test_tools
import
assert_eq_with_retry
,
TSV
cluster
=
ClickHouseCluster
(
__file__
)
instance
=
cluster
.
add_instance
(
'instance'
,
user_configs
=
[
"configs/users.d/assign_myquota.xml"
,
instance
=
cluster
.
add_instance
(
'instance'
,
user_configs
=
[
"configs/users.d/assign_myquota
_to_default_user
.xml"
,
"configs/users.d/drop_default_quota.xml"
,
"configs/users.d/quota.xml"
])
"configs/users.d/myquota.xml"
,
"configs/users.d/user_with_no_quota.xml"
])
def
check_system_quotas
(
canonical
):
...
...
@@ -49,9 +50,11 @@ def system_quotas_usage(canonical):
def
copy_quota_xml
(
local_file_name
,
reload_immediately
=
True
):
script_dir
=
os
.
path
.
dirname
(
os
.
path
.
realpath
(
__file__
))
instance
.
copy_file_to_container
(
os
.
path
.
join
(
script_dir
,
local_file_name
),
'/etc/clickhouse-server/users.d/quota.xml'
)
'/etc/clickhouse-server/users.d/
my
quota.xml'
)
if
reload_immediately
:
instance
.
query
(
"SYSTEM RELOAD CONFIG"
)
# We use the special user 'user_with_no_quota' here because
# we don't want SYSTEM RELOAD CONFIG to mess our quota consuming checks.
instance
.
query
(
"SYSTEM RELOAD CONFIG"
,
user
=
'user_with_no_quota'
)
@
pytest
.
fixture
(
scope
=
"module"
,
autouse
=
True
)
...
...
@@ -71,12 +74,12 @@ def started_cluster():
@
pytest
.
fixture
(
autouse
=
True
)
def
reset_quotas_and_usage_info
():
try
:
yield
finally
:
copy_quota_xml
(
'simpliest.xml'
)
# To reset usage info.
instance
.
query
(
"DROP QUOTA IF EXISTS qA, qB"
)
copy_quota_xml
(
'simpliest.xml'
)
# To reset usage info.
copy_quota_xml
(
'normal_limits.xml'
)
yield
finally
:
pass
def
test_quota_from_users_xml
():
...
...
@@ -379,4 +382,11 @@ def test_query_inserts():
instance
.
query
(
"INSERT INTO test_table values(1)"
)
system_quota_usage
(
[[
"myQuota"
,
"default"
,
31556952
,
1
,
1000
,
0
,
500
,
1
,
500
,
0
,
"
\\
N"
,
0
,
"
\\
N"
,
0
,
"
\\
N"
,
0
,
1000
,
0
,
"
\\
N"
,
"
\\
N"
]])
\ No newline at end of file
[[
"myQuota"
,
"default"
,
31556952
,
1
,
1000
,
0
,
500
,
1
,
500
,
0
,
"
\\
N"
,
0
,
"
\\
N"
,
0
,
"
\\
N"
,
0
,
1000
,
0
,
"
\\
N"
,
"
\\
N"
]])
def
test_consumption_show_tables_quota
():
instance
.
query
(
"SHOW TABLES"
)
assert
re
.
match
(
"myQuota
\\
tdefault
\\
t.*
\\
t31556952
\\
t1
\\
t1000
\\
t1
\\
t500
\\
t0
\\
t500
\\
t0
\\
t
\\\\
N
\\
t1
\\
t
\\\\
N
\\
t19
\\
t
\\\\
N
\\
t1
\\
t1000
\\
t35
\\
t
\\\\
N
\\
t.*
\\
t
\\\\
N
\n
"
,
instance
.
query
(
"SHOW QUOTA"
))
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录