Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
33cff5ee
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
33cff5ee
编写于
6月 30, 2019
作者:
A
Alexey Milovidov
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'origin/master' into sanych73-prepared_statements
上级
6ad07172
a69990ce
变更
13
隐藏空白更改
内联
并排
Showing
13 changed file
with
227 addition
and
41 deletion
+227
-41
contrib/libhdfs3-cmake/CMake/Platform.cmake
contrib/libhdfs3-cmake/CMake/Platform.cmake
+7
-2
dbms/src/Common/ThreadPool.cpp
dbms/src/Common/ThreadPool.cpp
+9
-1
dbms/src/Common/tests/CMakeLists.txt
dbms/src/Common/tests/CMakeLists.txt
+0
-12
dbms/src/Common/tests/gtest_shell_command.cpp
dbms/src/Common/tests/gtest_shell_command.cpp
+72
-0
dbms/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp
dbms/src/Common/tests/gtest_thread_pool_concurrent_wait.cpp
+8
-3
dbms/src/Common/tests/gtest_thread_pool_limit.cpp
dbms/src/Common/tests/gtest_thread_pool_limit.cpp
+32
-0
dbms/src/Common/tests/gtest_thread_pool_loop.cpp
dbms/src/Common/tests/gtest_thread_pool_loop.cpp
+10
-4
dbms/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
...src/Common/tests/gtest_thread_pool_schedule_exception.cpp
+38
-0
dbms/src/Interpreters/Context.cpp
dbms/src/Interpreters/Context.cpp
+1
-0
dbms/src/Interpreters/ProcessList.cpp
dbms/src/Interpreters/ProcessList.cpp
+32
-18
dbms/src/Interpreters/ProcessList.h
dbms/src/Interpreters/ProcessList.h
+1
-1
dbms/tests/queries/0_stateless/00600_replace_running_query.reference
...queries/0_stateless/00600_replace_running_query.reference
+4
-0
dbms/tests/queries/0_stateless/00600_replace_running_query.sh
.../tests/queries/0_stateless/00600_replace_running_query.sh
+13
-0
未找到文件。
contrib/libhdfs3-cmake/CMake/Platform.cmake
浏览文件 @
33cff5ee
...
...
@@ -15,9 +15,14 @@ IF(CMAKE_COMPILER_IS_GNUCXX)
STRING
(
REGEX MATCHALL
"[0-9]+"
GCC_COMPILER_VERSION
${
GCC_COMPILER_VERSION
}
)
LIST
(
LENGTH GCC_COMPILER_VERSION GCC_COMPILER_VERSION_LENGTH
)
LIST
(
GET GCC_COMPILER_VERSION 0 GCC_COMPILER_VERSION_MAJOR
)
LIST
(
GET GCC_COMPILER_VERSION 0 GCC_COMPILER_VERSION_MINOR
)
if
(
GCC_COMPILER_VERSION_LENGTH GREATER 1
)
LIST
(
GET GCC_COMPILER_VERSION 1 GCC_COMPILER_VERSION_MINOR
)
else
()
set
(
GCC_COMPILER_VERSION_MINOR 0
)
endif
()
SET
(
GCC_COMPILER_VERSION_MAJOR
${
GCC_COMPILER_VERSION_MAJOR
}
CACHE INTERNAL
"gcc major version"
)
SET
(
GCC_COMPILER_VERSION_MINOR
${
GCC_COMPILER_VERSION_MINOR
}
CACHE INTERNAL
"gcc minor version"
)
...
...
dbms/src/Common/ThreadPool.cpp
浏览文件 @
33cff5ee
...
...
@@ -30,10 +30,18 @@ template <typename Thread>
template
<
typename
ReturnType
>
ReturnType
ThreadPoolImpl
<
Thread
>::
scheduleImpl
(
Job
job
,
int
priority
,
std
::
optional
<
uint64_t
>
wait_microseconds
)
{
auto
on_error
=
[]
auto
on_error
=
[
&
]
{
if
constexpr
(
std
::
is_same_v
<
ReturnType
,
void
>
)
{
if
(
first_exception
)
{
std
::
exception_ptr
exception
;
std
::
swap
(
exception
,
first_exception
);
std
::
rethrow_exception
(
exception
);
}
throw
DB
::
Exception
(
"Cannot schedule a task"
,
DB
::
ErrorCodes
::
CANNOT_SCHEDULE_TASK
);
}
else
return
false
;
};
...
...
dbms/src/Common/tests/CMakeLists.txt
浏览文件 @
33cff5ee
...
...
@@ -41,9 +41,6 @@ target_link_libraries (compact_array PRIVATE clickhouse_common_io ${Boost_FILESY
add_executable
(
radix_sort radix_sort.cpp
)
target_link_libraries
(
radix_sort PRIVATE clickhouse_common_io
)
add_executable
(
shell_command_test shell_command_test.cpp
)
target_link_libraries
(
shell_command_test PRIVATE clickhouse_common_io
)
add_executable
(
arena_with_free_lists arena_with_free_lists.cpp
)
target_link_libraries
(
arena_with_free_lists PRIVATE clickhouse_compression clickhouse_common_io
)
...
...
@@ -53,15 +50,6 @@ target_link_libraries (pod_array PRIVATE clickhouse_common_io)
add_executable
(
thread_creation_latency thread_creation_latency.cpp
)
target_link_libraries
(
thread_creation_latency PRIVATE clickhouse_common_io
)
add_executable
(
thread_pool thread_pool.cpp
)
target_link_libraries
(
thread_pool PRIVATE clickhouse_common_io
)
add_executable
(
thread_pool_2 thread_pool_2.cpp
)
target_link_libraries
(
thread_pool_2 PRIVATE clickhouse_common_io
)
add_executable
(
thread_pool_3 thread_pool_3.cpp
)
target_link_libraries
(
thread_pool_3 PRIVATE clickhouse_common_io
)
add_executable
(
multi_version multi_version.cpp
)
target_link_libraries
(
multi_version PRIVATE clickhouse_common_io
)
add_check
(
multi_version
)
...
...
dbms/src/Common/tests/
shell_command_test
.cpp
→
dbms/src/Common/tests/
gtest_shell_command
.cpp
浏览文件 @
33cff5ee
...
...
@@ -4,48 +4,62 @@
#include <IO/copyData.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/ReadBufferFromString.h>
#include <IO/ReadHelpers.h>
#include <chrono>
#include <thread>
#pragma GCC diagnostic ignored "-Wsign-compare"
#ifdef __clang__
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#pragma clang diagnostic ignored "-Wundef"
#endif
#include <gtest/gtest.h>
using
namespace
DB
;
int
main
(
int
,
char
**
)
try
TEST
(
ShellCommand
,
Execute
)
{
{
auto
command
=
ShellCommand
::
execute
(
"echo 'Hello, world!'"
);
auto
command
=
ShellCommand
::
execute
(
"echo 'Hello, world!'"
);
WriteBufferFromFileDescriptor
out
(
STDOUT_FILENO
);
copyData
(
command
->
out
,
out
);
std
::
string
res
;
readStringUntilEOF
(
res
,
command
->
out
);
command
->
wait
();
command
->
wait
(
);
}
EXPECT_EQ
(
res
,
"Hello, world!
\n
"
);
}
{
auto
command
=
ShellCommand
::
executeDirect
(
"/bin/echo"
,
{
"Hello, world!"
});
TEST
(
ShellCommand
,
ExecuteDirect
)
{
auto
command
=
ShellCommand
::
executeDirect
(
"/bin/echo"
,
{
"Hello, world!"
});
WriteBufferFromFileDescriptor
out
(
STDOUT_FILENO
);
copyData
(
command
->
out
,
out
);
std
::
string
res
;
readStringUntilEOF
(
res
,
command
->
out
);
command
->
wait
();
command
->
wait
(
);
}
EXPECT_EQ
(
res
,
"Hello, world!
\n
"
);
}
{
auto
command
=
ShellCommand
::
execute
(
"cat"
);
TEST
(
ShellCommand
,
ExecuteWithInput
)
{
auto
command
=
ShellCommand
::
execute
(
"cat"
);
String
in_str
=
"Hello, world!
\n
"
;
ReadBufferFromString
in
(
in_str
);
copyData
(
in
,
command
->
in
);
command
->
in
.
close
();
String
in_str
=
"Hello, world!
\n
"
;
ReadBufferFromString
in
(
in_str
);
copyData
(
in
,
command
->
in
);
command
->
in
.
close
();
WriteBufferFromFileDescriptor
out
(
STDOUT_FILENO
);
copyData
(
command
->
out
,
out
);
std
::
string
res
;
readStringUntilEOF
(
res
,
command
->
out
);
command
->
wait
();
command
->
wait
(
);
}
EXPECT_EQ
(
res
,
"Hello, world!
\n
"
);
}
TEST
(
ShellCommand
,
AutoWait
)
{
// <defunct> hunting:
for
(
int
i
=
0
;
i
<
1000
;
++
i
)
{
...
...
@@ -56,8 +70,3 @@ try
// std::cerr << "inspect me: ps auxwwf" << "\n";
// std::this_thread::sleep_for(std::chrono::seconds(100));
}
catch
(...)
{
std
::
cerr
<<
getCurrentExceptionMessage
(
false
)
<<
"
\n
"
;
return
1
;
}
dbms/src/Common/tests/
thread_pool
.cpp
→
dbms/src/Common/tests/
gtest_thread_pool_concurrent_wait
.cpp
浏览文件 @
33cff5ee
#include <Common/ThreadPool.h>
#pragma GCC diagnostic ignored "-Wsign-compare"
#ifdef __clang__
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#pragma clang diagnostic ignored "-Wundef"
#endif
#include <gtest/gtest.h>
/** Reproduces bug in ThreadPool.
* It get stuck if we call 'wait' many times from many other threads simultaneously.
*/
int
main
(
int
,
char
**
)
TEST
(
ThreadPool
,
ConcurrentWait
)
{
auto
worker
=
[]
{
...
...
@@ -29,6 +36,4 @@ int main(int, char **)
waiting_pool
.
schedule
([
&
pool
]{
pool
.
wait
();
});
waiting_pool
.
wait
();
return
0
;
}
dbms/src/Common/tests/
thread_pool_3
.cpp
→
dbms/src/Common/tests/
gtest_thread_pool_limit
.cpp
浏览文件 @
33cff5ee
#include <
mutex
>
#include <
atomic
>
#include <iostream>
#include <Common/ThreadPool.h>
#pragma GCC diagnostic ignored "-Wsign-compare"
#ifdef __clang__
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#pragma clang diagnostic ignored "-Wundef"
#endif
#include <gtest/gtest.h>
/// Test for thread self-removal when number of free threads in pool is too large.
/// Just checks that nothing weird happens.
template
<
typename
Pool
>
void
test
()
int
test
()
{
Pool
pool
(
10
,
2
,
10
);
std
::
mutex
mutex
;
std
::
atomic
<
int
>
counter
{
0
}
;
for
(
size_t
i
=
0
;
i
<
10
;
++
i
)
pool
.
schedule
([
&
]{
std
::
lock_guard
lock
(
mutex
);
std
::
cerr
<<
'.'
;
});
pool
.
schedule
([
&
]{
++
counter
;
});
pool
.
wait
();
return
counter
;
}
int
main
(
int
,
char
**
)
TEST
(
ThreadPool
,
ThreadRemoval
)
{
test
<
FreeThreadPool
>
();
std
::
cerr
<<
'\n'
;
test
<
ThreadPool
>
();
std
::
cerr
<<
'\n'
;
return
0
;
EXPECT_EQ
(
test
<
FreeThreadPool
>
(),
10
);
EXPECT_EQ
(
test
<
ThreadPool
>
(),
10
);
}
dbms/src/Common/tests/
thread_pool_2
.cpp
→
dbms/src/Common/tests/
gtest_thread_pool_loop
.cpp
浏览文件 @
33cff5ee
...
...
@@ -2,10 +2,17 @@
#include <iostream>
#include <Common/ThreadPool.h>
#pragma GCC diagnostic ignored "-Wsign-compare"
#ifdef __clang__
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#pragma clang diagnostic ignored "-Wundef"
#endif
#include <gtest/gtest.h>
int
main
(
int
,
char
**
)
TEST
(
ThreadPool
,
Loop
)
{
std
::
atomic
<
size_
t
>
res
{
0
};
std
::
atomic
<
in
t
>
res
{
0
};
for
(
size_t
i
=
0
;
i
<
1000
;
++
i
)
{
...
...
@@ -16,6 +23,5 @@ int main(int, char **)
pool
.
wait
();
}
std
::
cerr
<<
res
<<
"
\n
"
;
return
0
;
EXPECT_EQ
(
res
,
16000
);
}
dbms/src/Common/tests/gtest_thread_pool_schedule_exception.cpp
0 → 100644
浏览文件 @
33cff5ee
#include <iostream>
#include <stdexcept>
#include <Common/ThreadPool.h>
#pragma GCC diagnostic ignored "-Wsign-compare"
#ifdef __clang__
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#pragma clang diagnostic ignored "-Wundef"
#endif
#include <gtest/gtest.h>
bool
check
()
{
ThreadPool
pool
(
10
);
pool
.
schedule
([]{
throw
std
::
runtime_error
(
"Hello, world!"
);
});
try
{
for
(
size_t
i
=
0
;
i
<
100
;
++
i
)
pool
.
schedule
([]{});
/// An exception will be rethrown from this method.
}
catch
(
const
std
::
runtime_error
&
)
{
return
true
;
}
pool
.
wait
();
return
false
;
}
TEST
(
ThreadPool
,
ExceptionFromSchedule
)
{
EXPECT_TRUE
(
check
());
}
dbms/src/Interpreters/Context.cpp
浏览文件 @
33cff5ee
...
...
@@ -277,6 +277,7 @@ struct ContextShared
/// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference).
/// TODO: Get rid of this.
system_logs
.
reset
();
embedded_dictionaries
.
reset
();
external_dictionaries
.
reset
();
external_models
.
reset
();
...
...
dbms/src/Interpreters/ProcessList.cpp
浏览文件 @
33cff5ee
...
...
@@ -87,10 +87,9 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
{
std
::
unique_lock
lock
(
mutex
);
const
auto
max_wait_ms
=
settings
.
queue_max_wait_ms
.
totalMilliseconds
();
if
(
!
is_unlimited_query
&&
max_size
&&
processes
.
size
()
>=
max_size
)
{
auto
max_wait_ms
=
settings
.
queue_max_wait_ms
.
totalMilliseconds
();
if
(
!
max_wait_ms
||
!
have_space
.
wait_for
(
lock
,
std
::
chrono
::
milliseconds
(
max_wait_ms
),
[
&
]{
return
processes
.
size
()
<
max_size
;
}))
throw
Exception
(
"Too many simultaneous queries. Maximum: "
+
toString
(
max_size
),
ErrorCodes
::
TOO_MANY_SIMULTANEOUS_QUERIES
);
}
...
...
@@ -117,20 +116,41 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as
+
", maximum: "
+
settings
.
max_concurrent_queries_for_user
.
toString
(),
ErrorCodes
::
TOO_MANY_SIMULTANEOUS_QUERIES
);
auto
range
=
user_process_list
->
second
.
queries
.
equal_range
(
client_info
.
current_query_id
);
if
(
range
.
first
!=
range
.
second
)
auto
running_query
=
user_process_list
->
second
.
queries
.
find
(
client_info
.
current_query_id
);
if
(
running_query
!=
user_process_list
->
second
.
queries
.
end
())
{
if
(
!
settings
.
replace_running_query
)
throw
Exception
(
"Query with id = "
+
client_info
.
current_query_id
+
" is already running."
,
ErrorCodes
::
QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING
);
/// Ask queries to cancel. They will check this flag.
for
(
auto
it
=
range
.
first
;
it
!=
range
.
second
;
++
it
)
it
->
second
->
is_killed
.
store
(
true
,
std
::
memory_order_relaxed
);
}
running_query
->
second
->
is_killed
.
store
(
true
,
std
::
memory_order_relaxed
);
if
(
!
max_wait_ms
||
!
have_space
.
wait_for
(
lock
,
std
::
chrono
::
milliseconds
(
max_wait_ms
),
[
&
]
{
running_query
=
user_process_list
->
second
.
queries
.
find
(
client_info
.
current_query_id
);
if
(
running_query
==
user_process_list
->
second
.
queries
.
end
())
return
true
;
running_query
->
second
->
is_killed
.
store
(
true
,
std
::
memory_order_relaxed
);
return
false
;
}))
throw
Exception
(
"Query with id = "
+
client_info
.
current_query_id
+
" is already running and can't be stopped"
,
ErrorCodes
::
QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING
);
}
}
}
/// Check other users running query with our query_id
for
(
const
auto
&
user_process_list
:
user_to_queries
)
{
if
(
user_process_list
.
first
==
client_info
.
current_user
)
continue
;
if
(
auto
running_query
=
user_process_list
.
second
.
queries
.
find
(
client_info
.
current_query_id
);
running_query
!=
user_process_list
.
second
.
queries
.
end
())
throw
Exception
(
"Query with id = "
+
client_info
.
current_query_id
+
" is already running by user "
+
user_process_list
.
first
,
ErrorCodes
::
QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING
);
}
auto
process_it
=
processes
.
emplace
(
processes
.
end
(),
query_
,
client_info
,
settings
.
max_memory_usage
,
settings
.
memory_tracker_fault_probability
,
priorities
.
insert
(
settings
.
priority
));
...
...
@@ -226,17 +246,12 @@ ProcessListEntry::~ProcessListEntry()
bool
found
=
false
;
auto
range
=
user_process_list
.
queries
.
equal_range
(
query_id
);
if
(
range
.
first
!=
range
.
second
)
if
(
auto
running_query
=
user_process_list
.
queries
.
find
(
query_id
);
running_query
!=
user_process_list
.
queries
.
end
())
{
for
(
auto
jt
=
range
.
first
;
jt
!=
range
.
second
;
++
jt
)
if
(
running_query
->
second
==
process_list_element_ptr
)
{
if
(
jt
->
second
==
process_list_element_ptr
)
{
user_process_list
.
queries
.
erase
(
jt
);
found
=
true
;
break
;
}
user_process_list
.
queries
.
erase
(
running_query
->
first
);
found
=
true
;
}
}
...
...
@@ -245,8 +260,7 @@ ProcessListEntry::~ProcessListEntry()
LOG_ERROR
(
&
Logger
::
get
(
"ProcessList"
),
"Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser"
);
std
::
terminate
();
}
parent
.
have_space
.
notify_one
();
parent
.
have_space
.
notify_all
();
/// If there are no more queries for the user, then we will reset memory tracker and network throttler.
if
(
user_process_list
.
queries
.
empty
())
...
...
dbms/src/Interpreters/ProcessList.h
浏览文件 @
33cff5ee
...
...
@@ -203,7 +203,7 @@ struct ProcessListForUser
ProcessListForUser
();
/// query_id -> ProcessListElement(s). There can be multiple queries with the same query_id as long as all queries except one are cancelled.
using
QueryToElement
=
std
::
unordered_m
ultim
ap
<
String
,
QueryStatus
*>
;
using
QueryToElement
=
std
::
unordered_map
<
String
,
QueryStatus
*>
;
QueryToElement
queries
;
ProfileEvents
::
Counters
user_performance_counters
{
VariableContext
::
User
,
&
ProfileEvents
::
global_counters
};
...
...
dbms/tests/queries/0_stateless/00600_replace_running_query.reference
浏览文件 @
33cff5ee
0
1 0
3 0
2 0
44
dbms/tests/queries/0_stateless/00600_replace_running_query.sh
浏览文件 @
33cff5ee
...
...
@@ -9,3 +9,16 @@ $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL?query_id=hello&replace_running_query=1" -d
sleep
0.1
# First query (usually) should be received by the server after this sleep.
$CLICKHOUSE_CURL
-sS
"
$CLICKHOUSE_URL
?query_id=hello&replace_running_query=1"
-d
'SELECT 0'
wait
${
CLICKHOUSE_CLIENT
}
--user
=
readonly
--query_id
=
42
--query
=
'SELECT 1, sleep(1)'
&
sleep
0.1
(
${
CLICKHOUSE_CLIENT
}
--query_id
=
42
--query
=
'SELECT 43'
||
:
)
2>&1 |
grep
-F
'is already running by user'
>
/dev/null
wait
${
CLICKHOUSE_CLIENT
}
--query
=
'SELECT 3, sleep(1)'
&
sleep
0.1
${
CLICKHOUSE_CLIENT
}
--query_id
=
42
--query
=
'SELECT 2, sleep(1)'
&
sleep
0.1
(
${
CLICKHOUSE_CLIENT
}
--query_id
=
42
--replace_running_query
=
1
--queue_max_wait_ms
=
500
--query
=
'SELECT 43'
||
:
)
2>&1 |
grep
-F
'cant be stopped'
>
/dev/null
${
CLICKHOUSE_CLIENT
}
--query_id
=
42
--replace_running_query
=
1
--query
=
'SELECT 44'
wait
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录