Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
a0b9bf5e
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
a0b9bf5e
编写于
4月 08, 2017
作者:
A
Alexey Milovidov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Moved code to cpp [#CLICKHOUSE-2].
上级
82766247
变更
33
隐藏空白更改
内联
并排
Showing
33 changed file
with
329 addition
and
241 deletion
+329
-241
dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h
dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h
+1
-0
dbms/src/AggregateFunctions/AggregateFunctionUniq.h
dbms/src/AggregateFunctions/AggregateFunctionUniq.h
+1
-0
dbms/src/Client/Benchmark.cpp
dbms/src/Client/Benchmark.cpp
+1
-0
dbms/src/Common/Allocator.cpp
dbms/src/Common/Allocator.cpp
+165
-0
dbms/src/Common/Allocator.h
dbms/src/Common/Allocator.h
+21
-168
dbms/src/Common/CombinedCardinalityEstimator.h
dbms/src/Common/CombinedCardinalityEstimator.h
+1
-0
dbms/src/Common/ExternalTable.h
dbms/src/Common/ExternalTable.h
+6
-0
dbms/src/Common/HyperLogLogWithSmallSetOptimization.h
dbms/src/Common/HyperLogLogWithSmallSetOptimization.h
+4
-1
dbms/src/DataStreams/AsynchronousBlockInputStream.h
dbms/src/DataStreams/AsynchronousBlockInputStream.h
+1
-0
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp
...eams/MergingAggregatedMemoryEfficientBlockInputStream.cpp
+1
-0
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h
...treams/MergingAggregatedMemoryEfficientBlockInputStream.h
+2
-0
dbms/src/DataStreams/ParallelInputsProcessor.h
dbms/src/DataStreams/ParallelInputsProcessor.h
+1
-0
dbms/src/Dictionaries/CacheDictionary.cpp
dbms/src/Dictionaries/CacheDictionary.cpp
+22
-0
dbms/src/Dictionaries/CacheDictionary.h
dbms/src/Dictionaries/CacheDictionary.h
+1
-18
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
+22
-0
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
+0
-17
dbms/src/Dictionaries/DictionaryStructure.cpp
dbms/src/Dictionaries/DictionaryStructure.cpp
+1
-0
dbms/src/Functions/FunctionsStringSearch.cpp
dbms/src/Functions/FunctionsStringSearch.cpp
+6
-0
dbms/src/Functions/FunctionsTransform.h
dbms/src/Functions/FunctionsTransform.h
+6
-0
dbms/src/IO/CompressedWriteBuffer.cpp
dbms/src/IO/CompressedWriteBuffer.cpp
+1
-0
dbms/src/IO/MemoryReadWriteBuffer.cpp
dbms/src/IO/MemoryReadWriteBuffer.cpp
+2
-0
dbms/src/IO/MemoryReadWriteBuffer.h
dbms/src/IO/MemoryReadWriteBuffer.h
+2
-0
dbms/src/IO/ReadWriteBufferFromHTTP.cpp
dbms/src/IO/ReadWriteBufferFromHTTP.cpp
+3
-2
dbms/src/Interpreters/Aggregator.cpp
dbms/src/Interpreters/Aggregator.cpp
+1
-0
dbms/src/Interpreters/DictionaryFactory.cpp
dbms/src/Interpreters/DictionaryFactory.cpp
+1
-0
dbms/src/Interpreters/ExternalDictionaries.cpp
dbms/src/Interpreters/ExternalDictionaries.cpp
+40
-1
dbms/src/Interpreters/ExternalDictionaries.h
dbms/src/Interpreters/ExternalDictionaries.h
+3
-34
dbms/src/Interpreters/InterpreterOptimizeQuery.cpp
dbms/src/Interpreters/InterpreterOptimizeQuery.cpp
+6
-0
dbms/src/Interpreters/Users.cpp
dbms/src/Interpreters/Users.cpp
+1
-0
dbms/src/Storages/MergeTree/MergeTreeReader.cpp
dbms/src/Storages/MergeTree/MergeTreeReader.cpp
+1
-0
dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
+1
-0
dbms/src/Storages/StorageBuffer.cpp
dbms/src/Storages/StorageBuffer.cpp
+1
-0
dbms/src/Storages/StorageFile.h
dbms/src/Storages/StorageFile.h
+3
-0
未找到文件。
dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h
浏览文件 @
a0b9bf5e
...
...
@@ -25,6 +25,7 @@ namespace ErrorCodes
extern
const
int
TOO_LESS_ARGUMENTS_FOR_FUNCTION
;
extern
const
int
TOO_MUCH_ARGUMENTS_FOR_FUNCTION
;
extern
const
int
SYNTAX_ERROR
;
extern
const
int
BAD_ARGUMENTS
;
}
/// helper type for comparing `std::pair`s using solely the .first member
...
...
dbms/src/AggregateFunctions/AggregateFunctionUniq.h
浏览文件 @
a0b9bf5e
...
...
@@ -16,6 +16,7 @@
#include <Common/HashTable/HashSet.h>
#include <Common/HyperLogLogWithSmallSetOptimization.h>
#include <Common/CombinedCardinalityEstimator.h>
#include <Common/MemoryTracker.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnTuple.h>
...
...
dbms/src/Client/Benchmark.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -52,6 +52,7 @@ namespace ErrorCodes
extern
const
int
POCO_EXCEPTION
;
extern
const
int
STD_EXCEPTION
;
extern
const
int
UNKNOWN_EXCEPTION
;
extern
const
int
BAD_ARGUMENTS
;
}
class
Benchmark
...
...
dbms/src/Common/Allocator.cpp
0 → 100644
浏览文件 @
a0b9bf5e
#if !defined(__APPLE__) && !defined(__FreeBSD__)
#include <malloc.h>
#endif
#include <cstdlib>
#include <sys/mman.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
#include <Common/Allocator.h>
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
BAD_ARGUMENTS
;
extern
const
int
CANNOT_ALLOCATE_MEMORY
;
extern
const
int
CANNOT_MUNMAP
;
extern
const
int
CANNOT_MREMAP
;
}
}
/** Многие современные аллокаторы (например, tcmalloc) не умеют делать mremap для realloc,
* даже в случае достаточно больших кусков памяти.
* Хотя это позволяет увеличить производительность и уменьшить потребление памяти во время realloc-а.
* Чтобы это исправить, делаем mremap самостоятельно, если кусок памяти достаточно большой.
* Порог (64 МБ) выбран достаточно большим, так как изменение адресного пространства
* довольно сильно тормозит, особенно в случае наличия большого количества потоков.
* Рассчитываем, что набор операций mmap/что-то сделать/mremap может выполняться всего лишь около 1000 раз в секунду.
*
* PS. Также это требуется, потому что tcmalloc не может выделить кусок памяти больше 16 GB.
*/
static
constexpr
size_t
MMAP_THRESHOLD
=
64
*
(
1
<<
20
);
static
constexpr
size_t
MMAP_MIN_ALIGNMENT
=
4096
;
static
constexpr
size_t
MALLOC_MIN_ALIGNMENT
=
8
;
template
<
bool
clear_memory_
>
void
*
Allocator
<
clear_memory_
>::
alloc
(
size_t
size
,
size_t
alignment
)
{
if
(
current_memory_tracker
)
current_memory_tracker
->
alloc
(
size
);
void
*
buf
;
if
(
size
>=
MMAP_THRESHOLD
)
{
if
(
alignment
>
MMAP_MIN_ALIGNMENT
)
throw
DB
::
Exception
(
"Too large alignment: more than page size."
,
DB
::
ErrorCodes
::
BAD_ARGUMENTS
);
buf
=
mmap
(
NULL
,
size
,
PROT_READ
|
PROT_WRITE
,
MAP_PRIVATE
|
MAP_ANONYMOUS
,
-
1
,
0
);
if
(
MAP_FAILED
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot mmap."
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
);
/// No need for zero-fill, because mmap guarantees it.
}
else
{
if
(
alignment
<=
MALLOC_MIN_ALIGNMENT
)
{
if
(
clear_memory
)
buf
=
::
calloc
(
size
,
1
);
else
buf
=
::
malloc
(
size
);
if
(
nullptr
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot malloc."
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
);
}
else
{
buf
=
nullptr
;
int
res
=
posix_memalign
(
&
buf
,
alignment
,
size
);
if
(
0
!=
res
)
DB
::
throwFromErrno
(
"Cannot allocate memory (posix_memalign)"
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
,
res
);
if
(
clear_memory
)
memset
(
buf
,
0
,
size
);
}
}
return
buf
;
}
template
<
bool
clear_memory_
>
void
Allocator
<
clear_memory_
>::
free
(
void
*
buf
,
size_t
size
)
{
if
(
size
>=
MMAP_THRESHOLD
)
{
if
(
0
!=
munmap
(
buf
,
size
))
DB
::
throwFromErrno
(
"Allocator: Cannot munmap."
,
DB
::
ErrorCodes
::
CANNOT_MUNMAP
);
}
else
{
::
free
(
buf
);
}
if
(
current_memory_tracker
)
current_memory_tracker
->
free
(
size
);
}
template
<
bool
clear_memory_
>
void
*
Allocator
<
clear_memory_
>::
realloc
(
void
*
buf
,
size_t
old_size
,
size_t
new_size
,
size_t
alignment
)
{
#if !defined(__APPLE__) && !defined(__FreeBSD__)
if
(
old_size
<
MMAP_THRESHOLD
&&
new_size
<
MMAP_THRESHOLD
&&
alignment
<=
MALLOC_MIN_ALIGNMENT
)
{
if
(
current_memory_tracker
)
current_memory_tracker
->
realloc
(
old_size
,
new_size
);
buf
=
::
realloc
(
buf
,
new_size
);
if
(
nullptr
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot realloc."
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
);
if
(
clear_memory
)
memset
(
reinterpret_cast
<
char
*>
(
buf
)
+
old_size
,
0
,
new_size
-
old_size
);
}
else
if
(
old_size
>=
MMAP_THRESHOLD
&&
new_size
>=
MMAP_THRESHOLD
)
{
if
(
current_memory_tracker
)
current_memory_tracker
->
realloc
(
old_size
,
new_size
);
buf
=
mremap
(
buf
,
old_size
,
new_size
,
MREMAP_MAYMOVE
);
if
(
MAP_FAILED
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot mremap."
,
DB
::
ErrorCodes
::
CANNOT_MREMAP
);
/// No need for zero-fill, because mmap guarantees it.
}
#else
// TODO: We need to use mmap/calloc on Apple too.
if
((
old_size
<
MMAP_THRESHOLD
&&
new_size
<
MMAP_THRESHOLD
&&
alignment
<=
MALLOC_MIN_ALIGNMENT
)
||
(
old_size
>=
MMAP_THRESHOLD
&&
new_size
>=
MMAP_THRESHOLD
))
{
if
(
current_memory_tracker
)
current_memory_tracker
->
realloc
(
old_size
,
new_size
);
buf
=
::
realloc
(
buf
,
new_size
);
if
(
nullptr
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot realloc."
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
);
if
(
clear_memory
)
memset
(
reinterpret_cast
<
char
*>
(
buf
)
+
old_size
,
0
,
new_size
-
old_size
);
}
#endif
else
{
void
*
new_buf
=
alloc
(
new_size
,
alignment
);
memcpy
(
new_buf
,
buf
,
old_size
);
free
(
buf
,
old_size
);
buf
=
new_buf
;
}
return
buf
;
}
/// Explicit template instantinations.
template
class
Allocator
<
true
>;
template
class
Allocator
<
false
>;
dbms/src/Common/Allocator.h
浏览文件 @
a0b9bf5e
#pragma once
#if !defined(__APPLE__) && !defined(__FreeBSD__)
#include <malloc.h>
#endif
#include <cstdlib>
#include <string.h>
#include <sys/mman.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
BAD_ARGUMENTS
;
extern
const
int
CANNOT_ALLOCATE_MEMORY
;
extern
const
int
CANNOT_MUNMAP
;
extern
const
int
CANNOT_MREMAP
;
}
}
/** При использовании AllocatorWithStackMemory, размещённом на стеке,
* GCC 4.9 ошибочно делает предположение, что мы можем вызывать free от указателя на стек.
* На самом деле, комбинация условий внутри AllocatorWithStackMemory этого не допускает.
*/
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
#endif
/** Отвечает за выделение/освобождение памяти. Используется, например, в PODArray, Arena.
...
...
@@ -47,144 +17,18 @@ class Allocator
protected:
static
constexpr
bool
clear_memory
=
clear_memory_
;
private:
/** Многие современные аллокаторы (например, tcmalloc) не умеют делать mremap для realloc,
* даже в случае достаточно больших кусков памяти.
* Хотя это позволяет увеличить производительность и уменьшить потребление памяти во время realloc-а.
* Чтобы это исправить, делаем mremap самостоятельно, если кусок памяти достаточно большой.
* Порог (64 МБ) выбран достаточно большим, так как изменение адресного пространства
* довольно сильно тормозит, особенно в случае наличия большого количества потоков.
* Рассчитываем, что набор операций mmap/что-то сделать/mremap может выполняться всего лишь около 1000 раз в секунду.
*
* PS. Также это требуется, потому что tcmalloc не может выделить кусок памяти больше 16 GB.
*/
static
constexpr
size_t
MMAP_THRESHOLD
=
64
*
(
1
<<
20
);
static
constexpr
size_t
MMAP_MIN_ALIGNMENT
=
4096
;
static
constexpr
size_t
MALLOC_MIN_ALIGNMENT
=
8
;
public:
/// Выделить кусок памяти.
void
*
alloc
(
size_t
size
,
size_t
alignment
=
0
)
{
if
(
current_memory_tracker
)
current_memory_tracker
->
alloc
(
size
);
void
*
buf
;
if
(
size
>=
MMAP_THRESHOLD
)
{
if
(
alignment
>
MMAP_MIN_ALIGNMENT
)
throw
DB
::
Exception
(
"Too large alignment: more than page size."
,
DB
::
ErrorCodes
::
BAD_ARGUMENTS
);
buf
=
mmap
(
NULL
,
size
,
PROT_READ
|
PROT_WRITE
,
MAP_PRIVATE
|
MAP_ANONYMOUS
,
-
1
,
0
);
if
(
MAP_FAILED
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot mmap."
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
);
/// Заполнение нулями не нужно - mmap сам это делает.
}
else
{
if
(
alignment
<=
MALLOC_MIN_ALIGNMENT
)
{
if
(
clear_memory
)
buf
=
::
calloc
(
size
,
1
);
else
buf
=
::
malloc
(
size
);
if
(
nullptr
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot malloc."
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
);
}
else
{
buf
=
nullptr
;
int
res
=
posix_memalign
(
&
buf
,
alignment
,
size
);
if
(
0
!=
res
)
DB
::
throwFromErrno
(
"Cannot allocate memory (posix_memalign)"
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
,
res
);
if
(
clear_memory
)
memset
(
buf
,
0
,
size
);
}
}
return
buf
;
}
/// Освободить память.
void
free
(
void
*
buf
,
size_t
size
)
{
if
(
size
>=
MMAP_THRESHOLD
)
{
if
(
0
!=
munmap
(
buf
,
size
))
DB
::
throwFromErrno
(
"Allocator: Cannot munmap."
,
DB
::
ErrorCodes
::
CANNOT_MUNMAP
);
}
else
{
::
free
(
buf
);
}
/// Allocate memory range.
void
*
alloc
(
size_t
size
,
size_t
alignment
=
0
);
if
(
current_memory_tracker
)
current_memory_tracker
->
free
(
size
);
}
/// Free memory range.
void
free
(
void
*
buf
,
size_t
size
);
/**
Увеличить размер куска памяти
.
*
Содержимое старого куска памяти переезжает в начало нового
.
*
Положение куска памяти может измениться
.
/**
Enlarge memory range
.
*
Data from old range is moved to the beginning of new range
.
*
Address of memory range could change
.
*/
void
*
realloc
(
void
*
buf
,
size_t
old_size
,
size_t
new_size
,
size_t
alignment
=
0
)
{
#if !defined(__APPLE__) && !defined(__FreeBSD__)
if
(
old_size
<
MMAP_THRESHOLD
&&
new_size
<
MMAP_THRESHOLD
&&
alignment
<=
MALLOC_MIN_ALIGNMENT
)
{
if
(
current_memory_tracker
)
current_memory_tracker
->
realloc
(
old_size
,
new_size
);
buf
=
::
realloc
(
buf
,
new_size
);
if
(
nullptr
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot realloc."
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
);
if
(
clear_memory
)
memset
(
reinterpret_cast
<
char
*>
(
buf
)
+
old_size
,
0
,
new_size
-
old_size
);
}
else
if
(
old_size
>=
MMAP_THRESHOLD
&&
new_size
>=
MMAP_THRESHOLD
)
{
if
(
current_memory_tracker
)
current_memory_tracker
->
realloc
(
old_size
,
new_size
);
buf
=
mremap
(
buf
,
old_size
,
new_size
,
MREMAP_MAYMOVE
);
if
(
MAP_FAILED
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot mremap."
,
DB
::
ErrorCodes
::
CANNOT_MREMAP
);
/// Заполнение нулями не нужно.
}
#else
// TODO: We need to use mmap/calloc on Apple too.
if
((
old_size
<
MMAP_THRESHOLD
&&
new_size
<
MMAP_THRESHOLD
&&
alignment
<=
MALLOC_MIN_ALIGNMENT
)
||
(
old_size
>=
MMAP_THRESHOLD
&&
new_size
>=
MMAP_THRESHOLD
))
{
if
(
current_memory_tracker
)
current_memory_tracker
->
realloc
(
old_size
,
new_size
);
buf
=
::
realloc
(
buf
,
new_size
);
if
(
nullptr
==
buf
)
DB
::
throwFromErrno
(
"Allocator: Cannot realloc."
,
DB
::
ErrorCodes
::
CANNOT_ALLOCATE_MEMORY
);
if
(
clear_memory
)
memset
(
reinterpret_cast
<
char
*>
(
buf
)
+
old_size
,
0
,
new_size
-
old_size
);
}
#endif
else
{
void
*
new_buf
=
alloc
(
new_size
,
alignment
);
memcpy
(
new_buf
,
buf
,
old_size
);
free
(
buf
,
old_size
);
buf
=
new_buf
;
}
return
buf
;
}
void
*
realloc
(
void
*
buf
,
size_t
old_size
,
size_t
new_size
,
size_t
alignment
=
0
);
protected:
static
constexpr
size_t
getStackThreshold
()
...
...
@@ -194,7 +38,16 @@ protected:
};
/** Аллокатор с оптимизацией для маленьких кусков памяти.
/** При использовании AllocatorWithStackMemory, размещённом на стеке,
* GCC 4.9 ошибочно делает предположение, что мы можем вызывать free от указателя на стек.
* На самом деле, комбинация условий внутри AllocatorWithStackMemory этого не допускает.
*/
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
#endif
/** Allocator with optimization to place small memory ranges in automatic memory.
*/
template
<
typename
Base
,
size_t
N
=
64
>
class
AllocatorWithStackMemory
:
private
Base
...
...
@@ -223,15 +76,15 @@ public:
void
*
realloc
(
void
*
buf
,
size_t
old_size
,
size_t
new_size
)
{
///
Было в stack_memory, там и останется
.
///
Was in stack_memory, will remain there
.
if
(
new_size
<=
N
)
return
buf
;
///
Уже не помещалось в
stack_memory.
///
Already was big enough to not fit in
stack_memory.
if
(
old_size
>
N
)
return
Base
::
realloc
(
buf
,
old_size
,
new_size
);
///
Было в stack_memory, но теперь не помещается
.
///
Was in stack memory, but now will not fit there
.
void
*
new_buf
=
Base
::
alloc
(
new_size
);
memcpy
(
new_buf
,
buf
,
old_size
);
return
new_buf
;
...
...
dbms/src/Common/CombinedCardinalityEstimator.h
浏览文件 @
a0b9bf5e
...
...
@@ -3,6 +3,7 @@
#include <Common/HashTable/SmallTable.h>
#include <Common/HashTable/HashSet.h>
#include <Common/HyperLogLogCounter.h>
#include <Common/MemoryTracker.h>
#include <Core/Defines.h>
...
...
dbms/src/Common/ExternalTable.h
浏览文件 @
a0b9bf5e
...
...
@@ -19,6 +19,12 @@
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
BAD_ARGUMENTS
;
}
/// Базовый класс содержащий основную информацию о внешней таблице и
/// основные функции для извлечения этой информации из текстовых полей.
class
BaseExternalTable
...
...
dbms/src/Common/HyperLogLogWithSmallSetOptimization.h
浏览文件 @
a0b9bf5e
...
...
@@ -2,6 +2,8 @@
#include <Common/HyperLogLogCounter.h>
#include <Common/HashTable/SmallTable.h>
#include <Common/MemoryTracker.h>
namespace
DB
{
...
...
@@ -12,7 +14,8 @@ namespace DB
* Смотрите также более практичную реализацию в CombinedCardinalityEstimator.h,
* где используется также хэш-таблица для множеств среднего размера.
*/
template
<
template
<
typename
Key
,
UInt8
small_set_size
,
UInt8
K
,
...
...
dbms/src/DataStreams/AsynchronousBlockInputStream.h
浏览文件 @
a0b9bf5e
...
...
@@ -6,6 +6,7 @@
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/ThreadPool.h>
#include <Common/MemoryTracker.h>
namespace
CurrentMetrics
...
...
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp
浏览文件 @
a0b9bf5e
#include <future>
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
#include <DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h>
...
...
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h
浏览文件 @
a0b9bf5e
...
...
@@ -7,6 +7,8 @@
#include <condition_variable>
class
MemoryTracker
;
namespace
DB
{
...
...
dbms/src/DataStreams/ParallelInputsProcessor.h
浏览文件 @
a0b9bf5e
...
...
@@ -11,6 +11,7 @@
#include <DataStreams/IProfilingBlockInputStream.h>
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
/** Allows to process multiple block input streams (sources) in parallel, using specified number of threads.
...
...
dbms/src/Dictionaries/CacheDictionary.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -6,11 +6,33 @@
#include <Common/HashTable/Hash.h>
#include <Common/Stopwatch.h>
#include <Common/ProfilingScopedRWLock.h>
#include <Common/ProfileEvents.h>
#include <Common/CurrentMetrics.h>
#include <ext/size.hpp>
#include <ext/range.hpp>
#include <ext/map.hpp>
namespace
ProfileEvents
{
extern
const
Event
DictCacheKeysRequested
;
extern
const
Event
DictCacheKeysRequestedMiss
;
extern
const
Event
DictCacheKeysRequestedFound
;
extern
const
Event
DictCacheKeysExpired
;
extern
const
Event
DictCacheKeysNotFound
;
extern
const
Event
DictCacheKeysHit
;
extern
const
Event
DictCacheRequestTimeNs
;
extern
const
Event
DictCacheRequests
;
extern
const
Event
DictCacheLockWriteNs
;
extern
const
Event
DictCacheLockReadNs
;
}
namespace
CurrentMetrics
{
extern
const
Metric
DictCacheRequests
;
}
namespace
DB
{
...
...
dbms/src/Dictionaries/CacheDictionary.h
浏览文件 @
a0b9bf5e
...
...
@@ -4,6 +4,7 @@
#include <Dictionaries/IDictionarySource.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Common/ArenaWithFreeLists.h>
#include <Common/CurrentMetrics.h>
#include <Columns/ColumnString.h>
#include <ext/bit_cast.hpp>
#include <Poco/RWLock.h>
...
...
@@ -15,24 +16,6 @@
#include <tuple>
#include <random>
namespace
ProfileEvents
{
extern
const
Event
DictCacheKeysRequested
;
extern
const
Event
DictCacheKeysRequestedMiss
;
extern
const
Event
DictCacheKeysRequestedFound
;
extern
const
Event
DictCacheKeysExpired
;
extern
const
Event
DictCacheKeysNotFound
;
extern
const
Event
DictCacheKeysHit
;
extern
const
Event
DictCacheRequestTimeNs
;
extern
const
Event
DictCacheRequests
;
extern
const
Event
DictCacheLockWriteNs
;
extern
const
Event
DictCacheLockReadNs
;
}
namespace
CurrentMetrics
{
extern
const
Metric
DictCacheRequests
;
}
namespace
DB
{
...
...
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -3,8 +3,30 @@
#include <Common/randomSeed.h>
#include <Common/Stopwatch.h>
#include <Common/ProfilingScopedRWLock.h>
#include <Common/ProfileEvents.h>
#include <Common/CurrentMetrics.h>
#include <ext/range.hpp>
namespace
ProfileEvents
{
extern
const
Event
DictCacheKeysRequested
;
extern
const
Event
DictCacheKeysRequestedMiss
;
extern
const
Event
DictCacheKeysRequestedFound
;
extern
const
Event
DictCacheKeysExpired
;
extern
const
Event
DictCacheKeysNotFound
;
extern
const
Event
DictCacheKeysHit
;
extern
const
Event
DictCacheRequestTimeNs
;
extern
const
Event
DictCacheLockWriteNs
;
extern
const
Event
DictCacheLockReadNs
;
}
namespace
CurrentMetrics
{
extern
const
Metric
DictCacheRequests
;
}
namespace
DB
{
...
...
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
浏览文件 @
a0b9bf5e
...
...
@@ -20,23 +20,6 @@
#include <tuple>
#include <random>
namespace
ProfileEvents
{
extern
const
Event
DictCacheKeysRequested
;
extern
const
Event
DictCacheKeysRequestedMiss
;
extern
const
Event
DictCacheKeysRequestedFound
;
extern
const
Event
DictCacheKeysExpired
;
extern
const
Event
DictCacheKeysNotFound
;
extern
const
Event
DictCacheKeysHit
;
extern
const
Event
DictCacheRequestTimeNs
;
extern
const
Event
DictCacheLockWriteNs
;
extern
const
Event
DictCacheLockReadNs
;
}
namespace
CurrentMetrics
{
extern
const
Metric
DictCacheRequests
;
}
namespace
DB
{
...
...
dbms/src/Dictionaries/DictionaryStructure.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -11,6 +11,7 @@ namespace ErrorCodes
extern
const
int
UNKNOWN_TYPE
;
extern
const
int
ARGUMENT_OUT_OF_BOUND
;
extern
const
int
TYPE_MISMATCH
;
extern
const
int
BAD_ARGUMENTS
;
}
...
...
dbms/src/Functions/FunctionsStringSearch.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -23,6 +23,12 @@
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
BAD_ARGUMENTS
;
}
/** Implementation details for functions of 'position' family depending on ASCII/UTF8 and case sensitiveness.
*/
struct
PositionCaseSensitiveASCII
...
...
dbms/src/Functions/FunctionsTransform.h
浏览文件 @
a0b9bf5e
...
...
@@ -19,6 +19,12 @@
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
BAD_ARGUMENTS
;
}
/** transform(x, from_array, to_array[, default]) - преобразовать x согласно переданному явным образом соответствию.
*/
...
...
dbms/src/IO/CompressedWriteBuffer.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -10,6 +10,7 @@
#include <zstd.h>
#include <Common/unaligned.h>
#include <Core/Types.h>
#include <IO/CompressedWriteBuffer.h>
...
...
dbms/src/IO/MemoryReadWriteBuffer.cpp
浏览文件 @
a0b9bf5e
#include <IO/MemoryReadWriteBuffer.h>
#include <common/likely.h>
#include <boost/noncopyable.hpp>
namespace
DB
{
...
...
dbms/src/IO/MemoryReadWriteBuffer.h
浏览文件 @
a0b9bf5e
...
...
@@ -5,6 +5,8 @@
#include <IO/IReadableWriteBuffer.h>
#include <Common/Allocator.h>
#include <Core/Defines.h>
#include <boost/noncopyable.hpp>
namespace
DB
{
...
...
dbms/src/IO/ReadWriteBufferFromHTTP.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -7,13 +7,15 @@
#include <Poco/Net/HTTPResponse.h>
#include <IO/ReadBufferFromIStream.h>
#include <Common/SimpleCache.h>
#include <Common/config.h>
#include <Core/Types.h>
#include <common/logger_useful.h>
#include <Common/config.h>
#if Poco_NetSSL_FOUND
#include <Poco/Net/HTTPSClientSession.h>
#endif
namespace
DB
{
...
...
@@ -33,7 +35,6 @@ static Poco::Net::IPAddress resolveHost(const String & host)
static
SimpleCache
<
decltype
(
resolveHostImpl
),
&
resolveHostImpl
>
cache
;
return
cache
(
host
);
}
// ==========
ReadWriteBufferFromHTTP
::
ReadWriteBufferFromHTTP
(
...
...
dbms/src/Interpreters/Aggregator.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -20,6 +20,7 @@
#include <Interpreters/Aggregator.h>
#include <Common/ClickHouseRevision.h>
#include <Common/MemoryTracker.h>
#include <Interpreters/config_compile.h>
...
...
dbms/src/Interpreters/DictionaryFactory.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -19,6 +19,7 @@ namespace ErrorCodes
extern
const
int
UNKNOWN_ELEMENT_IN_CONFIG
;
extern
const
int
UNSUPPORTED_METHOD
;
extern
const
int
TOO_SMALL_BUFFER_SIZE
;
extern
const
int
BAD_ARGUMENTS
;
}
...
...
dbms/src/Interpreters/ExternalDictionaries.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -3,6 +3,7 @@
#include <Dictionaries/DictionaryStructure.h>
#include <Dictionaries/IDictionarySource.h>
#include <Common/StringUtils.h>
#include <Common/MemoryTracker.h>
#include <ext/scope_guard.hpp>
#include <Poco/Util/Application.h>
#include <Poco/Glob.h>
...
...
@@ -11,7 +12,7 @@
namespace
{
/// 5 seconds
const
auto
check_period_sec
=
5
;
const
auto
backoff_initial_sec
=
5
;
/// 10 minutes
const
auto
backoff_max_sec
=
10
*
60
;
...
...
@@ -27,6 +28,44 @@ namespace ErrorCodes
}
void
ExternalDictionaries
::
reloadPeriodically
()
{
setThreadName
(
"ExterDictReload"
);
while
(
true
)
{
if
(
destroy
.
tryWait
(
check_period_sec
*
1000
))
return
;
reloadImpl
();
}
}
ExternalDictionaries
::
ExternalDictionaries
(
Context
&
context
,
const
bool
throw_on_error
)
:
context
(
context
),
log
(
&
Logger
::
get
(
"ExternalDictionaries"
))
{
{
/** During synchronous loading of external dictionaries at moment of query execution,
* we should not use per query memory limit.
*/
TemporarilyDisableMemoryTracker
temporarily_disable_memory_tracker
;
reloadImpl
(
throw_on_error
);
}
reloading_thread
=
std
::
thread
{
&
ExternalDictionaries
::
reloadPeriodically
,
this
};
}
ExternalDictionaries
::~
ExternalDictionaries
()
{
destroy
.
set
();
reloading_thread
.
join
();
}
namespace
{
std
::
set
<
std
::
string
>
getDictionariesConfigPaths
(
const
Poco
::
Util
::
AbstractConfiguration
&
config
)
...
...
dbms/src/Interpreters/ExternalDictionaries.h
浏览文件 @
a0b9bf5e
...
...
@@ -37,8 +37,6 @@ class Context;
class
ExternalDictionaries
{
private:
static
const
auto
check_period_sec
=
5
;
friend
class
StorageSystemDictionaries
;
mutable
std
::
mutex
dictionaries_mutex
;
...
...
@@ -85,41 +83,12 @@ private:
void
reloadImpl
(
bool
throw_on_error
=
false
);
void
reloadFromFile
(
const
std
::
string
&
config_path
,
bool
throw_on_error
);
void
reloadPeriodically
()
{
setThreadName
(
"ExterDictReload"
);
while
(
true
)
{
if
(
destroy
.
tryWait
(
check_period_sec
*
1000
))
return
;
reloadImpl
();
}
}
void
reloadPeriodically
();
public:
/// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
ExternalDictionaries
(
Context
&
context
,
const
bool
throw_on_error
)
:
context
(
context
),
log
(
&
Logger
::
get
(
"ExternalDictionaries"
))
{
{
/** During synchronous loading of external dictionaries at moment of query execution,
* we should not use per query memory limit.
*/
TemporarilyDisableMemoryTracker
temporarily_disable_memory_tracker
;
reloadImpl
(
throw_on_error
);
}
reloading_thread
=
std
::
thread
{
&
ExternalDictionaries
::
reloadPeriodically
,
this
};
}
~
ExternalDictionaries
()
{
destroy
.
set
();
reloading_thread
.
join
();
}
ExternalDictionaries
(
Context
&
context
,
const
bool
throw_on_error
);
~
ExternalDictionaries
();
MultiVersion
<
IDictionaryBase
>::
Version
getDictionary
(
const
std
::
string
&
name
)
const
;
};
...
...
dbms/src/Interpreters/InterpreterOptimizeQuery.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -6,6 +6,12 @@
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
BAD_ARGUMENTS
;
}
BlockIO
InterpreterOptimizeQuery
::
execute
()
{
const
ASTOptimizeQuery
&
ast
=
typeid_cast
<
const
ASTOptimizeQuery
&>
(
*
query_ptr
);
...
...
dbms/src/Interpreters/Users.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -35,6 +35,7 @@ namespace ErrorCodes
extern
const
int
REQUIRED_PASSWORD
;
extern
const
int
WRONG_PASSWORD
;
extern
const
int
IP_ADDRESS_NOT_ALLOWED
;
extern
const
int
BAD_ARGUMENTS
;
}
...
...
dbms/src/Storages/MergeTree/MergeTreeReader.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -3,6 +3,7 @@
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeNullable.h>
#include <Common/escapeForFileName.h>
#include <Common/MemoryTracker.h>
#include <IO/CachedCompressedReadBuffer.h>
#include <IO/CompressedReadBufferFromFile.h>
#include <Columns/ColumnArray.h>
...
...
dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -7,6 +7,7 @@
#include <Columns/ColumnArray.h>
#include <Columns/ColumnNullable.h>
#include <Common/StringUtils.h>
#include <Common/MemoryTracker.h>
#include <Poco/File.h>
...
...
dbms/src/Storages/StorageBuffer.cpp
浏览文件 @
a0b9bf5e
...
...
@@ -9,6 +9,7 @@
#include <Parsers/ASTExpressionList.h>
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
#include <common/logger_useful.h>
#include <Poco/Ext/ThreadNumber.h>
...
...
dbms/src/Storages/StorageFile.h
浏览文件 @
a0b9bf5e
...
...
@@ -8,6 +8,9 @@
#include <common/logger_useful.h>
#include <atomic>
namespace
DB
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录