Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
1b0d1a44
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
1b0d1a44
编写于
5月 31, 2019
作者:
P
proller
提交者:
alexey-milovidov
5月 31, 2019
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Info about frozen parts in system.parts (#5471)
* wip * wip * wip * Clean * fix test * clean * fix test * fix test
上级
261dd566
变更
14
隐藏空白更改
内联
并排
Showing
14 changed file
with
60 addition
and
26 deletion
+60
-26
dbms/src/Common/localBackup.cpp
dbms/src/Common/localBackup.cpp
+3
-1
dbms/src/Common/localBackup.h
dbms/src/Common/localBackup.h
+1
-1
dbms/src/Storages/MergeTree/MergeTreeData.cpp
dbms/src/Storages/MergeTree/MergeTreeData.cpp
+1
-0
dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
+2
-4
dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
+6
-5
dbms/src/Storages/MergeTree/MergeTreeDataPart.h
dbms/src/Storages/MergeTree/MergeTreeDataPart.h
+3
-0
dbms/src/Storages/StorageMergeTree.cpp
dbms/src/Storages/StorageMergeTree.cpp
+2
-8
dbms/src/Storages/System/StorageSystemParts.cpp
dbms/src/Storages/System/StorageSystemParts.cpp
+4
-1
dbms/src/Storages/System/StorageSystemPartsColumns.cpp
dbms/src/Storages/System/StorageSystemPartsColumns.cpp
+2
-1
dbms/src/Storages/tests/transform_part_zk_nodes.cpp
dbms/src/Storages/tests/transform_part_zk_nodes.cpp
+0
-1
dbms/tests/queries/0_stateless/00428_partition.reference
dbms/tests/queries/0_stateless/00428_partition.reference
+0
-2
dbms/tests/queries/0_stateless/00428_partition.sh
dbms/tests/queries/0_stateless/00428_partition.sh
+3
-2
dbms/tests/queries/0_stateless/00952_part_frozen_info.reference
...ests/queries/0_stateless/00952_part_frozen_info.reference
+19
-0
dbms/tests/queries/0_stateless/00952_part_frozen_info.sql
dbms/tests/queries/0_stateless/00952_part_frozen_info.sql
+14
-0
未找到文件。
dbms/src/Common/localBackup.cpp
浏览文件 @
1b0d1a44
#include <Common/localBackup.h>
#include "localBackup.h"
#include <Common/createHardLink.h>
#include <Common/Exception.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/Path.h>
#include <Poco/File.h>
#include <string>
#include <iostream>
...
...
dbms/src/Common/localBackup.h
浏览文件 @
1b0d1a44
#pragma once
#include <Poco/Path.h>
#include <optional>
namespace
Poco
{
class
Path
;
}
namespace
DB
{
...
...
dbms/src/Storages/MergeTree/MergeTreeData.cpp
浏览文件 @
1b0d1a44
...
...
@@ -2829,6 +2829,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String &
String
backup_part_absolute_path
=
part_absolute_path
;
backup_part_absolute_path
.
replace
(
0
,
clickhouse_path
.
size
(),
backup_path
);
localBackup
(
part_absolute_path
,
backup_part_absolute_path
);
part
->
is_frozen
=
true
;
++
parts_processed
;
}
...
...
dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
浏览文件 @
1b0d1a44
#include <Storages/MergeTree/MergeTreeDataMergerMutator.h>
#include "MergeTreeDataMergerMutator.h"
#include <Storages/MergeTree/MergeTreeSequentialBlockInputStream.h>
#include <Storages/MergeTree/MergedBlockOutputStream.h>
#include <Storages/MergeTree/DiskSpaceMonitor.h>
...
...
@@ -25,12 +26,9 @@
#include <Common/SimpleIncrement.h>
#include <Common/interpolate.h>
#include <Common/typeid_cast.h>
#include <Common/localBackup.h>
#include <Common/createHardLink.h>
#include <Poco/File.h>
#include <Poco/DirectoryIterator.h>
#include <cmath>
#include <numeric>
#include <iomanip>
...
...
dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp
浏览文件 @
1b0d1a44
#include
<optional>
#include
"MergeTreeDataPart.h"
#include <optional>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Compression/CompressedReadBuffer.h>
...
...
@@ -14,13 +15,10 @@
#include <Common/StringUtils/StringUtils.h>
#include <Common/localBackup.h>
#include <Compression/CompressionInfo.h>
#include <Storages/MergeTree/MergeTreeDataPart.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Poco/DirectoryIterator.h>
#include <common/logger_useful.h>
#include <common/JSON.h>
...
...
@@ -728,7 +726,8 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con
void
MergeTreeDataPart
::
loadColumns
(
bool
require
)
{
String
path
=
getFullPath
()
+
"columns.txt"
;
if
(
!
Poco
::
File
(
path
).
exists
())
Poco
::
File
poco_file_path
{
path
};
if
(
!
poco_file_path
.
exists
())
{
if
(
require
)
throw
Exception
(
"No columns.txt in part "
+
name
,
ErrorCodes
::
NO_FILE_IN_DATA_PART
);
...
...
@@ -750,6 +749,8 @@ void MergeTreeDataPart::loadColumns(bool require)
return
;
}
is_frozen
=
!
poco_file_path
.
canWrite
();
ReadBufferFromFile
file
=
openForReading
(
path
);
columns
.
readText
(
file
);
}
...
...
dbms/src/Storages/MergeTree/MergeTreeDataPart.h
浏览文件 @
1b0d1a44
...
...
@@ -106,6 +106,9 @@ struct MergeTreeDataPart
/// If true it means that there are no ZooKeeper node for this part, so it should be deleted only from filesystem
bool
is_duplicate
=
false
;
/// Frozen by ALTER TABLE ... FREEZE ...
mutable
bool
is_frozen
=
false
;
/**
* Part state is a stage of its lifetime. States are ordered and state of a part could be increased only.
* Part state should be modified under data_parts mutex.
...
...
dbms/src/Storages/StorageMergeTree.cpp
浏览文件 @
1b0d1a44
#include
<Databases/IDatabase.h>
#include
"StorageMergeTree.h"
#include <Databases/IDatabase.h>
#include <Common/escapeForFileName.h>
#include <Common/typeid_cast.h>
#include <Common/FieldVisitors.h>
#include <Common/ThreadPool.h>
#include <Common/localBackup.h>
#include <Interpreters/InterpreterAlterQuery.h>
#include <Interpreters/PartLog.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/queryToString.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/MergeTree/ActiveDataPartSet.h>
#include <Storages/AlterCommands.h>
#include <Storages/PartitionCommands.h>
#include <Storages/StorageMergeTree.h>
#include <Storages/MergeTree/MergeTreeBlockOutputStream.h>
#include <Storages/MergeTree/DiskSpaceMonitor.h>
#include <Storages/MergeTree/MergeList.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/File.h>
#include <optional>
...
...
dbms/src/Storages/System/StorageSystemParts.cpp
浏览文件 @
1b0d1a44
#include "StorageSystemParts.h"
#include <Common/escapeForFileName.h>
#include <Columns/ColumnString.h>
#include <DataTypes/DataTypeString.h>
...
...
@@ -5,7 +7,6 @@
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDate.h>
#include <DataStreams/OneBlockInputStream.h>
#include <Storages/System/StorageSystemParts.h>
#include <Storages/VirtualColumnUtils.h>
#include <Databases/IDatabase.h>
...
...
@@ -38,6 +39,7 @@ StorageSystemParts::StorageSystemParts(const std::string & name)
{
"data_version"
,
std
::
make_shared
<
DataTypeUInt64
>
()},
{
"primary_key_bytes_in_memory"
,
std
::
make_shared
<
DataTypeUInt64
>
()},
{
"primary_key_bytes_in_memory_allocated"
,
std
::
make_shared
<
DataTypeUInt64
>
()},
{
"is_frozen"
,
std
::
make_shared
<
DataTypeUInt8
>
()},
{
"database"
,
std
::
make_shared
<
DataTypeString
>
()},
{
"table"
,
std
::
make_shared
<
DataTypeString
>
()},
...
...
@@ -96,6 +98,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
columns
[
i
++
]
->
insert
(
static_cast
<
UInt64
>
(
part
->
info
.
getDataVersion
()));
columns
[
i
++
]
->
insert
(
part
->
getIndexSizeInBytes
());
columns
[
i
++
]
->
insert
(
part
->
getIndexSizeInAllocatedBytes
());
columns
[
i
++
]
->
insert
(
part
->
is_frozen
);
columns
[
i
++
]
->
insert
(
info
.
database
);
columns
[
i
++
]
->
insert
(
info
.
table
);
...
...
dbms/src/Storages/System/StorageSystemPartsColumns.cpp
浏览文件 @
1b0d1a44
#include "StorageSystemPartsColumns.h"
#include <Common/escapeForFileName.h>
#include <Columns/ColumnString.h>
#include <DataTypes/DataTypeString.h>
...
...
@@ -5,7 +7,6 @@
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDate.h>
#include <DataStreams/OneBlockInputStream.h>
#include <Storages/System/StorageSystemPartsColumns.h>
#include <Storages/VirtualColumnUtils.h>
#include <Databases/IDatabase.h>
#include <Parsers/queryToString.h>
...
...
dbms/src/Storages/tests/transform_part_zk_nodes.cpp
浏览文件 @
1b0d1a44
#include <Storages/MergeTree/ReplicatedMergeTreePartHeader.h>
#include <Storages/MergeTree/MergeTreeDataPartChecksum.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <Common/Exception.h>
...
...
dbms/tests/queries/0_stateless/00428_partition.reference
浏览文件 @
1b0d1a44
...
...
@@ -15,7 +15,6 @@
88cdc31ded355e7572d68d8cde525d3a shadow/1/data/test/partition_428/19700201_19700201_1_1_0/p.bin
9e688c58a5487b8eaf69c9e1005ad0bf shadow/1/data/test/partition_428/19700102_19700102_2_2_0/primary.idx
b01e3d4df40467db3f1c2d029f59f6a2 shadow/1/data/test/partition_428/19700201_19700201_1_1_0/checksums.txt
b026324c6904b2a9cb4b88d6d61c81d1 shadow/increment.txt
cfcb770c3ecd0990dcceb1bde129e6c6 shadow/1/data/test/partition_428/19700102_19700102_2_2_0/p.bin
e2af3bef1fd129aea73a890ede1e7a30 shadow/1/data/test/partition_428/19700201_19700201_1_1_0/k.bin
e6654eba9e88b001280d3bdd21ccc417 shadow/1/data/test/partition_428/19700102_19700102_2_2_0/checksums.txt
...
...
@@ -36,7 +35,6 @@ e6654eba9e88b001280d3bdd21ccc417 shadow/1/data/test/partition_428/19700102_1970
88cdc31ded355e7572d68d8cde525d3a shadow/1/data/test/partition_428/19700201_19700201_1_1_0/p.bin
9e688c58a5487b8eaf69c9e1005ad0bf shadow/1/data/test/partition_428/19700102_19700102_2_2_0/primary.idx
b01e3d4df40467db3f1c2d029f59f6a2 shadow/1/data/test/partition_428/19700201_19700201_1_1_0/checksums.txt
b026324c6904b2a9cb4b88d6d61c81d1 shadow/increment.txt
cfcb770c3ecd0990dcceb1bde129e6c6 shadow/1/data/test/partition_428/19700102_19700102_2_2_0/p.bin
e2af3bef1fd129aea73a890ede1e7a30 shadow/1/data/test/partition_428/19700201_19700201_1_1_0/k.bin
e6654eba9e88b001280d3bdd21ccc417 shadow/1/data/test/partition_428/19700102_19700102_2_2_0/checksums.txt
...
...
dbms/tests/queries/0_stateless/00428_partition.sh
浏览文件 @
1b0d1a44
...
...
@@ -26,7 +26,8 @@ done
$chl
"ALTER TABLE test.partition_428 FREEZE"
# Do `cd` for consistent output for reference
cd
$ch_dir
&&
find shadow
-type
f
-exec
md5sum
{}
\;
|
sort
# Do not check increment.txt - it can be changed by other tests with FREEZE
cd
$ch_dir
&&
find shadow
-type
f
-exec
md5sum
{}
\;
|
grep
"partition_428"
|
sed
's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g'
|
sort
|
uniq
$chl
"ALTER TABLE test.partition_428 DETACH PARTITION 197001"
$chl
"ALTER TABLE test.partition_428 ATTACH PARTITION 197001"
...
...
@@ -40,7 +41,7 @@ done
$chl
"ALTER TABLE test.partition_428 MODIFY COLUMN v1 Int8"
# Check the backup hasn't changed
cd
$ch_dir
&&
find shadow
-type
f
-exec
md5sum
{}
\;
|
sort
cd
$ch_dir
&&
find shadow
-type
f
-exec
md5sum
{}
\;
|
grep
"partition_428"
|
sed
's!shadow/[0-9]*/data/[a-z0-9_-]*/!shadow/1/data/test/!g'
|
sort
|
uniq
$chl
"OPTIMIZE TABLE test.partition_428"
...
...
dbms/tests/queries/0_stateless/00952_part_frozen_info.reference
0 → 100644
浏览文件 @
1b0d1a44
19701001_1_1_0 0
19701002_2_2_0 0
19701003_3_3_0 0
freeze one
19701001_1_1_0 0
19701002_2_2_0 1
19701003_3_3_0 0
freeze all
19701001_1_1_0 1
19701002_2_2_0 1
19701003_3_3_0 1
1970-10-01 00:00:01
1970-10-02 00:00:01
1970-10-02 00:00:02
1970-10-03 00:00:01
19701001_1_1_0 1
19701002_2_2_0 1
19701002_4_4_0 0
19701003_3_3_0 1
dbms/tests/queries/0_stateless/00952_part_frozen_info.sql
0 → 100644
浏览文件 @
1b0d1a44
DROP
TABLE
IF
EXISTS
part_info
;
CREATE
TABLE
part_info
(
t
DateTime
)
ENGINE
=
MergeTree
PARTITION
BY
toDate
(
t
)
ORDER
BY
(
t
);
INSERT
INTO
part_info
VALUES
(
toDateTime
(
'1970-10-01 00:00:01'
)),
(
toDateTime
(
'1970-10-02 00:00:01'
)),
(
toDateTime
(
'1970-10-03 00:00:01'
));
SELECT
name
,
is_frozen
FROM
system
.
parts
WHERE
`database`
=
currentDatabase
()
AND
`table`
=
'part_info'
;
SELECT
'freeze one'
;
ALTER
TABLE
part_info
FREEZE
PARTITION
toDate
(
'1970-10-02'
);
SELECT
name
,
is_frozen
FROM
system
.
parts
WHERE
`database`
=
currentDatabase
()
AND
`table`
=
'part_info'
;
SELECT
'freeze all'
;
ALTER
TABLE
part_info
FREEZE
;
SELECT
name
,
is_frozen
FROM
system
.
parts
WHERE
`database`
=
currentDatabase
()
AND
`table`
=
'part_info'
;
INSERT
INTO
part_info
VALUES
(
toDateTime
(
'1970-10-02 00:00:02'
));
select
*
from
part_info
order
by
t
;
SELECT
name
,
is_frozen
FROM
system
.
parts
WHERE
`database`
=
currentDatabase
()
AND
`table`
=
'part_info'
;
DROP
TABLE
part_info
;
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录