Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
34076005
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
34076005
编写于
11月 10, 2014
作者:
A
Andrey Mironov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
dbms: save all columns in a single ZooKeeper node [#METR-12739]
上级
2798a199
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
93 addition
and
101 deletion
+93
-101
dbms/include/DB/Core/Block.h
dbms/include/DB/Core/Block.h
+0
-1
dbms/include/DB/Storages/ColumnDefault.h
dbms/include/DB/Storages/ColumnDefault.h
+62
-19
dbms/src/Storages/StorageReplicatedMergeTree.cpp
dbms/src/Storages/StorageReplicatedMergeTree.cpp
+31
-81
未找到文件。
dbms/include/DB/Core/Block.h
浏览文件 @
34076005
...
...
@@ -21,7 +21,6 @@ namespace DB
*/
class
Context
;
struct
ColumnDefaults
;
class
Block
{
...
...
dbms/include/DB/Storages/ColumnDefault.h
浏览文件 @
34076005
...
...
@@ -64,50 +64,86 @@ namespace DB
return
lhs
.
type
==
rhs
.
type
&&
queryToString
(
lhs
.
expression
)
==
queryToString
(
rhs
.
expression
);
}
struct
ColumnDefaults
:
public
std
::
unordered_map
<
String
,
ColumnDefault
>
using
ColumnDefaults
=
std
::
unordered_map
<
String
,
ColumnDefault
>
;
template
<
bool
store
>
struct
ColumnsDescription
{
using
std
::
unordered_map
<
String
,
ColumnDefault
>::
unordered_map
;
template
<
typename
T
>
using
by_value_or_cref
=
typename
std
::
conditional
<
store
,
T
,
const
T
&>::
type
;
by_value_or_cref
<
NamesAndTypesList
>
columns
;
by_value_or_cref
<
NamesAndTypesList
>
materialized
;
by_value_or_cref
<
NamesAndTypesList
>
alias
;
by_value_or_cref
<
ColumnDefaults
>
defaults
;
/// @todo implement (de)serialization
String
toString
()
const
{
String
s
;
WriteBufferFromString
buf
{
s
};
writeString
(
"column
default
s format version: 1
\n
"
,
buf
);
writeText
(
size
(),
buf
);
writeString
(
"columns format version: 1
\n
"
,
buf
);
writeText
(
columns
.
size
()
+
materialized
.
size
()
+
alias
.
size
(),
buf
);
writeString
(
" columns:
\n
"
,
buf
);
for
(
const
auto
&
column_default
:
*
this
)
{
writeBackQuotedString
(
column_default
.
first
,
buf
);
writeChar
(
' '
,
buf
);
writeString
(
DB
::
toString
(
column_default
.
second
.
type
),
buf
);
writeChar
(
'\t'
,
buf
);
writeString
(
queryToString
(
column_default
.
second
.
expression
),
buf
);
writeChar
(
'\n'
,
buf
);
}
const
auto
write_columns
=
[
this
,
&
buf
]
(
const
NamesAndTypesList
&
columns
)
{
for
(
const
auto
&
column
:
columns
)
{
const
auto
it
=
defaults
.
find
(
column
.
name
);
writeBackQuotedString
(
column
.
name
,
buf
);
writeChar
(
' '
,
buf
);
writeString
(
column
.
type
->
getName
(),
buf
);
if
(
it
==
std
::
end
(
defaults
))
{
writeChar
(
'\n'
,
buf
);
continue
;
}
else
writeChar
(
'\t'
,
buf
);
writeString
(
DB
::
toString
(
it
->
second
.
type
),
buf
);
writeChar
(
'\t'
,
buf
);
writeString
(
queryToString
(
it
->
second
.
expression
),
buf
);
writeChar
(
'\n'
,
buf
);
}
};
write_columns
(
columns
);
write_columns
(
materialized
);
write_columns
(
alias
);
return
s
;
}
static
ColumnDefaults
parse
(
const
String
&
str
)
{
static
ColumnsDescription
parse
(
const
String
&
str
,
const
DataTypeFactory
&
data_type_factory
)
{
ReadBufferFromString
buf
{
str
};
ColumnDefaults
defaults
{};
assertString
(
"column
default
s format version: 1
\n
"
,
buf
);
assertString
(
"columns format version: 1
\n
"
,
buf
);
size_t
count
{};
readText
(
count
,
buf
);
assertString
(
" columns:
\n
"
,
buf
);
ParserTernaryOperatorExpression
expr_parser
;
ColumnsDescription
result
{};
for
(
size_t
i
=
0
;
i
<
count
;
++
i
)
{
String
column_name
;
readBackQuotedString
(
column_name
,
buf
);
assertString
(
" "
,
buf
);
String
type_name
;
readString
(
type_name
,
buf
);
auto
type
=
data_type_factory
.
get
(
type_name
);
if
(
*
buf
.
position
()
==
'\n'
)
{
assertString
(
"
\n
"
,
buf
);
result
.
columns
.
emplace_back
(
column_name
,
std
::
move
(
type
));
continue
;
}
assertString
(
"
\t
"
,
buf
);
String
default_type_str
;
readString
(
default_type_str
,
buf
);
const
auto
default_type
=
columnDefaultTypeFromString
(
default_type_str
);
...
...
@@ -124,12 +160,19 @@ namespace DB
if
(
!
expr_parser
.
parse
(
begin
,
end
,
default_expr
,
expected
))
throw
Exception
{
"Could not parse default expression"
,
DB
::
ErrorCodes
::
CANNOT_PARSE_TEXT
};
defaults
.
emplace
(
column_name
,
ColumnDefault
{
default_type
,
default_expr
});
if
(
ColumnDefaultType
::
Default
==
default_type
)
result
.
columns
.
emplace_back
(
column_name
,
std
::
move
(
type
));
else
if
(
ColumnDefaultType
::
Materialized
==
default_type
)
result
.
materialized
.
emplace_back
(
column_name
,
std
::
move
(
type
));
else
if
(
ColumnDefaultType
::
Alias
==
default_type
)
result
.
alias
.
emplace_back
(
column_name
,
std
::
move
(
type
));
result
.
defaults
.
emplace
(
column_name
,
ColumnDefault
{
default_type
,
default_expr
});
}
assertEOF
(
buf
);
return
defaults
;
return
result
;
}
};
}
dbms/src/Storages/StorageReplicatedMergeTree.cpp
浏览文件 @
34076005
...
...
@@ -207,13 +207,9 @@ void StorageReplicatedMergeTree::createTableIfNotExists()
zookeeper
->
getDefaultACL
(),
zkutil
::
CreateMode
::
Persistent
));
ops
.
push_back
(
new
zkutil
::
Op
::
Create
(
zookeeper_path
+
"/metadata"
,
metadata
.
str
(),
zookeeper
->
getDefaultACL
(),
zkutil
::
CreateMode
::
Persistent
));
ops
.
push_back
(
new
zkutil
::
Op
::
Create
(
zookeeper_path
+
"/columns"
,
data
.
getColumnsListNonMaterialized
().
toString
(),
zookeeper
->
getDefaultACL
(),
zkutil
::
CreateMode
::
Persistent
));
ops
.
push_back
(
new
zkutil
::
Op
::
Create
(
zookeeper_path
+
"/materialized_columns"
,
data
.
materialized_columns
.
toString
(),
zookeeper
->
getDefaultACL
(),
zkutil
::
CreateMode
::
Persistent
));
ops
.
push_back
(
new
zkutil
::
Op
::
Create
(
zookeeper_path
+
"/alias_columns"
,
data
.
alias_columns
.
toString
(),
zookeeper
->
getDefaultACL
(),
zkutil
::
CreateMode
::
Persistent
));
ops
.
push_back
(
new
zkutil
::
Op
::
Create
(
zookeeper_path
+
"/column_defaults"
,
data
.
column_defaults
.
toString
(),
ops
.
push_back
(
new
zkutil
::
Op
::
Create
(
zookeeper_path
+
"/columns"
,
ColumnsDescription
<
false
>
{
data
.
getColumnsListNonMaterialized
(),
data
.
materialized_columns
,
data
.
alias_columns
,
data
.
column_defaults
}.
toString
(),
zookeeper
->
getDefaultACL
(),
zkutil
::
CreateMode
::
Persistent
));
ops
.
push_back
(
new
zkutil
::
Op
::
Create
(
zookeeper_path
+
"/log"
,
""
,
zookeeper
->
getDefaultACL
(),
zkutil
::
CreateMode
::
Persistent
));
...
...
@@ -261,13 +257,15 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo
assertEOF
(
buf
);
zkutil
::
Stat
stat
;
auto
columns
=
NamesAndTypesList
::
parse
(
zookeeper
->
get
(
zookeeper_path
+
"/columns"
,
&
stat
),
context
.
getDataTypeFactory
());
auto
materialized_columns
=
NamesAndTypesList
::
parse
(
zookeeper
->
get
(
zookeeper_path
+
"/materialized_columns"
,
&
stat
),
context
.
getDataTypeFactory
());
auto
alias_columns
=
NamesAndTypesList
::
parse
(
zookeeper
->
get
(
zookeeper_path
+
"/alias_columns"
,
&
stat
),
context
.
getDataTypeFactory
());
auto
column_defaults
=
ColumnDefaults
::
parse
(
zookeeper
->
get
(
zookeeper_path
+
"/column_defaults"
,
&
stat
));
auto
columns_desc
=
ColumnsDescription
<
true
>::
parse
(
zookeeper
->
get
(
zookeeper_path
+
"/columns"
,
&
stat
),
context
.
getDataTypeFactory
());
auto
&
columns
=
columns_desc
.
columns
;
auto
&
materialized_columns
=
columns_desc
.
materialized
;
auto
&
alias_columns
=
columns_desc
.
alias
;
auto
&
column_defaults
=
columns_desc
.
defaults
;
columns_version
=
stat
.
version
;
if
(
columns
!=
data
.
getColumnsListNonMaterialized
()
||
materialized_columns
!=
data
.
materialized_columns
||
alias_columns
!=
data
.
alias_columns
||
...
...
@@ -424,10 +422,12 @@ void StorageReplicatedMergeTree::createReplica()
LOG_DEBUG
(
log
,
"Copied "
<<
source_queue
.
size
()
<<
" queue entries"
);
}
zookeeper
->
create
(
replica_path
+
"/columns"
,
data
.
getColumnsListNonMaterialized
().
toString
(),
zkutil
::
CreateMode
::
Persistent
);
zookeeper
->
create
(
replica_path
+
"/materialized_columns"
,
data
.
materialized_columns
.
toString
(),
zkutil
::
CreateMode
::
Persistent
);
zookeeper
->
create
(
replica_path
+
"/alias_columns"
,
data
.
alias_columns
.
toString
(),
zkutil
::
CreateMode
::
Persistent
);
zookeeper
->
create
(
replica_path
+
"/column_defaults"
,
data
.
column_defaults
.
toString
(),
zkutil
::
CreateMode
::
Persistent
);
zookeeper
->
create
(
replica_path
+
"/columns"
,
ColumnsDescription
<
false
>
{
data
.
getColumnsListNonMaterialized
(),
data
.
materialized_columns
,
data
.
alias_columns
,
data
.
column_defaults
}.
toString
(),
zkutil
::
CreateMode
::
Persistent
);
}
void
StorageReplicatedMergeTree
::
activateReplica
()
...
...
@@ -649,15 +649,6 @@ void StorageReplicatedMergeTree::checkPartAndAddToZooKeeper(const MergeTreeData:
ops
.
push_back
(
new
zkutil
::
Op
::
Check
(
zookeeper_path
+
"/columns"
,
expected_columns_version
));
ops
.
push_back
(
new
zkutil
::
Op
::
Check
(
zookeeper_path
+
"/materialized_columns"
,
expected_columns_version
));
ops
.
push_back
(
new
zkutil
::
Op
::
Check
(
zookeeper_path
+
"/alias_columns"
,
expected_columns_version
));
ops
.
push_back
(
new
zkutil
::
Op
::
Check
(
zookeeper_path
+
"/column_defaults"
,
expected_columns_version
));
ops
.
push_back
(
new
zkutil
::
Op
::
Create
(
replica_path
+
"/parts/"
+
part_name
,
""
,
...
...
@@ -1615,17 +1606,12 @@ void StorageReplicatedMergeTree::alterThread()
zkutil
::
Stat
stat
;
const
String
columns_str
=
zookeeper
->
get
(
zookeeper_path
+
"/columns"
,
&
stat
,
alter_thread_event
);
const
String
materialized_columns_str
=
zookeeper
->
get
(
zookeeper_path
+
"/materialized_columns"
,
&
stat
,
alter_thread_event
);
const
String
alias_columns_str
=
zookeeper
->
get
(
zookeeper_path
+
"/alias_columns"
,
&
stat
,
alter_thread_event
);
const
String
column_defaults_str
=
zookeeper
->
get
(
zookeeper_path
+
"/column_defaults"
,
&
stat
,
alter_thread_event
);
NamesAndTypesList
columns
=
NamesAndTypesList
::
parse
(
columns_str
,
context
.
getDataTypeFactory
());
NamesAndTypesList
materialized_columns
=
NamesAndTypesList
::
parse
(
materialized_columns_str
,
context
.
getDataTypeFactory
());
NamesAndTypesList
alias_columns
=
NamesAndTypesList
::
parse
(
alias_columns_str
,
context
.
getDataTypeFactory
());
ColumnDefaults
column_defaults
=
ColumnDefaults
::
parse
(
column_defaults_str
);
auto
columns_desc
=
ColumnsDescription
<
true
>::
parse
(
columns_str
,
context
.
getDataTypeFactory
());
auto
&
columns
=
columns_desc
.
columns
;
auto
&
materialized_columns
=
columns_desc
.
materialized
;
auto
&
alias_columns
=
columns_desc
.
alias
;
auto
&
column_defaults
=
columns_desc
.
defaults
;
bool
changed_version
=
(
stat
.
version
!=
columns_version
);
...
...
@@ -1744,10 +1730,7 @@ void StorageReplicatedMergeTree::alterThread()
}
/// Список столбцов для конкретной реплики.
zookeeper
->
set
(
replica_path
+
"/columns"
,
columns
.
toString
());
zookeeper
->
set
(
replica_path
+
"/materialized_columns"
,
materialized_columns
.
toString
());
zookeeper
->
set
(
replica_path
+
"/alias_columns"
,
alias_columns
.
toString
());
zookeeper
->
set
(
replica_path
+
"/column_defaults"
,
column_defaults
.
toString
());
zookeeper
->
set
(
replica_path
+
"/columns"
,
columns_str
);
if
(
changed_version
)
{
...
...
@@ -2389,9 +2372,6 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
NamesAndTypesList
new_alias_columns
;
ColumnDefaults
new_column_defaults
;
String
new_columns_str
;
String
new_materialized_columns_str
;
String
new_alias_columns_str
;
String
new_column_defaults_str
;
int
new_columns_version
;
zkutil
::
Stat
stat
;
...
...
@@ -2409,16 +2389,13 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
new_column_defaults
=
data
.
column_defaults
;
params
.
apply
(
new_columns
,
new_materialized_columns
,
new_alias_columns
,
new_column_defaults
);
new_columns_str
=
new_columns
.
toString
();
new_materialized_columns_str
=
new_materialized_columns
.
toString
();
new_alias_columns_str
=
new_alias_columns
.
toString
();
new_column_defaults_str
=
new_column_defaults
.
toString
();
new_columns_str
=
ColumnsDescription
<
false
>
{
new_columns
,
new_materialized_columns
,
new_alias_columns
,
new_column_defaults
}
.
toString
();
/// Делаем ALTER.
zookeeper
->
set
(
zookeeper_path
+
"/columns"
,
new_columns_str
,
-
1
,
&
stat
);
zookeeper
->
set
(
zookeeper_path
+
"/materialized_columns"
,
new_materialized_columns_str
,
-
1
,
&
stat
);
zookeeper
->
set
(
zookeeper_path
+
"/alias_columns"
,
new_alias_columns_str
,
-
1
,
&
stat
);
zookeeper
->
set
(
zookeeper_path
+
"/column_defaults"
,
new_column_defaults_str
,
-
1
,
&
stat
);
new_columns_version
=
stat
.
version
;
}
...
...
@@ -2446,18 +2423,9 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
while
(
!
shutdown_called
)
{
String
replica_columns_str
;
String
replica_materialized_columns_str
;
String
replica_alias_columns_str
;
String
replica_column_defaults_str
;
/// Реплику могли успеть удалить.
if
(
!
zookeeper
->
tryGet
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/columns"
,
replica_columns_str
,
&
stat
)
||
!
zookeeper
->
tryGet
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/materialized_columns"
,
replica_materialized_columns_str
,
&
stat
)
||
!
zookeeper
->
tryGet
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/alias_columns"
,
replica_alias_columns_str
,
&
stat
)
||
!
zookeeper
->
tryGet
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/column_defaults"
,
replica_column_defaults_str
,
&
stat
))
if
(
!
zookeeper
->
tryGet
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/columns"
,
replica_columns_str
,
&
stat
))
{
LOG_WARNING
(
log
,
replica
<<
" was removed"
);
break
;
...
...
@@ -2465,24 +2433,12 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
int
replica_columns_version
=
stat
.
version
;
if
(
replica_columns_str
==
new_columns_str
&&
replica_materialized_columns_str
==
new_materialized_columns_str
&&
replica_alias_columns_str
==
new_alias_columns_str
&&
replica_column_defaults_str
==
new_column_defaults_str
)
if
(
replica_columns_str
==
new_columns_str
)
break
;
if
(
!
zookeeper
->
exists
(
zookeeper_path
+
"/columns"
,
&
stat
))
throw
Exception
(
zookeeper_path
+
"/columns doesn't exist"
,
ErrorCodes
::
NOT_FOUND_NODE
);
if
(
!
zookeeper
->
exists
(
zookeeper_path
+
"/materialized_columns"
,
&
stat
))
throw
Exception
(
zookeeper_path
+
"/materialized_columns doesn't exist"
,
ErrorCodes
::
NOT_FOUND_NODE
);
if
(
!
zookeeper
->
exists
(
zookeeper_path
+
"/alias_columns"
,
&
stat
))
throw
Exception
(
zookeeper_path
+
"/alias_columns doesn't exist"
,
ErrorCodes
::
NOT_FOUND_NODE
);
if
(
!
zookeeper
->
exists
(
zookeeper_path
+
"/column_defaults"
,
&
stat
))
throw
Exception
(
zookeeper_path
+
"/column_defaults doesn't exist"
,
ErrorCodes
::
NOT_FOUND_NODE
);
if
(
stat
.
version
!=
new_columns_version
)
{
LOG_WARNING
(
log
,
zookeeper_path
+
"/columns changed before ALTER finished; "
...
...
@@ -2490,13 +2446,7 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params,
return
;
}
if
(
!
zookeeper
->
exists
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/columns"
,
&
stat
,
alter_query_event
)
||
!
zookeeper
->
exists
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/materialized_columns"
,
&
stat
,
alter_query_event
)
||
!
zookeeper
->
exists
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/alias_columns"
,
&
stat
,
alter_query_event
)
||
!
zookeeper
->
exists
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/column_defaults"
,
&
stat
,
alter_query_event
))
if
(
!
zookeeper
->
exists
(
zookeeper_path
+
"/replicas/"
+
replica
+
"/columns"
,
&
stat
,
alter_query_event
))
{
LOG_WARNING
(
log
,
replica
<<
" was removed"
);
break
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录