Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
16bb923d
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
16bb923d
编写于
12月 07, 2018
作者:
A
alexey-milovidov
提交者:
GitHub
12月 07, 2018
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #3769 from yandex/low-cardinality-native-format-conversion
Support insertion into LowCardinality from old native clients.
上级
02e8631e
2bb57ce4
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
170 addition
and
53 deletion
+170
-53
dbms/programs/server/TCPHandler.cpp
dbms/programs/server/TCPHandler.cpp
+17
-0
dbms/src/Columns/ColumnLowCardinality.cpp
dbms/src/Columns/ColumnLowCardinality.cpp
+1
-1
dbms/src/Core/Defines.h
dbms/src/Core/Defines.h
+2
-0
dbms/src/DataStreams/NativeBlockInputStream.cpp
dbms/src/DataStreams/NativeBlockInputStream.cpp
+4
-0
dbms/src/DataTypes/DataTypeLowCardinality.h
dbms/src/DataTypes/DataTypeLowCardinality.h
+9
-0
dbms/src/DataTypes/DataTypeLowCardinalityHelpers.cpp
dbms/src/DataTypes/DataTypeLowCardinalityHelpers.cpp
+137
-0
dbms/src/Functions/IFunction.cpp
dbms/src/Functions/IFunction.cpp
+0
-52
未找到文件。
dbms/programs/server/TCPHandler.cpp
浏览文件 @
16bb923d
...
...
@@ -30,6 +30,7 @@
#include <Storages/StorageMemory.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Core/ExternalTable.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include "TCPHandler.h"
...
...
@@ -361,6 +362,17 @@ void TCPHandler::processInsertQuery(const Settings & global_settings)
/// Send block to the client - table structure.
Block
block
=
state
.
io
.
out
->
getHeader
();
/// Support insert from old clients without low cardinality type.
if
(
client_revision
&&
client_revision
<
DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE
)
{
for
(
auto
&
col
:
block
)
{
col
.
type
=
recursiveRemoveLowCardinality
(
col
.
type
);
col
.
column
=
recursiveRemoveLowCardinality
(
col
.
column
);
}
}
sendData
(
block
);
readData
(
global_settings
);
...
...
@@ -743,8 +755,13 @@ void TCPHandler::initBlockInput()
else
state
.
maybe_compressed_in
=
in
;
Block
header
;
if
(
state
.
io
.
out
)
header
=
state
.
io
.
out
->
getHeader
();
state
.
block_in
=
std
::
make_shared
<
NativeBlockInputStream
>
(
*
state
.
maybe_compressed_in
,
header
,
client_revision
);
}
}
...
...
dbms/src/Columns/ColumnLowCardinality.cpp
浏览文件 @
16bb923d
...
...
@@ -163,7 +163,7 @@ void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, si
auto
*
low_cardinality_src
=
typeid_cast
<
const
ColumnLowCardinality
*>
(
&
src
);
if
(
!
low_cardinality_src
)
throw
Exception
(
"Expected ColumnLowCardinality, got"
+
src
.
getName
(),
ErrorCodes
::
ILLEGAL_COLUMN
);
throw
Exception
(
"Expected ColumnLowCardinality, got
"
+
src
.
getName
(),
ErrorCodes
::
ILLEGAL_COLUMN
);
if
(
&
low_cardinality_src
->
getDictionary
()
==
&
getDictionary
())
{
...
...
dbms/src/Core/Defines.h
浏览文件 @
16bb923d
...
...
@@ -52,6 +52,8 @@
/// (keys will be placed in different buckets and result will not be fully aggregated).
#define DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 54408
#define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE 54405
/// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
#define DBMS_TCP_PROTOCOL_VERSION 54226
...
...
dbms/src/DataStreams/NativeBlockInputStream.cpp
浏览文件 @
16bb923d
...
...
@@ -9,6 +9,7 @@
#include <ext/range.h>
#include <DataStreams/NativeBlockInputStream.h>
#include <DataTypes/DataTypeLowCardinality.h>
namespace
DB
...
...
@@ -152,6 +153,9 @@ Block NativeBlockInputStream::readImpl()
column
.
column
=
std
::
move
(
read_column
);
if
(
server_revision
&&
server_revision
<
DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE
)
column
.
column
=
recursiveLowCardinalityConversion
(
column
.
column
,
column
.
type
,
header
.
getByPosition
(
i
).
type
);
res
.
insert
(
std
::
move
(
column
));
if
(
use_index
)
...
...
dbms/src/DataTypes/DataTypeLowCardinality.h
浏览文件 @
16bb923d
...
...
@@ -164,4 +164,13 @@ private:
/// Returns dictionary type if type is DataTypeLowCardinality, type otherwise.
DataTypePtr
removeLowCardinality
(
const
DataTypePtr
&
type
);
/// Remove LowCardinality recursively from all nested types.
DataTypePtr
recursiveRemoveLowCardinality
(
const
DataTypePtr
&
type
);
/// Remove LowCardinality recursively from all nested columns.
ColumnPtr
recursiveRemoveLowCardinality
(
const
ColumnPtr
&
column
);
/// Convert column of type from_type to type to_type by converting nested LowCardinality columns.
ColumnPtr
recursiveLowCardinalityConversion
(
const
ColumnPtr
&
column
,
const
DataTypePtr
&
from_type
,
const
DataTypePtr
&
to_type
);
}
dbms/src/DataTypes/DataTypeLowCardinalityHelpers.cpp
0 → 100644
浏览文件 @
16bb923d
#include <Columns/ColumnArray.h>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnTuple.h>
#include <Columns/ColumnLowCardinality.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeTuple.h>
namespace
DB
{
namespace
ErrorCodes
{
extern
const
int
ILLEGAL_COLUMN
;
extern
const
int
TYPE_MISMATCH
;
}
DataTypePtr
recursiveRemoveLowCardinality
(
const
DataTypePtr
&
type
)
{
if
(
!
type
)
return
type
;
if
(
const
auto
*
array_type
=
typeid_cast
<
const
DataTypeArray
*>
(
type
.
get
()))
return
std
::
make_shared
<
DataTypeArray
>
(
recursiveRemoveLowCardinality
(
array_type
->
getNestedType
()));
if
(
const
auto
*
tuple_type
=
typeid_cast
<
const
DataTypeTuple
*>
(
type
.
get
()))
{
DataTypes
elements
=
tuple_type
->
getElements
();
for
(
auto
&
element
:
elements
)
element
=
recursiveRemoveLowCardinality
(
element
);
if
(
tuple_type
->
haveExplicitNames
())
return
std
::
make_shared
<
DataTypeTuple
>
(
elements
,
tuple_type
->
getElementNames
());
else
return
std
::
make_shared
<
DataTypeTuple
>
(
elements
);
}
if
(
const
auto
*
low_cardinality_type
=
typeid_cast
<
const
DataTypeLowCardinality
*>
(
type
.
get
()))
return
low_cardinality_type
->
getDictionaryType
();
return
type
;
}
ColumnPtr
recursiveRemoveLowCardinality
(
const
ColumnPtr
&
column
)
{
if
(
!
column
)
return
column
;
if
(
const
auto
*
column_array
=
typeid_cast
<
const
ColumnArray
*>
(
column
.
get
()))
return
ColumnArray
::
create
(
recursiveRemoveLowCardinality
(
column_array
->
getDataPtr
()),
column_array
->
getOffsetsPtr
());
if
(
const
auto
*
column_const
=
typeid_cast
<
const
ColumnConst
*>
(
column
.
get
()))
return
ColumnConst
::
create
(
recursiveRemoveLowCardinality
(
column_const
->
getDataColumnPtr
()),
column_const
->
size
());
if
(
const
auto
*
column_tuple
=
typeid_cast
<
const
ColumnTuple
*>
(
column
.
get
()))
{
Columns
columns
=
column_tuple
->
getColumns
();
for
(
auto
&
element
:
columns
)
element
=
recursiveRemoveLowCardinality
(
element
);
return
ColumnTuple
::
create
(
columns
);
}
if
(
const
auto
*
column_low_cardinality
=
typeid_cast
<
const
ColumnLowCardinality
*>
(
column
.
get
()))
return
column_low_cardinality
->
convertToFullColumn
();
return
column
;
}
ColumnPtr
recursiveLowCardinalityConversion
(
const
ColumnPtr
&
column
,
const
DataTypePtr
&
from_type
,
const
DataTypePtr
&
to_type
)
{
if
(
from_type
->
equals
(
*
to_type
))
return
column
;
if
(
const
auto
*
column_const
=
typeid_cast
<
const
ColumnConst
*>
(
column
.
get
()))
return
ColumnConst
::
create
(
recursiveLowCardinalityConversion
(
column_const
->
getDataColumnPtr
(),
from_type
,
to_type
),
column_const
->
size
());
if
(
const
auto
*
low_cardinality_type
=
typeid_cast
<
const
DataTypeLowCardinality
*>
(
from_type
.
get
()))
{
if
(
to_type
->
equals
(
*
low_cardinality_type
->
getDictionaryType
()))
return
column
->
convertToFullColumnIfLowCardinality
();
}
if
(
const
auto
*
low_cardinality_type
=
typeid_cast
<
const
DataTypeLowCardinality
*>
(
to_type
.
get
()))
{
if
(
from_type
->
equals
(
*
low_cardinality_type
->
getDictionaryType
()))
{
auto
col
=
low_cardinality_type
->
createColumn
();
static_cast
<
ColumnLowCardinality
&>
(
*
col
).
insertRangeFromFullColumn
(
*
column
,
0
,
column
->
size
());
return
std
::
move
(
col
);
}
}
if
(
const
auto
*
from_array_type
=
typeid_cast
<
const
DataTypeArray
*>
(
from_type
.
get
()))
{
if
(
const
auto
*
to_array_type
=
typeid_cast
<
const
DataTypeArray
*>
(
to_type
.
get
()))
{
const
auto
*
column_array
=
typeid_cast
<
const
ColumnArray
*>
(
column
.
get
());
if
(
!
column_array
)
throw
Exception
(
"Unexpected column "
+
column
->
getName
()
+
" for type "
+
from_type
->
getName
(),
ErrorCodes
::
ILLEGAL_COLUMN
);
auto
&
nested_from
=
from_array_type
->
getNestedType
();
auto
&
nested_to
=
to_array_type
->
getNestedType
();
return
ColumnArray
::
create
(
recursiveLowCardinalityConversion
(
column_array
->
getDataPtr
(),
nested_from
,
nested_to
),
column_array
->
getOffsetsPtr
());
}
}
if
(
const
auto
*
from_tuple_type
=
typeid_cast
<
const
DataTypeTuple
*>
(
from_type
.
get
()))
{
if
(
const
auto
*
to_tuple_type
=
typeid_cast
<
const
DataTypeTuple
*>
(
to_type
.
get
()))
{
const
auto
*
column_tuple
=
typeid_cast
<
const
ColumnTuple
*>
(
column
.
get
());
if
(
!
column_tuple
)
throw
Exception
(
"Unexpected column "
+
column
->
getName
()
+
" for type "
+
from_type
->
getName
(),
ErrorCodes
::
ILLEGAL_COLUMN
);
Columns
columns
=
column_tuple
->
getColumns
();
auto
&
from_elements
=
from_tuple_type
->
getElements
();
auto
&
to_elements
=
to_tuple_type
->
getElements
();
for
(
size_t
i
=
0
;
i
<
columns
.
size
();
++
i
)
{
auto
&
element
=
columns
[
i
];
element
=
recursiveLowCardinalityConversion
(
element
,
from_elements
.
at
(
i
),
to_elements
.
at
(
i
));
}
return
ColumnTuple
::
create
(
columns
);
}
}
throw
Exception
(
"Cannot convert: "
+
from_type
->
getName
()
+
" to "
+
to_type
->
getName
(),
ErrorCodes
::
TYPE_MISMATCH
);
}
}
dbms/src/Functions/IFunction.cpp
浏览文件 @
16bb923d
...
...
@@ -103,58 +103,6 @@ void PreparedFunctionImpl::createLowCardinalityResultCache(size_t cache_size)
}
static
DataTypePtr
recursiveRemoveLowCardinality
(
const
DataTypePtr
&
type
)
{
if
(
!
type
)
return
type
;
if
(
const
auto
*
array_type
=
typeid_cast
<
const
DataTypeArray
*>
(
type
.
get
()))
return
std
::
make_shared
<
DataTypeArray
>
(
recursiveRemoveLowCardinality
(
array_type
->
getNestedType
()));
if
(
const
auto
*
tuple_type
=
typeid_cast
<
const
DataTypeTuple
*>
(
type
.
get
()))
{
DataTypes
elements
=
tuple_type
->
getElements
();
for
(
auto
&
element
:
elements
)
element
=
recursiveRemoveLowCardinality
(
element
);
if
(
tuple_type
->
haveExplicitNames
())
return
std
::
make_shared
<
DataTypeTuple
>
(
elements
,
tuple_type
->
getElementNames
());
else
return
std
::
make_shared
<
DataTypeTuple
>
(
elements
);
}
if
(
const
auto
*
low_cardinality_type
=
typeid_cast
<
const
DataTypeLowCardinality
*>
(
type
.
get
()))
return
low_cardinality_type
->
getDictionaryType
();
return
type
;
}
static
ColumnPtr
recursiveRemoveLowCardinality
(
const
ColumnPtr
&
column
)
{
if
(
!
column
)
return
column
;
if
(
const
auto
*
column_array
=
typeid_cast
<
const
ColumnArray
*>
(
column
.
get
()))
return
ColumnArray
::
create
(
recursiveRemoveLowCardinality
(
column_array
->
getDataPtr
()),
column_array
->
getOffsetsPtr
());
if
(
const
auto
*
column_const
=
typeid_cast
<
const
ColumnConst
*>
(
column
.
get
()))
return
ColumnConst
::
create
(
recursiveRemoveLowCardinality
(
column_const
->
getDataColumnPtr
()),
column_const
->
size
());
if
(
const
auto
*
column_tuple
=
typeid_cast
<
const
ColumnTuple
*>
(
column
.
get
()))
{
Columns
columns
=
column_tuple
->
getColumns
();
for
(
auto
&
element
:
columns
)
element
=
recursiveRemoveLowCardinality
(
element
);
return
ColumnTuple
::
create
(
columns
);
}
if
(
const
auto
*
column_low_cardinality
=
typeid_cast
<
const
ColumnLowCardinality
*>
(
column
.
get
()))
return
column_low_cardinality
->
convertToFullColumn
();
return
column
;
}
ColumnPtr
wrapInNullable
(
const
ColumnPtr
&
src
,
const
Block
&
block
,
const
ColumnNumbers
&
args
,
size_t
result
,
size_t
input_rows_count
)
{
ColumnPtr
result_null_map_column
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录