Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
22a8e8ef
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
22a8e8ef
编写于
11月 22, 2019
作者:
N
Nikita Mikhaylov
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
better segmenating + move read functions to avoid linker errors
上级
d495e282
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
40 addition
and
36 deletion
+40
-36
dbms/src/IO/ReadHelpers.cpp
dbms/src/IO/ReadHelpers.cpp
+32
-0
dbms/src/IO/ReadHelpers.h
dbms/src/IO/ReadHelpers.h
+2
-30
dbms/src/Processors/Formats/Impl/CSVRowInputFormat.cpp
dbms/src/Processors/Formats/Impl/CSVRowInputFormat.cpp
+3
-4
dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
...src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
+3
-2
未找到文件。
dbms/src/IO/ReadHelpers.cpp
浏览文件 @
22a8e8ef
...
...
@@ -1053,4 +1053,36 @@ void skipToUnescapedNextLineOrEOF(ReadBuffer & buf)
}
}
/// TODO (akuzm) - write comments for this and next function.
void
saveUpToPosition
(
ReadBuffer
&
in
,
DB
::
Memory
<>
&
memory
,
char
*
current
)
{
assert
(
current
>=
in
.
position
());
assert
(
current
<=
in
.
buffer
().
end
());
const
int
old_bytes
=
memory
.
size
();
const
int
additional_bytes
=
current
-
in
.
position
();
const
int
new_bytes
=
old_bytes
+
additional_bytes
;
/// There are no new bytes to add to memory.
/// No need to do extra stuff.
if
(
new_bytes
==
0
)
return
;
memory
.
resize
(
new_bytes
);
memcpy
(
memory
.
data
()
+
old_bytes
,
in
.
position
(),
additional_bytes
);
in
.
position
()
=
current
;
}
bool
loadAtPosition
(
ReadBuffer
&
in
,
DB
::
Memory
<>
&
memory
,
char
*
&
current
)
{
assert
(
current
<=
in
.
buffer
().
end
());
if
(
current
<
in
.
buffer
().
end
())
return
true
;
saveUpToPosition
(
in
,
memory
,
current
);
bool
loaded_more
=
!
in
.
eof
();
assert
(
in
.
position
()
==
in
.
buffer
().
begin
());
current
=
in
.
position
();
return
loaded_more
;
}
}
dbms/src/IO/ReadHelpers.h
浏览文件 @
22a8e8ef
...
...
@@ -925,35 +925,7 @@ return std::make_unique<TReadBuffer>(args...);
}
/// TODO (akuzm) - write comments for this and next function.
void
saveUpToPosition
(
ReadBuffer
&
in
,
DB
::
Memory
<>
&
memory
,
char
*
current
)
{
assert
(
current
>=
in
.
position
());
assert
(
current
<=
in
.
buffer
().
end
());
const
int
old_bytes
=
memory
.
size
();
const
int
additional_bytes
=
current
-
in
.
position
();
const
int
new_bytes
=
old_bytes
+
additional_bytes
;
/// There are no new bytes to add to memory.
/// No need to do extra stuff.
if
(
new_bytes
==
0
)
return
;
memory
.
resize
(
new_bytes
);
memcpy
(
memory
.
data
()
+
old_bytes
,
in
.
position
(),
additional_bytes
);
in
.
position
()
=
current
;
}
bool
loadAtPosition
(
ReadBuffer
&
in
,
DB
::
Memory
<>
&
memory
,
char
*
&
current
)
{
assert
(
current
<=
in
.
buffer
().
end
());
if
(
current
<
in
.
buffer
().
end
())
return
true
;
saveUpToPosition
(
in
,
memory
,
current
);
bool
loaded_more
=
!
in
.
eof
();
assert
(
in
.
position
()
==
in
.
buffer
().
begin
());
current
=
in
.
position
();
return
loaded_more
;
}
void
saveUpToPosition
(
ReadBuffer
&
in
,
DB
::
Memory
<>
&
memory
,
char
*
current
);
bool
loadAtPosition
(
ReadBuffer
&
in
,
DB
::
Memory
<>
&
memory
,
char
*
&
current
);
}
dbms/src/Processors/Formats/Impl/CSVRowInputFormat.cpp
浏览文件 @
22a8e8ef
...
...
@@ -424,8 +424,6 @@ void registerInputFormatProcessorCSV(FormatFactory & factory)
bool
fileSegmentationEngineCSVImpl
(
ReadBuffer
&
in
,
DB
::
Memory
<>
&
memory
,
size_t
min_chunk_size
)
{
skipWhitespacesAndTabs
(
in
);
char
*
pos
=
in
.
position
();
bool
quotes
=
false
;
bool
need_more_data
=
true
;
...
...
@@ -474,8 +472,9 @@ bool fileSegmentationEngineCSVImpl(ReadBuffer & in, DB::Memory<> & memory, size_
}
}
}
loadAtPosition
(
in
,
memory
,
pos
);
return
true
;
saveUpToPosition
(
in
,
memory
,
pos
);
return
loadAtPosition
(
in
,
memory
,
pos
);
}
void
registerFileSegmentationEngineCSV
(
FormatFactory
&
factory
)
...
...
dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
浏览文件 @
22a8e8ef
...
...
@@ -325,8 +325,9 @@ bool fileSegmentationEngineJSONEachRowImpl(ReadBuffer & in, DB::Memory<> & memor
}
}
}
loadAtPosition
(
in
,
memory
,
pos
)
return
true
;
saveUpToPosition
(
in
,
memory
,
pos
);
return
loadAtPosition
(
in
,
memory
,
pos
);
}
void
registerFileSegmentationEngineJSONEachRow
(
FormatFactory
&
factory
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录