Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
2dot5
ClickHouse
提交
a8ce7530
C
ClickHouse
项目概览
2dot5
/
ClickHouse
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
C
ClickHouse
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
a8ce7530
编写于
5月 28, 2019
作者:
C
comunodi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Put keys in result block only if value exists
上级
b3d8ec3e
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
24 addition
and
46 deletion
+24
-46
dbms/src/Dictionaries/RedisBlockInputStream.cpp
dbms/src/Dictionaries/RedisBlockInputStream.cpp
+21
-20
dbms/src/Dictionaries/RedisDictionarySource.cpp
dbms/src/Dictionaries/RedisDictionarySource.cpp
+1
-25
dbms/src/Dictionaries/RedisDictionarySource.h
dbms/src/Dictionaries/RedisDictionarySource.h
+2
-1
未找到文件。
dbms/src/Dictionaries/RedisBlockInputStream.cpp
浏览文件 @
a8ce7530
...
...
@@ -36,6 +36,7 @@ namespace DB
extern
const
int
TYPE_MISMATCH
;
extern
const
int
LOGICAL_ERROR
;
extern
const
int
LIMIT_EXCEEDED
;
extern
const
int
SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT
;
}
...
...
@@ -59,11 +60,8 @@ namespace DB
bool
isNull
(
const
Poco
::
Redis
::
RedisType
::
Ptr
&
value
)
{
if
(
value
.
isNull
())
return
true
;
if
(
value
->
isBulkString
())
return
static_cast
<
const
Poco
::
Redis
::
Type
<
Poco
::
Redis
::
BulkString
>
*>
(
value
.
get
())
->
value
().
isNull
();
return
false
;
return
value
->
isBulkString
()
&&
static_cast
<
const
Poco
::
Redis
::
Type
<
Poco
::
Redis
::
BulkString
>
*>
(
value
.
get
())
->
value
().
isNull
();
}
std
::
string
getStringOrThrow
(
const
Poco
::
Redis
::
RedisType
::
Ptr
&
value
,
const
std
::
string
&
column_name
)
...
...
@@ -158,10 +156,6 @@ namespace DB
if
(
all_read
)
return
{};
for
(
size_t
i
=
0
;
i
<
5
;
++
i
)
if
(
description
.
sample_block
.
columns
()
>=
i
+
1
)
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
description
.
sample_block
.
getByPosition
(
i
).
dumpStructure
());
const
size_t
size
=
description
.
sample_block
.
columns
();
MutableColumns
columns
(
description
.
sample_block
.
columns
());
...
...
@@ -220,6 +214,7 @@ namespace DB
commandForValues
.
addRedisType
(
secondary_key
);
}
// FIXME: fix insert
Poco
::
Redis
::
Array
values
=
client
->
execute
<
Poco
::
Redis
::
Array
>
(
commandForValues
);
for
(
const
auto
&
value
:
values
)
{
...
...
@@ -235,10 +230,10 @@ namespace DB
}
else
{
size_t
num_rows
=
0
;
Poco
::
Redis
::
Command
commandForValues
(
"MGET"
);
while
(
num_rows
<
max_block_size
)
// keys.size() > 0
for
(
size_t
num_rows
=
0
;
num_rows
<
max_block_size
;
++
num_rows
)
{
if
(
cursor
>=
keys
.
size
())
{
...
...
@@ -247,23 +242,29 @@ namespace DB
}
const
auto
&
key
=
*
(
keys
.
begin
()
+
cursor
);
insertValueByIdx
(
0
,
key
);
commandForValues
.
addRedisType
(
key
);
++
num_rows
;
++
cursor
;
}
if
(
num_rows
==
0
)
return
{};
Poco
::
Redis
::
Array
values
=
client
->
execute
<
Poco
::
Redis
::
Array
>
(
commandForValues
);
for
(
const
auto
&
value
:
values
)
if
(
commandForValues
.
size
()
!=
values
.
size
()
+
1
)
throw
Exception
{
"Inconsistent sizes of keys and values in Redis request"
,
ErrorCodes
::
SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT
};
for
(
size_t
num_rows
=
0
;
num_rows
<
values
.
size
();
++
num_rows
)
{
if
(
isNull
(
value
))
const
auto
&
key
=
*
(
keys
.
begin
()
+
cursor
-
num_rows
-
1
);
const
auto
&
value
=
*
(
values
.
begin
()
+
values
.
size
()
-
num_rows
-
1
);
if
(
value
.
isNull
())
{
insertValueByIdx
(
0
,
key
);
insertDefaultValue
(
*
columns
[
1
],
*
description
.
sample_block
.
getByPosition
(
1
).
column
);
else
}
else
if
(
!
isNull
(
value
))
// null string means 'no value for requested key'
{
insertValueByIdx
(
0
,
key
);
insertValueByIdx
(
1
,
value
);
}
}
}
...
...
dbms/src/Dictionaries/RedisDictionarySource.cpp
浏览文件 @
a8ce7530
...
...
@@ -52,9 +52,6 @@ namespace DB
# include <ext/enumerate.h>
# include "RedisBlockInputStream.h"
# include "Poco/Logger.h"
# include "common/logger_useful.h"
namespace
DB
{
...
...
@@ -84,18 +81,13 @@ namespace DB
,
sample_block
{
sample_block
}
,
client
{
std
::
make_shared
<
Poco
::
Redis
::
Client
>
(
host
,
port
)}
{
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"in ctor"
);
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
dict_struct
.
attributes
.
size
());
if
(
dict_struct
.
attributes
.
size
()
!=
1
)
throw
Exception
{
"Invalid number of non key columns for Redis source: "
+
DB
::
toString
(
dict_struct
.
attributes
.
size
())
+
", expected 1"
,
ErrorCodes
::
INVALID_CONFIG_PARAMETER
};
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"After first check"
);
if
(
storage_type
==
RedisStorageType
::
HASH_MAP
)
{
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"SET STORAGE_TYPE"
);
if
(
!
dict_struct
.
key
.
has_value
())
throw
Exception
{
"Redis source with storage type
\'
hash_map
\'
must have key"
,
ErrorCodes
::
INVALID_CONFIG_PARAMETER
};
...
...
@@ -105,11 +97,8 @@ namespace DB
// suppose key[0] is primary key, key[1] is secondary key
}
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"After second check"
);
if
(
db_index
!=
0
)
{
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"SET DB_INDEX"
);
Poco
::
Redis
::
Command
command
(
"SELECT"
);
command
<<
static_cast
<
Int64
>
(
db_index
);
std
::
string
reply
=
client
->
execute
<
std
::
string
>
(
command
);
...
...
@@ -117,8 +106,6 @@ namespace DB
throw
Exception
{
"Selecting db with index "
+
DB
::
toString
(
db_index
)
+
" failed with reason "
+
reply
,
ErrorCodes
::
CANNOT_SELECT
};
}
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"After third check"
);
}
...
...
@@ -154,17 +141,11 @@ namespace DB
BlockInputStreamPtr
RedisDictionarySource
::
loadAll
()
{
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"Redis in loadAll"
);
Poco
::
Redis
::
Command
command_for_keys
(
"KEYS"
);
command_for_keys
<<
"*"
;
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"Command for keys: "
+
command_for_keys
.
toString
());
Poco
::
Redis
::
Array
keys
=
client
->
execute
<
Poco
::
Redis
::
Array
>
(
command_for_keys
);
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"Command for keys executed"
);
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"KEYS: "
+
keys
.
toString
());
if
(
storage_type
==
RedisStorageType
::
HASH_MAP
&&
dict_struct
.
key
->
size
()
==
2
)
{
Poco
::
Redis
::
Array
hkeys
;
...
...
@@ -173,13 +154,12 @@ namespace DB
Poco
::
Redis
::
Command
command_for_secondary_keys
(
"HKEYS"
);
command_for_secondary_keys
.
addRedisType
(
key
);
Poco
::
Redis
::
Array
reply_for_primary_key
=
client
->
execute
<
Poco
::
Redis
::
Array
>
(
command_for_secondary_keys
);
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"Command for hkeys executed"
);
Poco
::
SharedPtr
<
Poco
::
Redis
::
Array
>
primary_with_secondary
;
primary_with_secondary
->
addRedisType
(
key
);
for
(
const
auto
&
secondary_key
:
reply_for_primary_key
)
primary_with_secondary
->
addRedisType
(
secondary_key
);
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"HKEYS: "
+
primary_with_secondary
->
toString
());
hkeys
.
add
(
*
primary_with_secondary
);
}
keys
=
hkeys
;
...
...
@@ -191,8 +171,6 @@ namespace DB
BlockInputStreamPtr
RedisDictionarySource
::
loadIds
(
const
std
::
vector
<
UInt64
>
&
ids
)
{
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"Redis in loadIds"
);
if
(
storage_type
!=
RedisStorageType
::
SIMPLE
)
throw
Exception
{
"Cannot use loadIds with
\'
simple
\'
storage type"
,
ErrorCodes
::
UNSUPPORTED_METHOD
};
...
...
@@ -204,8 +182,6 @@ namespace DB
for
(
UInt64
id
:
ids
)
keys
<<
DB
::
toString
(
id
);
LOG_INFO
(
&
Logger
::
get
(
"Redis"
),
"KEYS: "
+
keys
.
toString
());
return
std
::
make_shared
<
RedisBlockInputStream
>
(
client
,
std
::
move
(
keys
),
sample_block
,
max_block_size
);
}
...
...
dbms/src/Dictionaries/RedisDictionarySource.h
浏览文件 @
a8ce7530
...
...
@@ -2,6 +2,7 @@
#include <Common/config.h>
#include <Core/Block.h>
#if USE_POCO_REDIS
# include "DictionaryStructure.h"
...
...
@@ -95,7 +96,7 @@ namespace DB
const
DictionaryStructure
dict_struct
;
const
std
::
string
host
;
const
UInt16
port
;
const
UInt8
db_index
;
// [0..15]
const
UInt8
db_index
;
const
RedisStorageType
::
Id
storage_type
;
Block
sample_block
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录