Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
cde504eb
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
cde504eb
编写于
5月 29, 2018
作者:
A
Aljoscha Krettek
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[FLINK-9470] Allow querying the key in KeyedProcessFunction
上级
53e66576
变更
6
显示空白变更内容
内联
并排
Showing
6 changed file
with
121 addition
and
0 deletion
+121
-0
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/KeyedProcessFunction.java
...e/flink/streaming/api/functions/KeyedProcessFunction.java
+6
-0
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/KeyedBroadcastProcessFunction.java
...aming/api/functions/co/KeyedBroadcastProcessFunction.java
+7
-0
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java
...e/flink/streaming/api/operators/KeyedProcessOperator.java
+6
-0
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperator.java
...eaming/api/operators/co/CoBroadcastWithKeyedOperator.java
+7
-0
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/KeyedProcessOperatorTest.java
...ink/streaming/api/operators/KeyedProcessOperatorTest.java
+44
-0
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperatorTest.java
...ng/api/operators/co/CoBroadcastWithKeyedOperatorTest.java
+51
-0
未找到文件。
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/KeyedProcessFunction.java
浏览文件 @
cde504eb
...
...
@@ -110,6 +110,11 @@ public abstract class KeyedProcessFunction<K, I, O> extends AbstractRichFunction
* @param value The record to emit.
*/
public
abstract
<
X
>
void
output
(
OutputTag
<
X
>
outputTag
,
X
value
);
/**
* Get key of the element being processed.
*/
public
abstract
K
getCurrentKey
();
}
/**
...
...
@@ -124,6 +129,7 @@ public abstract class KeyedProcessFunction<K, I, O> extends AbstractRichFunction
/**
* Get key of the firing timer.
*/
@Override
public
abstract
K
getCurrentKey
();
}
...
...
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/co/KeyedBroadcastProcessFunction.java
浏览文件 @
cde504eb
...
...
@@ -158,6 +158,12 @@ public abstract class KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> extends B
* A {@link TimerService} for querying time and registering timers.
*/
public
abstract
TimerService
timerService
();
/**
* Get key of the element being processed.
*/
public
abstract
KS
getCurrentKey
();
}
/**
...
...
@@ -174,6 +180,7 @@ public abstract class KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> extends B
/**
* Get the key of the firing timer.
*/
@Override
public
abstract
KS
getCurrentKey
();
}
}
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/KeyedProcessOperator.java
浏览文件 @
cde504eb
...
...
@@ -131,6 +131,12 @@ public class KeyedProcessOperator<K, IN, OUT>
output
.
collect
(
outputTag
,
new
StreamRecord
<>(
value
,
element
.
getTimestamp
()));
}
@Override
@SuppressWarnings
(
"unchecked"
)
public
K
getCurrentKey
()
{
return
(
K
)
KeyedProcessOperator
.
this
.
getCurrentKey
();
}
}
private
class
OnTimerContextImpl
extends
KeyedProcessFunction
<
K
,
IN
,
OUT
>.
OnTimerContext
{
...
...
flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperator.java
浏览文件 @
cde504eb
...
...
@@ -288,6 +288,13 @@ public class CoBroadcastWithKeyedOperator<KS, IN1, IN2, OUT>
}
return
state
;
}
@Override
@SuppressWarnings
(
"unchecked"
)
public
KS
getCurrentKey
()
{
return
(
KS
)
CoBroadcastWithKeyedOperator
.
this
.
getCurrentKey
();
}
}
private
class
OnTimerContextImpl
extends
KeyedBroadcastProcessFunction
<
KS
,
IN1
,
IN2
,
OUT
>.
OnTimerContext
{
...
...
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/KeyedProcessOperatorTest.java
浏览文件 @
cde504eb
...
...
@@ -23,6 +23,7 @@ import org.apache.flink.api.common.state.ValueStateDescriptor;
import
org.apache.flink.api.common.typeinfo.BasicTypeInfo
;
import
org.apache.flink.api.common.typeutils.base.IntSerializer
;
import
org.apache.flink.api.java.functions.KeySelector
;
import
org.apache.flink.api.java.tuple.Tuple2
;
import
org.apache.flink.runtime.checkpoint.OperatorSubtaskState
;
import
org.apache.flink.streaming.api.TimeDomain
;
import
org.apache.flink.streaming.api.TimerService
;
...
...
@@ -43,6 +44,7 @@ import org.junit.rules.ExpectedException;
import
java.util.concurrent.ConcurrentLinkedQueue
;
import
static
org
.
junit
.
Assert
.
assertEquals
;
import
static
org
.
junit
.
Assert
.
assertTrue
;
/**
* Tests {@link KeyedProcessOperator}.
...
...
@@ -52,6 +54,48 @@ public class KeyedProcessOperatorTest extends TestLogger {
@Rule
public
ExpectedException
expectedException
=
ExpectedException
.
none
();
@Test
public
void
testKeyQuerying
()
throws
Exception
{
class
KeyQueryingProcessFunction
extends
KeyedProcessFunction
<
Integer
,
Tuple2
<
Integer
,
String
>,
String
>
{
@Override
public
void
processElement
(
Tuple2
<
Integer
,
String
>
value
,
Context
ctx
,
Collector
<
String
>
out
)
throws
Exception
{
assertTrue
(
"Did not get expected key."
,
ctx
.
getCurrentKey
().
equals
(
value
.
f0
));
// we check that we receive this output, to ensure that the assert was actually checked
out
.
collect
(
value
.
f1
);
}
}
KeyedProcessOperator
<
Integer
,
Tuple2
<
Integer
,
String
>,
String
>
operator
=
new
KeyedProcessOperator
<>(
new
KeyQueryingProcessFunction
());
try
(
OneInputStreamOperatorTestHarness
<
Tuple2
<
Integer
,
String
>,
String
>
testHarness
=
new
KeyedOneInputStreamOperatorTestHarness
<>(
operator
,
(
in
)
->
in
.
f0
,
BasicTypeInfo
.
INT_TYPE_INFO
))
{
testHarness
.
setup
();
testHarness
.
open
();
testHarness
.
processElement
(
new
StreamRecord
<>(
Tuple2
.
of
(
5
,
"5"
),
12L
));
testHarness
.
processElement
(
new
StreamRecord
<>(
Tuple2
.
of
(
42
,
"42"
),
13L
));
ConcurrentLinkedQueue
<
Object
>
expectedOutput
=
new
ConcurrentLinkedQueue
<>();
expectedOutput
.
add
(
new
StreamRecord
<>(
"5"
,
12L
));
expectedOutput
.
add
(
new
StreamRecord
<>(
"42"
,
13L
));
TestHarnessUtil
.
assertOutputEquals
(
"Output was not correct."
,
expectedOutput
,
testHarness
.
getOutput
());
}
}
@Test
public
void
testTimestampAndWatermarkQuerying
()
throws
Exception
{
...
...
flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/co/CoBroadcastWithKeyedOperatorTest.java
浏览文件 @
cde504eb
...
...
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.state.ValueStateDescriptor;
import
org.apache.flink.api.common.typeinfo.BasicTypeInfo
;
import
org.apache.flink.api.common.typeinfo.TypeInformation
;
import
org.apache.flink.api.java.functions.KeySelector
;
import
org.apache.flink.api.java.tuple.Tuple2
;
import
org.apache.flink.runtime.checkpoint.OperatorSubtaskState
;
import
org.apache.flink.runtime.state.KeyedStateFunction
;
import
org.apache.flink.streaming.api.functions.co.KeyedBroadcastProcessFunction
;
...
...
@@ -70,6 +71,56 @@ public class CoBroadcastWithKeyedOperatorTest {
BasicTypeInfo
.
INT_TYPE_INFO
);
@Test
public
void
testKeyQuerying
()
throws
Exception
{
class
KeyQueryingProcessFunction
extends
KeyedBroadcastProcessFunction
<
Integer
,
Tuple2
<
Integer
,
String
>,
String
,
String
>
{
@Override
public
void
processElement
(
Tuple2
<
Integer
,
String
>
value
,
ReadOnlyContext
ctx
,
Collector
<
String
>
out
)
throws
Exception
{
assertTrue
(
"Did not get expected key."
,
ctx
.
getCurrentKey
().
equals
(
value
.
f0
));
// we check that we receive this output, to ensure that the assert was actually checked
out
.
collect
(
value
.
f1
);
}
@Override
public
void
processBroadcastElement
(
String
value
,
Context
ctx
,
Collector
<
String
>
out
)
throws
Exception
{
}
}
CoBroadcastWithKeyedOperator
<
Integer
,
Tuple2
<
Integer
,
String
>,
String
,
String
>
operator
=
new
CoBroadcastWithKeyedOperator
<>(
new
KeyQueryingProcessFunction
(),
Collections
.
emptyList
());
try
(
TwoInputStreamOperatorTestHarness
<
Tuple2
<
Integer
,
String
>,
String
,
String
>
testHarness
=
new
KeyedTwoInputStreamOperatorTestHarness
<>(
operator
,
(
in
)
->
in
.
f0
,
null
,
BasicTypeInfo
.
INT_TYPE_INFO
))
{
testHarness
.
setup
();
testHarness
.
open
();
testHarness
.
processElement1
(
new
StreamRecord
<>(
Tuple2
.
of
(
5
,
"5"
),
12L
));
testHarness
.
processElement1
(
new
StreamRecord
<>(
Tuple2
.
of
(
42
,
"42"
),
13L
));
ConcurrentLinkedQueue
<
Object
>
expectedOutput
=
new
ConcurrentLinkedQueue
<>();
expectedOutput
.
add
(
new
StreamRecord
<>(
"5"
,
12L
));
expectedOutput
.
add
(
new
StreamRecord
<>(
"42"
,
13L
));
TestHarnessUtil
.
assertOutputEquals
(
"Output was not correct."
,
expectedOutput
,
testHarness
.
getOutput
());
}
}
/** Test the iteration over the keyed state on the broadcast side. */
@Test
public
void
testAccessToKeyedStateIt
()
throws
Exception
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录