Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
73493335
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
73493335
编写于
5月 12, 2015
作者:
S
Stephan Ewen
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[FLINK-1959] [runtime] Support accumulators in chained functions after a non-UDF operation
上级
cf4f22ea
变更
2
显示空白变更内容
内联
并排
Showing
2 changed file
with
37 addition
and
23 deletion
+37
-23
flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
...ache/flink/api/common/accumulators/AccumulatorHelper.java
+8
-6
flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
...a/org/apache/flink/runtime/operators/RegularPactTask.java
+29
-17
未找到文件。
flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
浏览文件 @
73493335
...
@@ -40,7 +40,8 @@ public class AccumulatorHelper {
...
@@ -40,7 +40,8 @@ public class AccumulatorHelper {
if
(
ownAccumulator
==
null
)
{
if
(
ownAccumulator
==
null
)
{
// Take over counter from chained task
// Take over counter from chained task
target
.
put
(
otherEntry
.
getKey
(),
otherEntry
.
getValue
());
target
.
put
(
otherEntry
.
getKey
(),
otherEntry
.
getValue
());
}
else
{
}
else
{
// Both should have the same type
// Both should have the same type
AccumulatorHelper
.
compareAccumulatorTypes
(
otherEntry
.
getKey
(),
AccumulatorHelper
.
compareAccumulatorTypes
(
otherEntry
.
getKey
(),
ownAccumulator
.
getClass
(),
otherEntry
.
getValue
().
getClass
());
ownAccumulator
.
getClass
(),
otherEntry
.
getValue
().
getClass
());
...
@@ -122,13 +123,14 @@ public class AccumulatorHelper {
...
@@ -122,13 +123,14 @@ public class AccumulatorHelper {
return
builder
.
toString
();
return
builder
.
toString
();
}
}
public
static
void
resetAndClearAccumulators
(
public
static
void
resetAndClearAccumulators
(
Map
<
String
,
Accumulator
<?,
?>>
accumulators
)
{
Map
<
String
,
Accumulator
<?,
?>>
accumulators
)
{
if
(
accumulators
!=
null
)
{
for
(
Map
.
Entry
<
String
,
Accumulator
<?,
?>>
entry
:
accumulators
.
entrySet
())
{
for
(
Map
.
Entry
<
String
,
Accumulator
<?,
?>>
entry
:
accumulators
.
entrySet
())
{
entry
.
getValue
().
resetLocal
();
entry
.
getValue
().
resetLocal
();
}
}
accumulators
.
clear
();
accumulators
.
clear
();
}
}
}
public
static
Map
<
String
,
Accumulator
<?,
?>>
copy
(
final
Map
<
String
,
Accumulator
<?,
public
static
Map
<
String
,
Accumulator
<?,
?>>
copy
(
final
Map
<
String
,
Accumulator
<?,
?>>
accumulators
)
{
?>>
accumulators
)
{
...
...
flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
浏览文件 @
73493335
...
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.distributions.DataDistribution;
...
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.distributions.DataDistribution;
import
org.apache.flink.api.common.functions.GroupCombineFunction
;
import
org.apache.flink.api.common.functions.GroupCombineFunction
;
import
org.apache.flink.api.common.functions.Function
;
import
org.apache.flink.api.common.functions.Function
;
import
org.apache.flink.api.common.functions.Partitioner
;
import
org.apache.flink.api.common.functions.Partitioner
;
import
org.apache.flink.api.common.functions.RuntimeContext
;
import
org.apache.flink.api.common.functions.util.FunctionUtils
;
import
org.apache.flink.api.common.functions.util.FunctionUtils
;
import
org.apache.flink.api.common.typeutils.TypeComparator
;
import
org.apache.flink.api.common.typeutils.TypeComparator
;
import
org.apache.flink.api.common.typeutils.TypeComparatorFactory
;
import
org.apache.flink.api.common.typeutils.TypeComparatorFactory
;
...
@@ -70,6 +71,7 @@ import org.slf4j.LoggerFactory;
...
@@ -70,6 +71,7 @@ import org.slf4j.LoggerFactory;
import
java.io.IOException
;
import
java.io.IOException
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
...
@@ -508,14 +510,13 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
...
@@ -508,14 +510,13 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// JobManager. close() has been called earlier for all involved UDFs
// JobManager. close() has been called earlier for all involved UDFs
// (using this.stub.close() and closeChainedTasks()), so UDFs can no longer
// (using this.stub.close() and closeChainedTasks()), so UDFs can no longer
// modify accumulators;
// modify accumulators;
if
(
this
.
stub
!=
null
)
{
// collect the counters from the stub
// collect the counters from the udf in the core driver
if
(
FunctionUtils
.
getFunctionRuntimeContext
(
this
.
stub
,
this
.
runtimeUdfContext
)
!=
null
)
{
Map
<
String
,
Accumulator
<?,
?>>
accumulators
=
Map
<
String
,
Accumulator
<?,
?>>
accumulators
=
FunctionUtils
.
getFunctionRuntimeContext
(
this
.
stub
,
this
.
runtimeUdfContext
).
getAllAccumulators
();
FunctionUtils
.
getFunctionRuntimeContext
(
this
.
stub
,
this
.
runtimeUdfContext
).
getAllAccumulators
();
RegularPactTask
.
reportAndClearAccumulators
(
getEnvironment
(),
accumulators
,
this
.
chainedTasks
);
}
// collect accumulators from chained tasks and report them
}
reportAndClearAccumulators
(
getEnvironment
(),
accumulators
,
this
.
chainedTasks
);
}
}
catch
(
Exception
ex
)
{
catch
(
Exception
ex
)
{
// close the input, but do not report any exceptions, since we already have another root cause
// close the input, but do not report any exceptions, since we already have another root cause
...
@@ -572,16 +573,25 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
...
@@ -572,16 +573,25 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// We can merge here the accumulators from the stub and the chained
// We can merge here the accumulators from the stub and the chained
// tasks. Type conflicts can occur here if counters with same name but
// tasks. Type conflicts can occur here if counters with same name but
// different type were used.
// different type were used.
if
(!
chainedTasks
.
isEmpty
())
{
if
(
accumulators
==
null
)
{
accumulators
=
new
HashMap
<
String
,
Accumulator
<?,
?>>();
}
for
(
ChainedDriver
<?,
?>
chainedTask
:
chainedTasks
)
{
for
(
ChainedDriver
<?,
?>
chainedTask
:
chainedTasks
)
{
if
(
FunctionUtils
.
getFunctionRuntimeContext
(
chainedTask
.
getStub
(),
null
)
!=
null
)
{
RuntimeContext
rc
=
FunctionUtils
.
getFunctionRuntimeContext
(
chainedTask
.
getStub
(),
null
);
Map
<
String
,
Accumulator
<?,
?>>
chainedAccumulators
=
if
(
rc
!=
null
)
{
FunctionUtils
.
getFunctionRuntimeContext
(
chainedTask
.
getStub
(),
null
).
getAllAccumulators
();
Map
<
String
,
Accumulator
<?,
?>>
chainedAccumulators
=
rc
.
getAllAccumulators
();
if
(
chainedAccumulators
!=
null
)
{
AccumulatorHelper
.
mergeInto
(
accumulators
,
chainedAccumulators
);
AccumulatorHelper
.
mergeInto
(
accumulators
,
chainedAccumulators
);
}
}
}
}
}
}
// Don't report if the UDF didn't collect any accumulators
// Don't report if the UDF didn't collect any accumulators
if
(
accumulators
.
size
()
==
0
)
{
if
(
accumulators
==
null
||
accumulators
.
size
()
==
0
)
{
return
;
return
;
}
}
...
@@ -592,9 +602,11 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
...
@@ -592,9 +602,11 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// (e.g. in iterations) and we don't want to count twice. This may not be
// (e.g. in iterations) and we don't want to count twice. This may not be
// done before sending
// done before sending
AccumulatorHelper
.
resetAndClearAccumulators
(
accumulators
);
AccumulatorHelper
.
resetAndClearAccumulators
(
accumulators
);
for
(
ChainedDriver
<?,
?>
chainedTask
:
chainedTasks
)
{
for
(
ChainedDriver
<?,
?>
chainedTask
:
chainedTasks
)
{
if
(
FunctionUtils
.
getFunctionRuntimeContext
(
chainedTask
.
getStub
(),
null
)
!=
null
)
{
RuntimeContext
rc
=
FunctionUtils
.
getFunctionRuntimeContext
(
chainedTask
.
getStub
(),
null
);
AccumulatorHelper
.
resetAndClearAccumulators
(
FunctionUtils
.
getFunctionRuntimeContext
(
chainedTask
.
getStub
(),
null
).
getAllAccumulators
());
if
(
rc
!=
null
)
{
AccumulatorHelper
.
resetAndClearAccumulators
(
rc
.
getAllAccumulators
());
}
}
}
}
}
}
...
@@ -1140,7 +1152,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
...
@@ -1140,7 +1152,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
}
catch
(
InterruptedException
iex
)
{
}
catch
(
InterruptedException
iex
)
{
throw
new
RuntimeException
(
"Interrupted while waiting for input "
+
index
+
" to become available."
);
throw
new
RuntimeException
(
"Interrupted while waiting for input "
+
index
+
" to become available."
);
}
catch
(
IOException
ioex
)
{
}
catch
(
IOException
ioex
)
{
throw
new
RuntimeException
(
"An I/O Exception occurred whil
y ob
aining input "
+
index
+
"."
);
throw
new
RuntimeException
(
"An I/O Exception occurred whil
e obt
aining input "
+
index
+
"."
);
}
}
}
}
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录