Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
6e1de988
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
6e1de988
编写于
7月 30, 2015
作者:
F
Fabian Hueske
提交者:
Ufuk Celebi
8月 26, 2015
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[FLINK-2394] HadoopOutputFormats use correct OutputCommitters.
This closes #1056.
上级
bc627871
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
45 addition
and
30 deletion
+45
-30
flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormat.java
...ache/flink/api/java/hadoop/mapred/HadoopOutputFormat.java
+6
-0
flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatBase.java
.../flink/api/java/hadoop/mapred/HadoopOutputFormatBase.java
+8
-8
flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatBase.java
...ink/api/java/hadoop/mapreduce/HadoopOutputFormatBase.java
+21
-21
flink-scala/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
...he/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
+10
-1
未找到文件。
flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormat.java
浏览文件 @
6e1de988
...
...
@@ -22,6 +22,7 @@ import java.io.IOException;
import
org.apache.flink.api.java.tuple.Tuple2
;
import
org.apache.hadoop.mapred.JobConf
;
import
org.apache.hadoop.mapred.OutputCommitter
;
public
class
HadoopOutputFormat
<
K
,
V
>
extends
HadoopOutputFormatBase
<
K
,
V
,
Tuple2
<
K
,
V
>>
{
...
...
@@ -31,6 +32,11 @@ public class HadoopOutputFormat<K,V> extends HadoopOutputFormatBase<K, V, Tuple2
super
(
mapredOutputFormat
,
job
);
}
public
HadoopOutputFormat
(
org
.
apache
.
hadoop
.
mapred
.
OutputFormat
<
K
,
V
>
mapredOutputFormat
,
Class
<
OutputCommitter
>
outputCommitterClass
,
JobConf
job
)
{
this
(
mapredOutputFormat
,
job
);
super
.
getJobConf
().
setOutputCommitter
(
outputCommitterClass
);
}
@Override
public
void
writeRecord
(
Tuple2
<
K
,
V
>
record
)
throws
IOException
{
this
.
recordWriter
.
write
(
record
.
f0
,
record
.
f1
);
...
...
flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapred/HadoopOutputFormatBase.java
浏览文件 @
6e1de988
...
...
@@ -26,11 +26,11 @@ import org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopDummyProgressable;
import
org.apache.flink.api.java.hadoop.mapred.wrapper.HadoopDummyReporter
;
import
org.apache.flink.configuration.Configuration
;
import
org.apache.hadoop.conf.Configurable
;
import
org.apache.hadoop.mapred.FileOutputCommitter
;
import
org.apache.hadoop.mapred.JobConf
;
import
org.apache.hadoop.mapred.JobConfigurable
;
import
org.apache.hadoop.mapred.JobContext
;
import
org.apache.hadoop.mapred.JobID
;
import
org.apache.hadoop.mapred.OutputCommitter
;
import
org.apache.hadoop.mapred.RecordWriter
;
import
org.apache.hadoop.mapred.TaskAttemptContext
;
import
org.apache.hadoop.mapred.TaskAttemptID
;
...
...
@@ -48,7 +48,7 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
private
JobConf
jobConf
;
private
org
.
apache
.
hadoop
.
mapred
.
OutputFormat
<
K
,
V
>
mapredOutputFormat
;
protected
transient
RecordWriter
<
K
,
V
>
recordWriter
;
private
transient
FileOutputCommitter
fileO
utputCommitter
;
private
transient
OutputCommitter
o
utputCommitter
;
private
transient
TaskAttemptContext
context
;
private
transient
JobContext
jobContext
;
...
...
@@ -106,7 +106,7 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
throw
new
RuntimeException
(
e
);
}
this
.
fileOutputCommitter
=
new
File
OutputCommitter
();
this
.
outputCommitter
=
this
.
jobConf
.
get
OutputCommitter
();
try
{
this
.
jobContext
=
HadoopUtils
.
instantiateJobContext
(
this
.
jobConf
,
new
JobID
());
...
...
@@ -114,7 +114,7 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
throw
new
RuntimeException
(
e
);
}
this
.
fileO
utputCommitter
.
setupJob
(
jobContext
);
this
.
o
utputCommitter
.
setupJob
(
jobContext
);
this
.
recordWriter
=
this
.
mapredOutputFormat
.
getRecordWriter
(
null
,
this
.
jobConf
,
Integer
.
toString
(
taskNumber
+
1
),
new
HadoopDummyProgressable
());
}
...
...
@@ -127,8 +127,8 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
public
void
close
()
throws
IOException
{
this
.
recordWriter
.
close
(
new
HadoopDummyReporter
());
if
(
this
.
fileO
utputCommitter
.
needsTaskCommit
(
this
.
context
))
{
this
.
fileO
utputCommitter
.
commitTask
(
this
.
context
);
if
(
this
.
o
utputCommitter
.
needsTaskCommit
(
this
.
context
))
{
this
.
o
utputCommitter
.
commitTask
(
this
.
context
);
}
}
...
...
@@ -137,10 +137,10 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
try
{
JobContext
jobContext
=
HadoopUtils
.
instantiateJobContext
(
this
.
jobConf
,
new
JobID
());
FileOutputCommitter
fileOutputCommitter
=
new
File
OutputCommitter
();
OutputCommitter
outputCommitter
=
this
.
jobConf
.
get
OutputCommitter
();
// finalize HDFS output format
fileO
utputCommitter
.
commitJob
(
jobContext
);
o
utputCommitter
.
commitJob
(
jobContext
);
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
...
...
flink-java/src/main/java/org/apache/flink/api/java/hadoop/mapreduce/HadoopOutputFormatBase.java
浏览文件 @
6e1de988
...
...
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path;
import
org.apache.hadoop.mapreduce.Job
;
import
org.apache.hadoop.mapreduce.JobContext
;
import
org.apache.hadoop.mapreduce.JobID
;
import
org.apache.hadoop.mapreduce.OutputCommitter
;
import
org.apache.hadoop.mapreduce.RecordWriter
;
import
org.apache.hadoop.mapreduce.TaskAttemptContext
;
import
org.apache.hadoop.mapreduce.TaskAttemptID
;
...
...
@@ -45,7 +46,7 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
private
org
.
apache
.
hadoop
.
conf
.
Configuration
configuration
;
private
org
.
apache
.
hadoop
.
mapreduce
.
OutputFormat
<
K
,
V
>
mapreduceOutputFormat
;
protected
transient
RecordWriter
<
K
,
V
>
recordWriter
;
private
transient
FileOutputCommitter
fileO
utputCommitter
;
private
transient
OutputCommitter
o
utputCommitter
;
private
transient
TaskAttemptContext
context
;
private
transient
int
taskNumber
;
...
...
@@ -101,20 +102,16 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
try
{
this
.
context
=
HadoopUtils
.
instantiateTaskAttemptContext
(
this
.
configuration
,
taskAttemptID
);
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
this
.
fileOutputCommitter
=
new
FileOutputCommitter
(
new
Path
(
this
.
configuration
.
get
(
"mapred.output.dir"
)),
context
);
try
{
this
.
fileOutputCommitter
.
setupJob
(
HadoopUtils
.
instantiateJobContext
(
this
.
configuration
,
new
JobID
()));
this
.
outputCommitter
=
this
.
mapreduceOutputFormat
.
getOutputCommitter
(
this
.
context
);
this
.
outputCommitter
.
setupJob
(
HadoopUtils
.
instantiateJobContext
(
this
.
configuration
,
new
JobID
()));
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
// compatible for hadoop 2.2.0, the temporary output directory is different from hadoop 1.2.1
this
.
configuration
.
set
(
"mapreduce.task.output.dir"
,
this
.
fileOutputCommitter
.
getWorkPath
().
toString
());
if
(
outputCommitter
instanceof
FileOutputCommitter
)
{
this
.
configuration
.
set
(
"mapreduce.task.output.dir"
,
((
FileOutputCommitter
)
this
.
outputCommitter
).
getWorkPath
().
toString
());
}
try
{
this
.
recordWriter
=
this
.
mapreduceOutputFormat
.
getRecordWriter
(
this
.
context
);
...
...
@@ -135,8 +132,8 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
throw
new
IOException
(
"Could not close RecordReader."
,
e
);
}
if
(
this
.
fileO
utputCommitter
.
needsTaskCommit
(
this
.
context
))
{
this
.
fileO
utputCommitter
.
commitTask
(
this
.
context
);
if
(
this
.
o
utputCommitter
.
needsTaskCommit
(
this
.
context
))
{
this
.
o
utputCommitter
.
commitTask
(
this
.
context
);
}
Path
outputPath
=
new
Path
(
this
.
configuration
.
get
(
"mapred.output.dir"
));
...
...
@@ -152,28 +149,31 @@ public abstract class HadoopOutputFormatBase<K, V, T> extends RichOutputFormat<T
fs
.
rename
(
new
Path
(
outputPath
.
toString
()+
"/"
+
tmpFile
),
new
Path
(
outputPath
.
toString
()+
"/"
+
taskNumberStr
));
}
}
@Override
public
void
finalizeGlobal
(
int
parallelism
)
throws
IOException
{
JobContext
jobContext
;
TaskAttemptContext
taskContext
;
try
{
TaskAttemptID
taskAttemptID
=
TaskAttemptID
.
forName
(
"attempt__0000_r_"
+
String
.
format
(
"%"
+
(
6
-
Integer
.
toString
(
1
).
length
())
+
"s"
,
" "
).
replace
(
" "
,
"0"
)
+
Integer
.
toString
(
1
)
TaskAttemptID
taskAttemptID
=
TaskAttemptID
.
forName
(
"attempt__0000_r_"
+
String
.
format
(
"%"
+
(
6
-
Integer
.
toString
(
1
).
length
())
+
"s"
,
" "
).
replace
(
" "
,
"0"
)
+
Integer
.
toString
(
1
)
+
"_0"
);
jobContext
=
HadoopUtils
.
instantiateJobContext
(
this
.
configuration
,
new
JobID
());
taskContext
=
HadoopUtils
.
instantiateTaskAttemptContext
(
this
.
configuration
,
taskAttemptID
);
this
.
outputCommitter
=
this
.
mapreduceOutputFormat
.
getOutputCommitter
(
taskContext
);
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
this
.
fileOutputCommitter
=
new
FileOutputCommitter
(
new
Path
(
this
.
configuration
.
get
(
"mapred.output.dir"
)),
taskContext
);
// finalize HDFS output format
this
.
fileOutputCommitter
.
commitJob
(
jobContext
);
if
(
this
.
outputCommitter
!=
null
)
{
this
.
outputCommitter
.
commitJob
(
jobContext
);
}
}
// --------------------------------------------------------------------------------------------
...
...
flink-scala/src/main/scala/org/apache/flink/api/scala/hadoop/mapred/HadoopOutputFormat.scala
浏览文件 @
6e1de988
...
...
@@ -18,11 +18,20 @@
package
org.apache.flink.api.scala.hadoop.mapred
import
org.apache.flink.api.java.hadoop.mapred.HadoopOutputFormatBase
import
org.apache.hadoop.mapred.
{
JobConf
,
OutputFormat
}
import
org.apache.hadoop.mapred.
{
OutputCommitter
,
JobConf
,
OutputFormat
}
class
HadoopOutputFormat
[
K
,
V
](
mapredOutputFormat
:
OutputFormat
[
K
,
V
],
job
:
JobConf
)
extends
HadoopOutputFormatBase
[
K
,
V
,
(
K
,
V
)](
mapredOutputFormat
,
job
)
{
def
this
(
mapredOutputFormat
:
OutputFormat
[
K
,
V
],
outputCommitterClass
:
Class
[
OutputCommitter
],
job
:
JobConf
)
{
this
(
mapredOutputFormat
,
job
)
this
.
getJobConf
.
setOutputCommitter
(
outputCommitterClass
)
}
def
writeRecord
(
record
:
(
K
,
V
))
{
this
.
recordWriter
.
write
(
record
.
_1
,
record
.
_2
)
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录