Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
68ebfc48
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
68ebfc48
编写于
7月 14, 2014
作者:
G
gyfora
提交者:
Stephan Ewen
8月 18, 2014
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[streaming] WordCount performance testing
上级
413227c9
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
46 addition
and
42 deletion
+46
-42
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamSource.java
...atosphere/streaming/api/streamcomponent/StreamSource.java
+1
-1
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamTask.java
...tratosphere/streaming/api/streamcomponent/StreamTask.java
+9
-9
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/wordcount/WordCountCounter.java
...sphere/streaming/examples/wordcount/WordCountCounter.java
+0
-9
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/wordcount/WordCountSink.java
...atosphere/streaming/examples/wordcount/WordCountSink.java
+0
-4
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/wordcount/WordCountSourceSplitter.java
...streaming/examples/wordcount/WordCountSourceSplitter.java
+4
-15
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/faulttolerance/FaultToleranceUtil.java
...tosphere/streaming/faulttolerance/FaultToleranceUtil.java
+32
-4
未找到文件。
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamSource.java
浏览文件 @
68ebfc48
...
...
@@ -80,7 +80,7 @@ public class StreamSource extends AbstractInputTask<DummyIS> {
numberOfOutputChannels
[
i
]=
taskConfiguration
.
getInteger
(
"channels_"
+
i
,
0
);
}
recordBuffer
=
new
FaultToleranceUtil
(
outputs
,
sourceInstanceID
,
numberOfOutputChannels
);
recordBuffer
=
new
FaultToleranceUtil
(
outputs
,
sourceInstanceID
,
name
,
numberOfOutputChannels
);
userFunction
=
(
UserSourceInvokable
)
streamSourceHelper
.
getUserFunction
(
taskConfiguration
,
outputs
,
sourceInstanceID
,
name
,
recordBuffer
);
streamSourceHelper
.
setAckListener
(
recordBuffer
,
sourceInstanceID
,
outputs
);
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamTask.java
浏览文件 @
68ebfc48
...
...
@@ -68,19 +68,19 @@ public class StreamTask extends AbstractTask {
log
.
error
(
"Cannot register inputs/outputs for "
+
getClass
().
getSimpleName
(),
e
);
}
int
[]
numberOfOutputChannels
=
new
int
[
outputs
.
size
()];
for
(
int
i
=
0
;
i
<
numberOfOutputChannels
.
length
;
i
++
)
{
numberOfOutputChannels
[
i
]
=
taskConfiguration
.
getInteger
(
"channels_"
+
i
,
0
);
int
[]
numberOfOutputChannels
=
new
int
[
outputs
.
size
()];
for
(
int
i
=
0
;
i
<
numberOfOutputChannels
.
length
;
i
++)
{
numberOfOutputChannels
[
i
]
=
taskConfiguration
.
getInteger
(
"channels_"
+
i
,
0
);
}
recordBuffer
=
new
FaultToleranceUtil
(
outputs
,
taskInstanceID
,
numberOfOutputChannels
);
userFunction
=
(
UserTaskInvokable
)
streamTaskHelper
.
getUserFunction
(
taskConfiguration
,
outputs
,
taskInstanceID
,
name
,
recordBuffer
);
recordBuffer
=
new
FaultToleranceUtil
(
outputs
,
taskInstanceID
,
n
ame
,
n
umberOfOutputChannels
);
userFunction
=
(
UserTaskInvokable
)
streamTaskHelper
.
getUserFunction
(
taskConfiguration
,
outputs
,
taskInstanceID
,
name
,
recordBuffer
);
streamTaskHelper
.
setAckListener
(
recordBuffer
,
taskInstanceID
,
outputs
);
streamTaskHelper
.
setFailListener
(
recordBuffer
,
taskInstanceID
,
outputs
);
}
@Override
public
void
invoke
()
throws
Exception
{
log
.
debug
(
"TASK "
+
name
+
" invoked with instance id "
+
taskInstanceID
);
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/wordcount/WordCountCounter.java
浏览文件 @
68ebfc48
...
...
@@ -21,8 +21,6 @@ import java.util.Map;
import
eu.stratosphere.api.java.tuple.Tuple2
;
import
eu.stratosphere.streaming.api.invokable.UserTaskInvokable
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
import
eu.stratosphere.streaming.util.PerformanceCounter
;
import
eu.stratosphere.streaming.util.PerformanceTimer
;
public
class
WordCountCounter
extends
UserTaskInvokable
{
...
...
@@ -30,8 +28,6 @@ public class WordCountCounter extends UserTaskInvokable {
private
String
word
=
""
;
private
Integer
count
=
0
;
PerformanceCounter
pCounter
=
new
PerformanceCounter
(
"CounterEmitCounter"
,
1000
,
1000
,
""
);
PerformanceTimer
pTimer
=
new
PerformanceTimer
(
"CounterEmitTimer"
,
1000
,
1000
,
true
,
""
);
private
StreamRecord
outRecord
=
new
StreamRecord
(
new
Tuple2
<
String
,
Integer
>());
...
...
@@ -50,17 +46,12 @@ public class WordCountCounter extends UserTaskInvokable {
outRecord
.
setString
(
0
,
word
);
outRecord
.
setInteger
(
1
,
count
);
pTimer
.
startTimer
();
emit
(
outRecord
);
pTimer
.
stopTimer
();
pCounter
.
count
();
}
@Override
public
String
getResult
()
{
pCounter
.
writeCSV
(
"/home/strato/stratosphere-distrib/log/counter/Counter"
+
channelID
);
pTimer
.
writeCSV
(
"/home/strato/stratosphere-distrib/log/timer/Counter"
+
channelID
);
return
""
;
}
}
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/wordcount/WordCountSink.java
浏览文件 @
68ebfc48
...
...
@@ -17,20 +17,16 @@ package eu.stratosphere.streaming.examples.wordcount;
import
eu.stratosphere.streaming.api.invokable.UserSinkInvokable
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
import
eu.stratosphere.streaming.util.PerformanceCounter
;
public
class
WordCountSink
extends
UserSinkInvokable
{
PerformanceCounter
perf
=
new
PerformanceCounter
(
"SinkEmitCounter"
,
1000
,
1000
,
""
);
@Override
public
void
invoke
(
StreamRecord
record
)
throws
Exception
{
perf
.
count
();
}
@Override
public
String
getResult
()
{
perf
.
writeCSV
(
"/home/strato/stratosphere-distrib/log/counter/Sink"
);
return
""
;
}
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/examples/wordcount/WordCountSourceSplitter.java
浏览文件 @
68ebfc48
...
...
@@ -22,8 +22,6 @@ import java.io.FileReader;
import
eu.stratosphere.api.java.tuple.Tuple1
;
import
eu.stratosphere.streaming.api.invokable.UserSourceInvokable
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
import
eu.stratosphere.streaming.util.PerformanceCounter
;
import
eu.stratosphere.streaming.util.PerformanceTimer
;
public
class
WordCountSourceSplitter
extends
UserSourceInvokable
{
...
...
@@ -31,13 +29,11 @@ public class WordCountSourceSplitter extends UserSourceInvokable {
private
String
line
=
new
String
();
private
StreamRecord
outRecord
=
new
StreamRecord
(
new
Tuple1
<
String
>());
PerformanceCounter
pCounter
=
new
PerformanceCounter
(
"SourceEmitCounter"
,
1000
,
1000
,
""
);
PerformanceTimer
pTimer
=
new
PerformanceTimer
(
"SourceEmitTimer"
,
1000
,
1000
,
true
,
""
);
@Override
public
void
invoke
()
throws
Exception
{
for
(
int
i
=
0
;
i
<
10
;
i
++
)
{
while
(
true
)
{
try
{
br
=
new
BufferedReader
(
new
FileReader
(
"/home/strato/stratosphere-distrib/resources/hamlet.txt"
));
...
...
@@ -46,10 +42,7 @@ public class WordCountSourceSplitter extends UserSourceInvokable {
if
(
line
!=
""
)
{
for
(
String
word
:
line
.
split
(
" "
))
{
outRecord
.
setString
(
0
,
word
);
pTimer
.
startTimer
();
emit
(
outRecord
);
pTimer
.
stopTimer
();
pCounter
.
count
();
}
}
line
=
br
.
readLine
();
...
...
@@ -58,20 +51,16 @@ public class WordCountSourceSplitter extends UserSourceInvokable {
}
catch
(
FileNotFoundException
e
)
{
e
.
printStackTrace
();
}
// Thread.sleep(1);
}
Thread
.
sleep
(
15000
);
emit
(
outRecord
);
Thread
.
sleep
(
15000
);
emit
(
outRecord
);
}
@Override
public
String
getResult
()
{
pCounter
.
writeCSV
(
"/home/strato/stratosphere-distrib/log/counter/Source"
+
channelID
);
pTimer
.
writeCSV
(
"/home/strato/stratosphere-distrib/log/timer/Source"
+
channelID
);
return
""
;
}
}
\ No newline at end of file
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/faulttolerance/FaultToleranceUtil.java
浏览文件 @
68ebfc48
...
...
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
import
eu.stratosphere.nephele.io.RecordWriter
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
import
eu.stratosphere.streaming.api.streamrecord.UID
;
import
eu.stratosphere.streaming.util.PerformanceCounter
;
import
eu.stratosphere.streaming.util.PerformanceTracker
;
/**
...
...
@@ -42,7 +43,8 @@ public class FaultToleranceUtil {
boolean
exactlyOnce
;
private
FaultToleranceBuffer
buffer
;
public
PerformanceTracker
counter
;
public
PerformanceTracker
tracker
;
public
PerformanceCounter
counter
;
/**
* Creates fault tolerance buffer object for the given output channels and
...
...
@@ -71,10 +73,35 @@ public class FaultToleranceUtil {
this
.
buffer
=
new
AtLeastOnceFaultToleranceBuffer
(
numberOfChannels
,
sourceInstanceID
);
}
counter
=
new
PerformanceTracker
(
"pc"
,
1000
,
1000
,
14900
,
"/home/strato/stratosphere-distrib/log/counter/Buffer"
+
sourceInstanceID
);
tracker
=
new
PerformanceTracker
(
"pc"
,
1000
,
1000
,
30000
,
"/home/strato/stratosphere-distrib/log/counter/Buffer"
+
sourceInstanceID
);
counter
=
new
PerformanceCounter
(
"pc"
,
1000
,
1000
,
30000
,
"/home/strato/stratosphere-distrib/log/counter/Emitted"
+
sourceInstanceID
);
}
public
FaultToleranceUtil
(
List
<
RecordWriter
<
StreamRecord
>>
outputs
,
int
sourceInstanceID
,
String
componentName
,
int
[]
numberOfChannels
)
{
this
.
outputs
=
outputs
;
this
.
componentID
=
sourceInstanceID
;
exactlyOnce
=
false
;
if
(
exactlyOnce
)
{
this
.
buffer
=
new
ExactlyOnceFaultToleranceBuffer
(
numberOfChannels
,
sourceInstanceID
);
}
else
{
this
.
buffer
=
new
AtLeastOnceFaultToleranceBuffer
(
numberOfChannels
,
sourceInstanceID
);
}
tracker
=
new
PerformanceTracker
(
"pc"
,
1000
,
1000
,
10000
,
"/home/strato/stratosphere-distrib/log/counter/Buffer"
+
componentName
+
sourceInstanceID
);
counter
=
new
PerformanceCounter
(
"pc"
,
1000
,
1000
,
10000
,
"/home/strato/stratosphere-distrib/log/counter/Emitted"
+
componentName
+
sourceInstanceID
);
}
/**
* Adds the record to the fault tolerance buffer. This record will be
* monitored for acknowledgements and timeout.
...
...
@@ -85,7 +112,8 @@ public class FaultToleranceUtil {
public
void
addRecord
(
StreamRecord
streamRecord
)
{
buffer
.
add
(
streamRecord
);
counter
.
track
(
this
.
buffer
.
recordBuffer
.
size
());
tracker
.
track
(
this
.
buffer
.
recordBuffer
.
size
());
counter
.
count
();
}
public
void
addRecord
(
StreamRecord
streamRecord
,
int
output
)
{
...
...
@@ -107,7 +135,7 @@ public class FaultToleranceUtil {
// TODO: find a place to call timeoutRecords
public
void
ackRecord
(
UID
recordID
,
int
channel
)
{
buffer
.
ack
(
recordID
,
channel
);
tracker
.
track
(
this
.
buffer
.
recordBuffer
.
size
());
}
/**
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录