Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
f660d058
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
f660d058
编写于
7月 14, 2014
作者:
G
gaborhermann
提交者:
Stephan Ewen
8月 18, 2014
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[streaming] Implemented thread safe event publishing
上级
b1071d8e
变更
5
显示空白变更内容
内联
并排
Showing
5 changed file
with
97 addition
and
123 deletion
+97
-123
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/FaultTolerancyBuffer.java
...a/eu/stratosphere/streaming/api/FaultTolerancyBuffer.java
+1
-1
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamComponentHelper.java
.../streaming/api/streamcomponent/StreamComponentHelper.java
+55
-59
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamSink.java
...tratosphere/streaming/api/streamcomponent/StreamSink.java
+12
-24
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamSource.java
...atosphere/streaming/api/streamcomponent/StreamSource.java
+14
-7
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamTask.java
...tratosphere/streaming/api/streamcomponent/StreamTask.java
+15
-32
未找到文件。
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/FaultTolerancyBuffer.java
浏览文件 @
f660d058
...
...
@@ -15,7 +15,6 @@ import eu.stratosphere.types.Record;
public
class
FaultTolerancyBuffer
{
private
final
static
long
TIMEOUT
=
1000
;
private
final
static
long
TIMESTAMP_STORE_WINDOW
=
TIMEOUT
/
10
;
private
Long
timeOfLastUpdate
;
private
Map
<
String
,
StreamRecord
>
recordBuffer
;
...
...
@@ -47,6 +46,7 @@ public class FaultTolerancyBuffer {
addTimestamp
(
streamRecord
.
getId
());
}
//TODO: use this method!
private
void
timeoutRecords
()
{
Long
currentTime
=
System
.
currentTimeMillis
();
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamComponent
Factory
.java
→
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamComponent
Helper
.java
浏览文件 @
f660d058
package
eu.stratosphere.streaming.api.streamcomponent
;
import
java.util.List
;
import
java.util.Random
;
import
eu.stratosphere.configuration.Configuration
;
import
eu.stratosphere.nephele.event.task.AbstractTaskEvent
;
import
eu.stratosphere.nephele.event.task.EventListener
;
import
eu.stratosphere.nephele.io.ChannelSelector
;
import
eu.stratosphere.nephele.io.RecordReader
;
import
eu.stratosphere.nephele.io.RecordWriter
;
import
eu.stratosphere.nephele.template.AbstractInvokable
;
import
eu.stratosphere.streaming.api.AckEvent
;
import
eu.stratosphere.streaming.api.AckEventListener
;
import
eu.stratosphere.streaming.api.FailEvent
;
...
...
@@ -22,9 +25,10 @@ import eu.stratosphere.types.Key;
import
eu.stratosphere.types.Record
;
import
eu.stratosphere.types.StringValue
;
public
final
class
StreamComponentFactory
{
public
final
class
StreamComponentHelper
<
T
extends
AbstractInvokable
>
{
private
Random
random
=
new
Random
();
public
static
void
setAckListener
(
FaultTolerancyBuffer
recordBuffer
,
public
void
setAckListener
(
FaultTolerancyBuffer
recordBuffer
,
String
sourceInstanceID
,
List
<
RecordWriter
<
Record
>>
outputs
)
{
EventListener
eventListener
=
new
AckEventListener
(
sourceInstanceID
,
recordBuffer
);
...
...
@@ -34,7 +38,7 @@ public final class StreamComponentFactory {
}
}
public
static
void
setFailListener
(
FaultTolerancyBuffer
recordBuffer
,
public
void
setFailListener
(
FaultTolerancyBuffer
recordBuffer
,
String
sourceInstanceID
,
List
<
RecordWriter
<
Record
>>
outputs
)
{
EventListener
eventListener
=
new
FailEventListener
(
sourceInstanceID
,
recordBuffer
);
...
...
@@ -44,67 +48,46 @@ public final class StreamComponentFactory {
}
}
// for StreamTask
public
static
int
setConfigInputs
(
StreamTask
taskBase
,
Configuration
taskConfiguration
,
List
<
RecordReader
<
Record
>>
inputs
)
{
public
void
setConfigInputs
(
T
taskBase
,
Configuration
taskConfiguration
,
List
<
RecordReader
<
Record
>>
inputs
)
throws
Exception
{
int
numberOfInputs
=
taskConfiguration
.
getInteger
(
"numberOfInputs"
,
0
);
for
(
int
i
=
0
;
i
<
numberOfInputs
;
i
++)
{
inputs
.
add
(
new
RecordReader
<
Record
>(
taskBase
,
Record
.
class
));
}
return
numberOfInputs
;
if
(
taskBase
instanceof
StreamTask
)
{
inputs
.
add
(
new
RecordReader
<
Record
>((
StreamTask
)
taskBase
,
Record
.
class
));
}
else
if
(
taskBase
instanceof
StreamSink
)
{
inputs
.
add
(
new
RecordReader
<
Record
>((
StreamSink
)
taskBase
,
Record
.
class
));
}
else
{
throw
new
Exception
(
"Nonsupported object passed to setConfigInputs"
);
}
// this function can be removed as duplication of the above function if
// modification on kernel is allowed.
// for StreamSink
public
static
int
setConfigInputs
(
StreamSink
taskBase
,
Configuration
taskConfiguration
,
List
<
RecordReader
<
Record
>>
inputs
)
{
int
numberOfInputs
=
taskConfiguration
.
getInteger
(
"numberOfInputs"
,
0
);
for
(
int
i
=
0
;
i
<
numberOfInputs
;
i
++)
{
inputs
.
add
(
new
RecordReader
<
Record
>(
taskBase
,
Record
.
class
));
}
return
numberOfInputs
;
}
// for StreamTask
public
static
int
setConfigOutputs
(
StreamTask
taskBase
,
Configuration
taskConfiguration
,
public
void
setConfigOutputs
(
T
taskBase
,
Configuration
taskConfiguration
,
List
<
RecordWriter
<
Record
>>
outputs
,
List
<
ChannelSelector
<
Record
>>
partitioners
)
{
List
<
ChannelSelector
<
Record
>>
partitioners
)
throws
Exception
{
int
numberOfOutputs
=
taskConfiguration
.
getInteger
(
"numberOfOutputs"
,
0
);
for
(
int
i
=
1
;
i
<=
numberOfOutputs
;
i
++)
{
StreamComponentFactory
.
setPartitioner
(
taskConfiguration
,
i
,
partitioners
);
setPartitioner
(
taskConfiguration
,
i
,
partitioners
);
}
for
(
ChannelSelector
<
Record
>
outputPartitioner
:
partitioners
)
{
outputs
.
add
(
new
RecordWriter
<
Record
>(
taskBase
,
Record
.
class
,
outputPartitioner
));
}
return
numberOfOutputs
;
}
// this function can be removed as duplication of the above function if
// modification on kernel is allowed.
// for StreamSource
public
static
int
setConfigOutputs
(
StreamSource
taskBase
,
Configuration
taskConfiguration
,
List
<
RecordWriter
<
Record
>>
outputs
,
List
<
ChannelSelector
<
Record
>>
partitioners
)
{
int
numberOfOutputs
=
taskConfiguration
.
getInteger
(
"numberOfOutputs"
,
0
);
for
(
int
i
=
1
;
i
<=
numberOfOutputs
;
i
++)
{
StreamComponentFactory
.
setPartitioner
(
taskConfiguration
,
i
,
partitioners
);
if
(
taskBase
instanceof
StreamTask
)
{
outputs
.
add
(
new
RecordWriter
<
Record
>((
StreamTask
)
taskBase
,
Record
.
class
,
outputPartitioner
));
}
else
if
(
taskBase
instanceof
StreamSource
)
{
outputs
.
add
(
new
RecordWriter
<
Record
>((
StreamSource
)
taskBase
,
Record
.
class
,
outputPartitioner
));
}
else
{
throw
new
Exception
(
"Nonsupported object passed to setConfigOutputs"
);
}
for
(
ChannelSelector
<
Record
>
outputPartitioner
:
partitioners
)
{
outputs
.
add
(
new
RecordWriter
<
Record
>(
taskBase
,
Record
.
class
,
outputPartitioner
));
}
return
numberOfOutputs
;
}
public
static
UserSinkInvokable
s
etUserFunction
(
Configuration
taskConfiguration
)
{
public
UserSinkInvokable
g
etUserFunction
(
Configuration
taskConfiguration
)
{
Class
<?
extends
UserSinkInvokable
>
userFunctionClass
=
taskConfiguration
.
getClass
(
"userfunction"
,
DefaultSinkInvokable
.
class
,
...
...
@@ -119,12 +102,11 @@ public final class StreamComponentFactory {
return
userFunction
;
}
public
static
StreamInvokable
setUserFunction
(
Configuration
taskConfiguration
,
public
StreamInvokable
getUserFunction
(
Configuration
taskConfiguration
,
List
<
RecordWriter
<
Record
>>
outputs
,
String
instanceID
,
FaultTolerancyBuffer
recordBuffer
)
{
//Default value is a TaskInvokable even if it was called from a source
//
Default value is a TaskInvokable even if it was called from a source
Class
<?
extends
StreamInvokable
>
userFunctionClass
=
taskConfiguration
.
getClass
(
"userfunction"
,
DefaultTaskInvokable
.
class
,
StreamInvokable
.
class
);
...
...
@@ -139,8 +121,21 @@ public final class StreamComponentFactory {
return
userFunction
;
}
public
static
void
setPartitioner
(
Configuration
taskConfiguration
,
int
nrOutput
,
List
<
ChannelSelector
<
Record
>>
partitioners
)
{
//TODO: use TCP-like waiting
public
void
threadSafePublish
(
AbstractTaskEvent
e
,
RecordReader
<
Record
>
input
)
throws
InterruptedException
{
boolean
concurrentModificationOccured
=
false
;
while
(!
concurrentModificationOccured
)
{
try
{
input
.
publishEvent
(
e
);
concurrentModificationOccured
=
true
;
}
catch
(
Exception
exeption
)
{
Thread
.
sleep
(
random
.
nextInt
(
50
));
}
}
}
private
void
setPartitioner
(
Configuration
taskConfiguration
,
int
nrOutput
,
List
<
ChannelSelector
<
Record
>>
partitioners
)
{
Class
<?
extends
ChannelSelector
<
Record
>>
partitioner
=
taskConfiguration
.
getClass
(
"partitionerClass_"
+
nrOutput
,
DefaultPartitioner
.
class
,
ChannelSelector
.
class
);
...
...
@@ -165,4 +160,5 @@ public final class StreamComponentFactory {
System
.
out
.
println
(
e
);
}
}
}
\ No newline at end of file
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamSink.java
浏览文件 @
f660d058
...
...
@@ -17,7 +17,6 @@ package eu.stratosphere.streaming.api.streamcomponent;
import
java.util.LinkedList
;
import
java.util.List
;
import
java.util.Random
;
import
eu.stratosphere.configuration.Configuration
;
import
eu.stratosphere.nephele.io.RecordReader
;
...
...
@@ -32,20 +31,25 @@ public class StreamSink extends AbstractOutputTask {
private
List
<
RecordReader
<
Record
>>
inputs
;
private
UserSinkInvokable
userFunction
;
private
Random
rnd
=
new
Random
();
private
StreamComponentHelper
<
StreamSink
>
streamSinkHelper
;
public
StreamSink
()
{
// TODO: Make configuration file visible and call setClassInputs() here
inputs
=
new
LinkedList
<
RecordReader
<
Record
>>();
userFunction
=
null
;
streamSinkHelper
=
new
StreamComponentHelper
<
StreamSink
>();
}
@Override
public
void
registerInputOutput
()
{
Configuration
taskConfiguration
=
getTaskConfiguration
();
StreamComponentFactory
.
setConfigInputs
(
this
,
taskConfiguration
,
inputs
);
userFunction
=
StreamComponentFactory
.
setUserFunction
(
taskConfiguration
);
try
{
streamSinkHelper
.
setConfigInputs
(
this
,
taskConfiguration
,
inputs
);
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
userFunction
=
streamSinkHelper
.
getUserFunction
(
taskConfiguration
);
}
@Override
...
...
@@ -60,26 +64,10 @@ public class StreamSink extends AbstractOutputTask {
String
id
=
rec
.
getId
();
try
{
userFunction
.
invoke
(
rec
.
getRecord
());
//TODO create concurrent publish method
boolean
concurrentModificationOccured
=
false
;
while
(!
concurrentModificationOccured
)
{
try
{
input
.
publishEvent
(
new
AckEvent
(
id
));
concurrentModificationOccured
=
true
;
}
catch
(
Exception
f
)
{
Thread
.
sleep
(
rnd
.
nextInt
(
50
));
}
}
streamSinkHelper
.
threadSafePublish
(
new
AckEvent
(
id
),
input
);
}
catch
(
Exception
e
)
{
boolean
concurrentModificationOccured
=
false
;
while
(!
concurrentModificationOccured
)
{
try
{
input
.
publishEvent
(
new
FailEvent
(
id
));
concurrentModificationOccured
=
true
;
}
catch
(
Exception
f
)
{
Thread
.
sleep
(
rnd
.
nextInt
(
50
));
}
}
streamSinkHelper
.
threadSafePublish
(
new
FailEvent
(
id
),
input
);
}
}
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamSource.java
浏览文件 @
f660d058
...
...
@@ -59,15 +59,22 @@ public class StreamSource extends AbstractInputTask<RandIS> {
@Override
public
void
registerInputOutput
()
{
Configuration
taskConfiguration
=
getTaskConfiguration
();
StreamComponentFactory
.
setConfigOutputs
(
this
,
taskConfiguration
,
StreamComponentHelper
<
StreamSource
>
streamSourceHelper
=
new
StreamComponentHelper
<
StreamSource
>();
try
{
streamSourceHelper
.
setConfigOutputs
(
this
,
taskConfiguration
,
outputs
,
partitioners
);
recordBuffer
=
new
FaultTolerancyBuffer
(
outputs
,
sourceInstanceID
);
userFunction
=
(
UserSourceInvokable
)
StreamComponentFactory
.
setUserFunction
(
taskConfiguration
,
outputs
,
sourceInstanceID
,
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
recordBuffer
=
new
FaultTolerancyBuffer
(
outputs
,
sourceInstanceID
);
userFunction
=
(
UserSourceInvokable
)
streamSourceHelper
.
getUserFunction
(
taskConfiguration
,
outputs
,
sourceInstanceID
,
recordBuffer
);
StreamComponentFactory
.
setAckListener
(
recordBuffer
,
sourceInstanceID
,
streamSourceHelper
.
setAckListener
(
recordBuffer
,
sourceInstanceID
,
outputs
);
StreamComponentFactory
.
setFailListener
(
recordBuffer
,
sourceInstanceID
,
streamSourceHelper
.
setFailListener
(
recordBuffer
,
sourceInstanceID
,
outputs
);
}
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamTask.java
浏览文件 @
f660d058
...
...
@@ -17,7 +17,6 @@ package eu.stratosphere.streaming.api.streamcomponent;
import
java.util.LinkedList
;
import
java.util.List
;
import
java.util.Random
;
import
eu.stratosphere.configuration.Configuration
;
import
eu.stratosphere.nephele.io.ChannelSelector
;
...
...
@@ -39,8 +38,7 @@ public class StreamTask extends AbstractTask {
private
UserTaskInvokable
userFunction
;
private
static
int
numTasks
=
0
;
private
String
taskInstanceID
=
""
;
private
Random
rnd
=
new
Random
();
StreamComponentHelper
<
StreamTask
>
streamTaskHelper
;
private
FaultTolerancyBuffer
recordBuffer
;
...
...
@@ -52,24 +50,26 @@ public class StreamTask extends AbstractTask {
userFunction
=
null
;
numTasks
++;
taskInstanceID
=
Integer
.
toString
(
numTasks
);
streamTaskHelper
=
new
StreamComponentHelper
<
StreamTask
>();
}
@Override
public
void
registerInputOutput
()
{
Configuration
taskConfiguration
=
getTaskConfiguration
();
StreamComponentFactory
.
setConfigInputs
(
this
,
taskConfiguration
,
inputs
);
StreamComponentFactory
.
setConfigOutputs
(
this
,
taskConfiguration
,
outputs
,
try
{
streamTaskHelper
.
setConfigInputs
(
this
,
taskConfiguration
,
inputs
);
streamTaskHelper
.
setConfigOutputs
(
this
,
taskConfiguration
,
outputs
,
partitioners
);
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
recordBuffer
=
new
FaultTolerancyBuffer
(
outputs
,
taskInstanceID
);
userFunction
=
(
UserTaskInvokable
)
StreamComponentFactory
.
s
etUserFunction
(
userFunction
=
(
UserTaskInvokable
)
streamTaskHelper
.
g
etUserFunction
(
taskConfiguration
,
outputs
,
taskInstanceID
,
recordBuffer
);
StreamComponentFactory
.
setAckListener
(
recordBuffer
,
taskInstanceID
,
outputs
);
StreamComponentFactory
.
setFailListener
(
recordBuffer
,
taskInstanceID
,
outputs
);
streamTaskHelper
.
setAckListener
(
recordBuffer
,
taskInstanceID
,
outputs
);
streamTaskHelper
.
setFailListener
(
recordBuffer
,
taskInstanceID
,
outputs
);
}
@Override
...
...
@@ -82,29 +82,12 @@ public class StreamTask extends AbstractTask {
hasInput
=
true
;
StreamRecord
streamRecord
=
new
StreamRecord
(
input
.
next
());
String
id
=
streamRecord
.
getId
();
//
TODO create method for concurrent publishing
//
TODO create method for concurrent publishing
try
{
userFunction
.
invoke
(
streamRecord
.
getRecord
());
boolean
concurrentModificationOccured
=
false
;
while
(!
concurrentModificationOccured
)
{
try
{
input
.
publishEvent
(
new
AckEvent
(
id
));
concurrentModificationOccured
=
true
;
streamTaskHelper
.
threadSafePublish
(
new
AckEvent
(
id
),
input
);
}
catch
(
Exception
e
)
{
Thread
.
sleep
(
rnd
.
nextInt
(
50
));
}
}
}
catch
(
Exception
e
)
{
boolean
concurrentModificationOccured
=
false
;
while
(!
concurrentModificationOccured
)
{
try
{
input
.
publishEvent
(
new
FailEvent
(
id
));
concurrentModificationOccured
=
true
;
}
catch
(
Exception
f
)
{
Thread
.
sleep
(
rnd
.
nextInt
(
50
));
}
}
streamTaskHelper
.
threadSafePublish
(
new
FailEvent
(
id
),
input
);
}
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录