Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
8f77a938
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
8f77a938
编写于
7月 14, 2014
作者:
G
gyfora
提交者:
Stephan Ewen
8月 18, 2014
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[streaming] streamcollector refactor
上级
c3fe7358
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
236 addition
and
92 deletion
+236
-92
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/StreamCollector.java
...n/java/eu/stratosphere/streaming/api/StreamCollector.java
+10
-19
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/StreamCollectorManager.java
...eu/stratosphere/streaming/api/StreamCollectorManager.java
+8
-10
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamSource.java
...atosphere/streaming/api/streamcomponent/StreamSource.java
+0
-3
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/FlatMapTest.java
.../test/java/eu/stratosphere/streaming/api/FlatMapTest.java
+164
-25
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/MapTest.java
.../src/test/java/eu/stratosphere/streaming/api/MapTest.java
+51
-31
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/PrintTest.java
...rc/test/java/eu/stratosphere/streaming/api/PrintTest.java
+2
-0
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/StreamCollectorTest.java
...va/eu/stratosphere/streaming/api/StreamCollectorTest.java
+1
-4
未找到文件。
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/StreamCollector.java
浏览文件 @
8f77a938
...
@@ -15,8 +15,6 @@
...
@@ -15,8 +15,6 @@
package
eu.stratosphere.streaming.api
;
package
eu.stratosphere.streaming.api
;
import
java.util.List
;
import
eu.stratosphere.api.java.tuple.Tuple
;
import
eu.stratosphere.api.java.tuple.Tuple
;
import
eu.stratosphere.nephele.io.RecordWriter
;
import
eu.stratosphere.nephele.io.RecordWriter
;
import
eu.stratosphere.pact.runtime.plugable.SerializationDelegate
;
import
eu.stratosphere.pact.runtime.plugable.SerializationDelegate
;
...
@@ -32,17 +30,16 @@ public class StreamCollector<T extends Tuple> implements Collector<T> {
...
@@ -32,17 +30,16 @@ public class StreamCollector<T extends Tuple> implements Collector<T> {
protected
int
counter
=
0
;
protected
int
counter
=
0
;
protected
int
channelID
;
protected
int
channelID
;
private
long
timeOfLastRecordEmitted
=
System
.
currentTimeMillis
();;
private
long
timeOfLastRecordEmitted
=
System
.
currentTimeMillis
();;
private
List
<
RecordWriter
<
StreamRecord
>>
outputs
;
private
RecordWriter
<
StreamRecord
>
output
;
public
StreamCollector
(
int
batchSize
,
long
batchTimeout
,
int
channelID
,
public
StreamCollector
(
int
batchSize
,
long
batchTimeout
,
int
channelID
,
SerializationDelegate
<
Tuple
>
serializationDelegate
,
SerializationDelegate
<
Tuple
>
serializationDelegate
,
RecordWriter
<
StreamRecord
>
output
)
{
List
<
RecordWriter
<
StreamRecord
>>
outputs
)
{
this
.
batchSize
=
batchSize
;
this
.
batchSize
=
batchSize
;
this
.
batchTimeout
=
batchTimeout
;
this
.
batchTimeout
=
batchTimeout
;
this
.
streamRecord
=
new
ArrayStreamRecord
(
batchSize
);
this
.
streamRecord
=
new
ArrayStreamRecord
(
batchSize
);
this
.
streamRecord
.
setSeralizationDelegate
(
serializationDelegate
);
this
.
streamRecord
.
setSeralizationDelegate
(
serializationDelegate
);
this
.
channelID
=
channelID
;
this
.
channelID
=
channelID
;
this
.
output
s
=
outputs
;
this
.
output
=
output
;
}
}
public
StreamCollector
(
int
batchSize
,
long
batchTimeout
,
int
channelID
,
public
StreamCollector
(
int
batchSize
,
long
batchTimeout
,
int
channelID
,
...
@@ -53,7 +50,6 @@ public class StreamCollector<T extends Tuple> implements Collector<T> {
...
@@ -53,7 +50,6 @@ public class StreamCollector<T extends Tuple> implements Collector<T> {
// TODO reconsider emitting mechanism at timeout (find a place to timeout)
// TODO reconsider emitting mechanism at timeout (find a place to timeout)
@Override
@Override
public
void
collect
(
T
tuple
)
{
public
void
collect
(
T
tuple
)
{
//TODO: move copy to StreamCollector2
streamRecord
.
setTuple
(
counter
,
tuple
);
streamRecord
.
setTuple
(
counter
,
tuple
);
counter
++;
counter
++;
...
@@ -77,19 +73,14 @@ public class StreamCollector<T extends Tuple> implements Collector<T> {
...
@@ -77,19 +73,14 @@ public class StreamCollector<T extends Tuple> implements Collector<T> {
counter
=
0
;
counter
=
0
;
streamRecord
.
setId
(
channelID
);
streamRecord
.
setId
(
channelID
);
if
(
outputs
==
null
)
{
try
{
System
.
out
.
println
(
streamRecord
);
output
.
emit
(
streamRecord
);
}
else
{
output
.
flush
();
for
(
RecordWriter
<
StreamRecord
>
output
:
outputs
)
{
}
catch
(
Exception
e
)
{
try
{
e
.
printStackTrace
();
output
.
emit
(
streamRecord
);
System
.
out
.
println
(
"emit fail"
);
output
.
flush
();
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
System
.
out
.
println
(
"emit fail"
);
}
}
}
}
}
}
@Override
@Override
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/StreamCollectorManager.java
浏览文件 @
8f77a938
...
@@ -33,8 +33,9 @@ public class StreamCollectorManager<T extends Tuple> implements Collector<T> {
...
@@ -33,8 +33,9 @@ public class StreamCollectorManager<T extends Tuple> implements Collector<T> {
int
keyPostition
;
int
keyPostition
;
// TODO consider channelID
// TODO consider channelID
public
StreamCollectorManager
(
List
<
Integer
>
batchSizesOfNotPartitioned
,
List
<
Integer
>
batchSizesOfPartitioned
,
public
StreamCollectorManager
(
List
<
Integer
>
batchSizesOfNotPartitioned
,
List
<
Integer
>
parallelismOfOutput
,
int
keyPosition
,
long
batchTimeout
,
int
channelID
,
List
<
Integer
>
batchSizesOfPartitioned
,
List
<
Integer
>
parallelismOfOutput
,
int
keyPosition
,
long
batchTimeout
,
int
channelID
,
SerializationDelegate
<
Tuple
>
serializationDelegate
,
SerializationDelegate
<
Tuple
>
serializationDelegate
,
List
<
RecordWriter
<
StreamRecord
>>
partitionedOutputs
,
List
<
RecordWriter
<
StreamRecord
>>
partitionedOutputs
,
List
<
RecordWriter
<
StreamRecord
>>
notPartitionedOutputs
)
{
List
<
RecordWriter
<
StreamRecord
>>
notPartitionedOutputs
)
{
...
@@ -47,19 +48,16 @@ public class StreamCollectorManager<T extends Tuple> implements Collector<T> {
...
@@ -47,19 +48,16 @@ public class StreamCollectorManager<T extends Tuple> implements Collector<T> {
this
.
keyPostition
=
keyPosition
;
this
.
keyPostition
=
keyPosition
;
for
(
int
i
=
0
;
i
<
batchSizesOfNotPartitioned
.
size
();
i
++)
{
for
(
int
i
=
0
;
i
<
batchSizesOfNotPartitioned
.
size
();
i
++)
{
List
<
RecordWriter
<
StreamRecord
>>
output
=
new
ArrayList
<
RecordWriter
<
StreamRecord
>>();
notPartitionedCollectors
.
add
(
new
StreamCollector
<
Tuple
>(
batchSizesOfNotPartitioned
output
.
add
(
notPartitionedOutputs
.
get
(
i
));
.
get
(
i
),
batchTimeout
,
channelID
,
serializationDelegate
,
notPartitionedOutputs
notPartitionedCollectors
.
add
(
new
StreamCollector
<
Tuple
>(
batchSizesOfNotPartitioned
.
get
(
i
),
.
get
(
i
)));
batchTimeout
,
channelID
,
serializationDelegate
,
output
));
}
}
for
(
int
i
=
0
;
i
<
batchSizesOfPartitioned
.
size
();
i
++)
{
for
(
int
i
=
0
;
i
<
batchSizesOfPartitioned
.
size
();
i
++)
{
StreamCollector
<
Tuple
>[]
collectors
=
new
StreamCollector
[
parallelismOfOutput
.
get
(
i
)];
StreamCollector
<
Tuple
>[]
collectors
=
new
StreamCollector
[
parallelismOfOutput
.
get
(
i
)];
for
(
int
j
=
0
;
j
<
collectors
.
length
;
j
++)
{
for
(
int
j
=
0
;
j
<
collectors
.
length
;
j
++)
{
List
<
RecordWriter
<
StreamRecord
>>
output
=
new
ArrayList
<
RecordWriter
<
StreamRecord
>>();
output
.
add
(
partitionedOutputs
.
get
(
i
));
collectors
[
j
]
=
new
StreamCollector
<
Tuple
>(
batchSizesOfPartitioned
.
get
(
i
),
collectors
[
j
]
=
new
StreamCollector
<
Tuple
>(
batchSizesOfPartitioned
.
get
(
i
),
batchTimeout
,
channelID
,
serializationDelegate
,
output
);
batchTimeout
,
channelID
,
serializationDelegate
,
partitionedOutputs
.
get
(
i
)
);
}
}
partitionedCollectors
.
add
(
collectors
);
partitionedCollectors
.
add
(
collectors
);
}
}
...
@@ -69,7 +67,7 @@ public class StreamCollectorManager<T extends Tuple> implements Collector<T> {
...
@@ -69,7 +67,7 @@ public class StreamCollectorManager<T extends Tuple> implements Collector<T> {
@Override
@Override
public
void
collect
(
T
tuple
)
{
public
void
collect
(
T
tuple
)
{
T
copiedTuple
=
StreamRecord
.
copyTuple
(
tuple
);
T
copiedTuple
=
StreamRecord
.
copyTuple
(
tuple
);
for
(
StreamCollector
<
Tuple
>
collector
:
notPartitionedCollectors
)
{
for
(
StreamCollector
<
Tuple
>
collector
:
notPartitionedCollectors
)
{
collector
.
collect
(
copiedTuple
);
collector
.
collect
(
copiedTuple
);
}
}
...
...
flink-addons/flink-streaming/src/main/java/eu/stratosphere/streaming/api/streamcomponent/StreamSource.java
浏览文件 @
8f77a938
...
@@ -21,13 +21,10 @@ import java.util.List;
...
@@ -21,13 +21,10 @@ import java.util.List;
import
org.apache.commons.logging.Log
;
import
org.apache.commons.logging.Log
;
import
org.apache.commons.logging.LogFactory
;
import
org.apache.commons.logging.LogFactory
;
import
eu.stratosphere.api.java.tuple.Tuple
;
import
eu.stratosphere.configuration.Configuration
;
import
eu.stratosphere.configuration.Configuration
;
import
eu.stratosphere.nephele.execution.Environment
;
import
eu.stratosphere.nephele.io.ChannelSelector
;
import
eu.stratosphere.nephele.io.ChannelSelector
;
import
eu.stratosphere.nephele.io.RecordWriter
;
import
eu.stratosphere.nephele.io.RecordWriter
;
import
eu.stratosphere.nephele.template.AbstractInputTask
;
import
eu.stratosphere.nephele.template.AbstractInputTask
;
import
eu.stratosphere.streaming.api.StreamCollector
;
import
eu.stratosphere.streaming.api.invokable.UserSourceInvokable
;
import
eu.stratosphere.streaming.api.invokable.UserSourceInvokable
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
import
eu.stratosphere.streaming.api.streamrecord.StreamRecord
;
import
eu.stratosphere.streaming.examples.DummyIS
;
import
eu.stratosphere.streaming.examples.DummyIS
;
...
...
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/FlatMapTest.java
浏览文件 @
8f77a938
...
@@ -15,19 +15,21 @@
...
@@ -15,19 +15,21 @@
package
eu.stratosphere.streaming.api
;
package
eu.stratosphere.streaming.api
;
import
static
org
.
junit
.
Assert
.
assertTrue
;
import
static
org
.
junit
.
Assert
.*;
import
static
org
.
junit
.
Assert
.
fail
;
import
java.io.ByteArrayInputStream
;
import
java.io.ByteArrayInputStream
;
import
java.io.ObjectInputStream
;
import
java.io.ObjectInputStream
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Set
;
import
org.junit.Test
;
import
org.junit.Test
;
import
eu.stratosphere.api.java.functions.FlatMapFunction
;
import
eu.stratosphere.api.java.functions.FlatMapFunction
;
import
eu.stratosphere.api.java.tuple.Tuple
;
import
eu.stratosphere.api.java.tuple.Tuple
;
import
eu.stratosphere.api.java.tuple.Tuple1
;
import
eu.stratosphere.api.java.tuple.Tuple1
;
import
eu.stratosphere.api.java.tuple.Tuple2
;
import
eu.stratosphere.api.java.typeutils.TupleTypeInfo
;
import
eu.stratosphere.api.java.typeutils.TupleTypeInfo
;
import
eu.stratosphere.api.java.typeutils.TypeExtractor
;
import
eu.stratosphere.api.java.typeutils.TypeExtractor
;
import
eu.stratosphere.configuration.Configuration
;
import
eu.stratosphere.configuration.Configuration
;
...
@@ -37,6 +39,7 @@ import eu.stratosphere.nephele.jobgraph.JobOutputVertex;
...
@@ -37,6 +39,7 @@ import eu.stratosphere.nephele.jobgraph.JobOutputVertex;
import
eu.stratosphere.nephele.jobgraph.JobTaskVertex
;
import
eu.stratosphere.nephele.jobgraph.JobTaskVertex
;
import
eu.stratosphere.streaming.api.MapTest.MyMap
;
import
eu.stratosphere.streaming.api.MapTest.MyMap
;
import
eu.stratosphere.streaming.api.MapTest.MySink
;
import
eu.stratosphere.streaming.api.MapTest.MySink
;
import
eu.stratosphere.streaming.api.PrintTest.MyFlatMap
;
import
eu.stratosphere.streaming.api.invokable.UserSinkInvokable
;
import
eu.stratosphere.streaming.api.invokable.UserSinkInvokable
;
import
eu.stratosphere.streaming.api.invokable.UserSourceInvokable
;
import
eu.stratosphere.streaming.api.invokable.UserSourceInvokable
;
import
eu.stratosphere.streaming.api.invokable.UserTaskInvokable
;
import
eu.stratosphere.streaming.api.invokable.UserTaskInvokable
;
...
@@ -47,57 +50,193 @@ public class FlatMapTest {
...
@@ -47,57 +50,193 @@ public class FlatMapTest {
public
static
final
class
MyFlatMap
extends
FlatMapFunction
<
Tuple1
<
Integer
>,
Tuple1
<
Integer
>>
{
public
static
final
class
MyFlatMap
extends
FlatMapFunction
<
Tuple1
<
Integer
>,
Tuple1
<
Integer
>>
{
@Override
@Override
public
void
flatMap
(
Tuple1
<
Integer
>
value
,
public
void
flatMap
(
Tuple1
<
Integer
>
value
,
Collector
<
Tuple1
<
Integer
>>
out
)
throws
Exception
{
Collector
<
Tuple1
<
Integer
>>
out
)
throws
Exception
{
out
.
collect
(
new
Tuple1
<
Integer
>(
value
.
f0
*
value
.
f0
));
out
.
collect
(
new
Tuple1
<
Integer
>(
value
.
f0
*
value
.
f0
));
}
}
}
public
static
final
class
ParallelFlatMap
extends
FlatMapFunction
<
Tuple1
<
Integer
>,
Tuple1
<
Integer
>>
{
@Override
public
void
flatMap
(
Tuple1
<
Integer
>
value
,
Collector
<
Tuple1
<
Integer
>>
out
)
throws
Exception
{
numberOfElements
++;
}
}
public
static
final
class
GenerateSequenceFlatMap
extends
FlatMapFunction
<
Tuple1
<
Long
>,
Tuple1
<
Long
>>
{
@Override
public
void
flatMap
(
Tuple1
<
Long
>
value
,
Collector
<
Tuple1
<
Long
>>
out
)
throws
Exception
{
out
.
collect
(
new
Tuple1
<
Long
>(
value
.
f0
*
value
.
f0
));
}
}
}
public
static
final
class
MySink
extends
SinkFunction
<
Tuple1
<
Integer
>>
{
public
static
final
class
MySink
extends
SinkFunction
<
Tuple1
<
Integer
>>
{
@Override
@Override
public
void
invoke
(
Tuple1
<
Integer
>
tuple
)
{
public
void
invoke
(
Tuple1
<
Integer
>
tuple
)
{
result
.
add
(
tuple
.
f0
);
result
.
add
(
tuple
.
f0
);
System
.
out
.
println
(
"result "
+
tuple
.
f0
);
}
}
}
}
public
static
final
class
MySource
extends
Source
Function
<
Tuple1
<
Integer
>>
{
public
static
final
class
FromElementsSink
extends
Sink
Function
<
Tuple1
<
Integer
>>
{
@Override
@Override
public
void
invoke
(
Collector
<
Tuple1
<
Integer
>>
collector
)
public
void
invoke
(
Tuple1
<
Integer
>
tuple
)
{
throws
Exception
{
fromElementsResult
.
add
(
tuple
.
f0
);
for
(
int
i
=
0
;
i
<
10
;
i
++){
}
collector
.
collect
(
new
Tuple1
<
Integer
>(
i
));
}
}
public
static
final
class
FromCollectionSink
extends
SinkFunction
<
Tuple1
<
Integer
>>
{
@Override
public
void
invoke
(
Tuple1
<
Integer
>
tuple
)
{
fromCollectionResult
.
add
(
tuple
.
f0
);
}
}
public
static
final
class
GenerateSequenceSink
extends
SinkFunction
<
Tuple1
<
Long
>>
{
@Override
public
void
invoke
(
Tuple1
<
Long
>
tuple
)
{
generateSequenceResult
.
add
(
tuple
.
f0
);
}
}
private
static
void
fillExpectedList
()
{
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
expected
.
add
(
i
*
i
);
}
}
private
static
void
fillFromElementsExpected
()
{
fromElementsExpected
.
add
(
4
);
fromElementsExpected
.
add
(
25
);
fromElementsExpected
.
add
(
81
);
}
private
static
void
fillSequenceSet
()
{
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
sequenceExpected
.
add
(
i
*
i
);
}
}
}
}
private
static
void
fillExpectedList
(){
private
static
void
fillLongSequenceSet
()
{
for
(
int
i
=
0
;
i
<
10
;
i
++){
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
expected
.
add
(
i
*
i
);
sequenceLongExpected
.
add
((
long
)(
i
*
i
));
}
}
private
static
void
fillFromCollectionSet
()
{
if
(
fromCollectionSet
.
isEmpty
()){
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
fromCollectionSet
.
add
(
i
);
}
}
}
}
}
private
static
final
int
PARALELISM
=
1
;
private
static
final
int
PARALELISM
=
1
;
private
static
List
<
Integer
>
expected
=
new
ArrayList
<
Integer
>();
private
static
int
numberOfElements
=
0
;
private
static
List
<
Integer
>
result
=
new
ArrayList
<
Integer
>();
private
static
Set
<
Integer
>
expected
=
new
HashSet
<
Integer
>();
private
static
Set
<
Integer
>
result
=
new
HashSet
<
Integer
>();
private
static
Set
<
Integer
>
fromElementsExpected
=
new
HashSet
<
Integer
>();
private
static
Set
<
Integer
>
fromElementsResult
=
new
HashSet
<
Integer
>();
private
static
Set
<
Integer
>
fromCollectionSet
=
new
HashSet
<
Integer
>();
private
static
Set
<
Integer
>
sequenceExpected
=
new
HashSet
<
Integer
>();
private
static
Set
<
Long
>
sequenceLongExpected
=
new
HashSet
<
Long
>();
private
static
Set
<
Integer
>
fromCollectionResult
=
new
HashSet
<
Integer
>();
private
static
Set
<
Long
>
generateSequenceResult
=
new
HashSet
<
Long
>();
@Test
@Test
public
void
test
()
throws
Exception
{
public
void
test
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
(
2
,
1000
);
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
(
2
,
1000
);
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
.
addSource
(
new
MySource
(),
1
).
flatMap
(
new
MyFlatMap
(),
PARALELISM
).
addSink
(
new
MySink
());
fillFromCollectionSet
();
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
.
fromCollection
(
fromCollectionSet
)
.
flatMap
(
new
MyFlatMap
(),
PARALELISM
).
addSink
(
new
MySink
());
env
.
execute
();
env
.
execute
();
fillExpectedList
();
fillExpectedList
();
assertTrue
(
expected
.
equals
(
result
));
assertTrue
(
expected
.
equals
(
result
));
}
@Test
public
void
parallelShuffleconnectTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
fillFromCollectionSet
();
DataStream
<
Tuple1
<
Integer
>>
source
=
env
.
fromCollection
(
fromCollectionSet
);
DataStream
<
Tuple1
<
Integer
>>
map
=
source
.
flatMap
(
new
ParallelFlatMap
(),
1
).
addSink
(
new
MySink
());
DataStream
<
Tuple1
<
Integer
>>
map2
=
source
.
flatMap
(
new
ParallelFlatMap
(),
1
).
addSink
(
new
MySink
());
env
.
execute
();
assertEquals
(
20
,
numberOfElements
);
numberOfElements
=
0
;
}
@Test
public
void
fromElementsTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
DataStream
<
Tuple1
<
Integer
>>
map
=
env
.
fromElements
(
2
,
5
,
9
).
flatMap
(
new
MyFlatMap
(),
1
);
DataStream
<
Tuple1
<
Integer
>>
sink
=
map
.
addSink
(
new
FromElementsSink
());
fillFromElementsExpected
();
env
.
execute
();
assertEquals
(
fromElementsExpected
,
fromElementsResult
);
}
@Test
public
void
fromCollectionTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
fillFromCollectionSet
();
DataStream
<
Tuple1
<
Integer
>>
map
=
env
.
fromCollection
(
fromCollectionSet
).
flatMap
(
new
MyFlatMap
(),
1
);
DataStream
<
Tuple1
<
Integer
>>
sink
=
map
.
addSink
(
new
FromCollectionSink
());
fillSequenceSet
();
env
.
execute
();
assertEquals
(
sequenceExpected
,
fromCollectionResult
);
}
@Test
public
void
generateSequenceTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
DataStream
<
Tuple1
<
Long
>>
map
=
env
.
generateSequence
(
0
,
9
).
flatMap
(
new
GenerateSequenceFlatMap
(),
1
);
DataStream
<
Tuple1
<
Long
>>
sink
=
map
.
addSink
(
new
GenerateSequenceSink
());
fillLongSequenceSet
();
env
.
execute
();
assertEquals
(
sequenceLongExpected
,
generateSequenceResult
);
}
}
}
}
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/MapTest.java
浏览文件 @
8f77a938
...
@@ -18,7 +18,9 @@ package eu.stratosphere.streaming.api;
...
@@ -18,7 +18,9 @@ package eu.stratosphere.streaming.api;
import
static
org
.
junit
.
Assert
.*;
import
static
org
.
junit
.
Assert
.*;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Set
;
import
org.junit.Test
;
import
org.junit.Test
;
...
@@ -33,27 +35,6 @@ public class MapTest {
...
@@ -33,27 +35,6 @@ public class MapTest {
@Override
@Override
public
void
invoke
(
Collector
<
Tuple1
<
Integer
>>
collector
)
throws
Exception
{
public
void
invoke
(
Collector
<
Tuple1
<
Integer
>>
collector
)
throws
Exception
{
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
System
.
out
.
println
(
"source "
+
i
);
collector
.
collect
(
new
Tuple1
<
Integer
>(
i
));
}
}
}
public
static
final
class
MyFieldsSource
extends
SourceFunction
<
Tuple1
<
Integer
>>
{
@Override
public
void
invoke
(
Collector
<
Tuple1
<
Integer
>>
collector
)
throws
Exception
{
for
(
int
i
=
0
;
i
<
MAXSOURCE
;
i
++)
{
collector
.
collect
(
new
Tuple1
<
Integer
>(
5
));
}
}
}
public
static
final
class
MyDiffFieldsSource
extends
SourceFunction
<
Tuple1
<
Integer
>>
{
@Override
public
void
invoke
(
Collector
<
Tuple1
<
Integer
>>
collector
)
throws
Exception
{
for
(
int
i
=
0
;
i
<
9
;
i
++)
{
collector
.
collect
(
new
Tuple1
<
Integer
>(
i
));
collector
.
collect
(
new
Tuple1
<
Integer
>(
i
));
}
}
}
}
...
@@ -63,7 +44,6 @@ public class MapTest {
...
@@ -63,7 +44,6 @@ public class MapTest {
@Override
@Override
public
Tuple1
<
Integer
>
map
(
Tuple1
<
Integer
>
value
)
throws
Exception
{
public
Tuple1
<
Integer
>
map
(
Tuple1
<
Integer
>
value
)
throws
Exception
{
System
.
out
.
println
(
"mymap "
+
map
);
map
++;
map
++;
return
new
Tuple1
<
Integer
>(
value
.
f0
*
value
.
f0
);
return
new
Tuple1
<
Integer
>(
value
.
f0
*
value
.
f0
);
}
}
...
@@ -76,7 +56,6 @@ public class MapTest {
...
@@ -76,7 +56,6 @@ public class MapTest {
@Override
@Override
public
Tuple1
<
Integer
>
map
(
Tuple1
<
Integer
>
value
)
throws
Exception
{
public
Tuple1
<
Integer
>
map
(
Tuple1
<
Integer
>
value
)
throws
Exception
{
counter
++;
counter
++;
if
(
counter
==
MAXSOURCE
)
if
(
counter
==
MAXSOURCE
)
allInOne
=
true
;
allInOne
=
true
;
return
new
Tuple1
<
Integer
>(
value
.
f0
*
value
.
f0
);
return
new
Tuple1
<
Integer
>(
value
.
f0
*
value
.
f0
);
...
@@ -140,13 +119,12 @@ public class MapTest {
...
@@ -140,13 +119,12 @@ public class MapTest {
@Override
@Override
public
void
invoke
(
Tuple1
<
Integer
>
tuple
)
{
public
void
invoke
(
Tuple1
<
Integer
>
tuple
)
{
System
.
out
.
println
(
"sink "
+
graphResult
);
graphResult
++;
graphResult
++;
}
}
}
}
private
static
List
<
Integer
>
expected
=
new
ArrayLis
t
<
Integer
>();
private
static
Set
<
Integer
>
expected
=
new
HashSe
t
<
Integer
>();
private
static
List
<
Integer
>
result
=
new
ArrayLis
t
<
Integer
>();
private
static
Set
<
Integer
>
result
=
new
HashSe
t
<
Integer
>();
private
static
int
broadcastResult
=
0
;
private
static
int
broadcastResult
=
0
;
private
static
int
shuffleResult
=
0
;
private
static
int
shuffleResult
=
0
;
private
static
int
fieldsResult
=
0
;
private
static
int
fieldsResult
=
0
;
...
@@ -157,19 +135,49 @@ public class MapTest {
...
@@ -157,19 +135,49 @@ public class MapTest {
private
static
final
int
MAXSOURCE
=
10
;
private
static
final
int
MAXSOURCE
=
10
;
private
static
boolean
allInOne
=
false
;
private
static
boolean
allInOne
=
false
;
private
static
boolean
threeInAll
=
true
;
private
static
boolean
threeInAll
=
true
;
private
static
Set
<
Integer
>
fromCollectionSet
=
new
HashSet
<
Integer
>();
private
static
List
<
Integer
>
fromCollectionFields
=
new
ArrayList
<
Integer
>();
private
static
Set
<
Integer
>
fromCollectionDiffFieldsSet
=
new
HashSet
<
Integer
>();
private
static
void
fillExpectedList
()
{
private
static
void
fillExpectedList
()
{
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
expected
.
add
(
i
*
i
);
expected
.
add
(
i
*
i
);
}
}
}
}
private
static
void
fillFromCollectionSet
()
{
if
(
fromCollectionSet
.
isEmpty
()){
for
(
int
i
=
0
;
i
<
10
;
i
++)
{
fromCollectionSet
.
add
(
i
);
}
}
}
private
static
void
fillFromCollectionFieldsSet
()
{
if
(
fromCollectionFields
.
isEmpty
()){
for
(
int
i
=
0
;
i
<
MAXSOURCE
;
i
++)
{
fromCollectionFields
.
add
(
5
);
}
}
}
private
static
void
fillFromCollectionDiffFieldsSet
()
{
if
(
fromCollectionDiffFieldsSet
.
isEmpty
()){
for
(
int
i
=
0
;
i
<
9
;
i
++)
{
fromCollectionDiffFieldsSet
.
add
(
i
);
}
}
}
@Test
@Test
public
void
mapTest
()
throws
Exception
{
public
void
mapTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
.
addSource
(
new
MySource
(),
1
)
fillFromCollectionSet
();
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
.
fromCollection
(
fromCollectionSet
)
.
map
(
new
MyMap
(),
PARALELISM
).
addSink
(
new
MySink
());
.
map
(
new
MyMap
(),
PARALELISM
).
addSink
(
new
MySink
());
env
.
execute
();
env
.
execute
();
...
@@ -182,8 +190,11 @@ public class MapTest {
...
@@ -182,8 +190,11 @@ public class MapTest {
@Test
@Test
public
void
broadcastSinkTest
()
throws
Exception
{
public
void
broadcastSinkTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
fillFromCollectionSet
();
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
.
addSource
(
new
MySource
(),
1
)
.
fromCollection
(
fromCollectionSet
)
.
broadcast
()
.
broadcast
()
.
map
(
new
MyMap
(),
3
)
.
map
(
new
MyMap
(),
3
)
.
addSink
(
new
MyBroadcastSink
());
.
addSink
(
new
MyBroadcastSink
());
...
@@ -196,8 +207,11 @@ public class MapTest {
...
@@ -196,8 +207,11 @@ public class MapTest {
@Test
@Test
public
void
shuffleSinkTest
()
throws
Exception
{
public
void
shuffleSinkTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
fillFromCollectionSet
();
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
.
addSource
(
new
MySource
(),
1
)
.
fromCollection
(
fromCollectionSet
)
.
map
(
new
MyMap
(),
3
)
.
map
(
new
MyMap
(),
3
)
.
addSink
(
new
MyShufflesSink
());
.
addSink
(
new
MyShufflesSink
());
env
.
execute
();
env
.
execute
();
...
@@ -222,8 +236,11 @@ public class MapTest {
...
@@ -222,8 +236,11 @@ public class MapTest {
@Test
@Test
public
void
fieldsMapTest
()
throws
Exception
{
public
void
fieldsMapTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
fillFromCollectionFieldsSet
();
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
.
addSource
(
new
MyFieldsSource
(),
1
)
.
fromCollection
(
fromCollectionFields
)
.
partitionBy
(
0
)
.
partitionBy
(
0
)
.
map
(
new
MyFieldsMap
(),
3
)
.
map
(
new
MyFieldsMap
(),
3
)
.
addSink
(
new
MyFieldsSink
());
.
addSink
(
new
MyFieldsSink
());
...
@@ -236,8 +253,11 @@ public class MapTest {
...
@@ -236,8 +253,11 @@ public class MapTest {
@Test
@Test
public
void
diffFieldsMapTest
()
throws
Exception
{
public
void
diffFieldsMapTest
()
throws
Exception
{
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
StreamExecutionEnvironment
env
=
new
StreamExecutionEnvironment
();
fillFromCollectionDiffFieldsSet
();
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
DataStream
<
Tuple1
<
Integer
>>
dataStream
=
env
.
addSource
(
new
MyDiffFieldsSource
(),
1
)
.
fromCollection
(
fromCollectionDiffFieldsSet
)
.
partitionBy
(
0
)
.
partitionBy
(
0
)
.
map
(
new
MyDiffFieldsMap
(),
3
)
.
map
(
new
MyDiffFieldsMap
(),
3
)
.
addSink
(
new
MyDiffFieldsSink
());
.
addSink
(
new
MyDiffFieldsSink
());
...
...
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/PrintTest.java
浏览文件 @
8f77a938
...
@@ -53,4 +53,6 @@ public class PrintTest {
...
@@ -53,4 +53,6 @@ public class PrintTest {
env
.
execute
();
env
.
execute
();
}
}
}
}
flink-addons/flink-streaming/src/test/java/eu/stratosphere/streaming/api/StreamCollectorTest.java
浏览文件 @
8f77a938
...
@@ -65,11 +65,8 @@ public class StreamCollectorTest {
...
@@ -65,11 +65,8 @@ public class StreamCollectorTest {
@Test
@Test
public
void
recordWriter
()
{
public
void
recordWriter
()
{
MockRecordWriter
recWriter
=
MockRecordWriterFactory
.
create
();
MockRecordWriter
recWriter
=
MockRecordWriterFactory
.
create
();
ArrayList
<
RecordWriter
<
StreamRecord
>>
rwList
=
new
ArrayList
<
RecordWriter
<
StreamRecord
>>();
rwList
.
add
(
recWriter
);
StreamCollector
collector
=
new
StreamCollector
(
2
,
1000
,
0
,
null
,
r
wList
);
StreamCollector
collector
=
new
StreamCollector
(
2
,
1000
,
0
,
null
,
r
ecWriter
);
collector
.
collect
(
new
Tuple1
<
Integer
>(
3
));
collector
.
collect
(
new
Tuple1
<
Integer
>(
3
));
collector
.
collect
(
new
Tuple1
<
Integer
>(
4
));
collector
.
collect
(
new
Tuple1
<
Integer
>(
4
));
collector
.
collect
(
new
Tuple1
<
Integer
>(
5
));
collector
.
collect
(
new
Tuple1
<
Integer
>(
5
));
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录