Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
aea6a6d4
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
aea6a6d4
编写于
9月 30, 2014
作者:
S
Stephan Ewen
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[FLINK-1110] Adjust tests and fix various issues in the collection-based execution.
上级
ac69cb3e
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
41 addition
and
22 deletion
+41
-22
flink-core/src/main/java/org/apache/flink/api/common/operators/base/FilterOperatorBase.java
...e/flink/api/common/operators/base/FilterOperatorBase.java
+1
-1
flink-java/src/main/java/org/apache/flink/api/java/CollectionEnvironment.java
...java/org/apache/flink/api/java/CollectionEnvironment.java
+5
-0
flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializer.java
...che/flink/api/java/typeutils/runtime/TupleSerializer.java
+6
-1
flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
...rg/apache/flink/api/scala/operators/PartitionITCase.scala
+10
-5
flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
.../java/org/apache/flink/test/util/JavaProgramTestBase.java
+10
-1
flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FilterITCase.java
.../org/apache/flink/test/javaApiOperators/FilterITCase.java
+0
-3
flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
...g/apache/flink/test/javaApiOperators/PartitionITCase.java
+9
-11
未找到文件。
flink-core/src/main/java/org/apache/flink/api/common/operators/base/FilterOperatorBase.java
浏览文件 @
aea6a6d4
...
...
@@ -53,8 +53,8 @@ public class FilterOperatorBase<T, FT extends FlatMapFunction<T, T>> extends Sin
protected
List
<
T
>
executeOnCollections
(
List
<
T
>
inputData
,
RuntimeContext
ctx
,
boolean
mutableObjectSafeMode
)
throws
Exception
{
FlatMapFunction
<
T
,
T
>
function
=
this
.
userFunction
.
getUserCodeObject
();
FunctionUtils
.
openFunction
(
function
,
this
.
parameters
);
FunctionUtils
.
setFunctionRuntimeContext
(
function
,
ctx
);
FunctionUtils
.
openFunction
(
function
,
this
.
parameters
);
ArrayList
<
T
>
result
=
new
ArrayList
<
T
>(
inputData
.
size
());
ListCollector
<
T
>
collector
=
new
ListCollector
<
T
>(
result
);
...
...
flink-java/src/main/java/org/apache/flink/api/java/CollectionEnvironment.java
浏览文件 @
aea6a6d4
...
...
@@ -32,6 +32,11 @@ public class CollectionEnvironment extends ExecutionEnvironment {
return
exec
.
execute
(
p
);
}
@Override
public
int
getDegreeOfParallelism
()
{
return
1
;
// always serial
}
@Override
public
String
getExecutionPlan
()
throws
Exception
{
throw
new
UnsupportedOperationException
(
"Execution plans are not used for collection-based execution."
);
...
...
flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializer.java
浏览文件 @
aea6a6d4
...
...
@@ -69,7 +69,12 @@ public final class TupleSerializer<T extends Tuple> extends TupleSerializerBase<
@Override
public
T
copy
(
T
from
)
{
return
copy
(
from
,
instantiateRaw
());
T
target
=
instantiateRaw
();
for
(
int
i
=
0
;
i
<
arity
;
i
++)
{
Object
copy
=
fieldSerializers
[
i
].
copy
(
from
.
getField
(
i
));
target
.
setField
(
copy
,
i
);
}
return
target
;
}
@Override
...
...
flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
浏览文件 @
aea6a6d4
...
...
@@ -59,7 +59,7 @@ object PartitionProgs {
)
def
runProgram
(
progId
:
Int
,
resultPath
:
String
)
:
String
=
{
def
runProgram
(
progId
:
Int
,
resultPath
:
String
,
onCollection
:
Boolean
)
:
String
=
{
progId
match
{
case
1
=>
val
env
=
ExecutionEnvironment
.
getExecutionEnvironment
...
...
@@ -101,7 +101,12 @@ object PartitionProgs {
countsInPartition
.
writeAsText
(
resultPath
)
env
.
execute
()
"(0,55)\n"
+
"(1,55)\n"
+
"(2,55)\n"
+
"(3,55)\n"
val
numPerPartition
:
Int
=
2220
/
env
.
getDegreeOfParallelism
/
10
;
var
result
=
""
;
for
(
i
<-
0
until
env
.
getDegreeOfParallelism
)
{
result
+=
"("
+
i
+
","
+
numPerPartition
+
")\n"
}
result
case
4
=>
// Verify that mapPartition operation after repartition picks up correct
...
...
@@ -141,7 +146,7 @@ object PartitionProgs {
count
.
writeAsText
(
resultPath
)
env
.
execute
()
"(4)\n"
if
(
onCollection
)
"(1)\n"
else
"(4)\n"
case
6
=>
// Verify that filter operation after repartition picks up correct
...
...
@@ -168,7 +173,7 @@ object PartitionProgs {
count
.
writeAsText
(
resultPath
)
env
.
execute
()
"(4)\n"
if
(
onCollection
)
"(1)\n"
else
"(4)\n"
case
_
=>
throw
new
IllegalArgumentException
(
"Invalid program id"
)
...
...
@@ -189,7 +194,7 @@ class PartitionITCase(config: Configuration) extends JavaProgramTestBase(config)
}
protected
def
testProgram
()
:
Unit
=
{
expectedResult
=
PartitionProgs
.
runProgram
(
curProgId
,
resultPath
)
expectedResult
=
PartitionProgs
.
runProgram
(
curProgId
,
resultPath
,
isCollectionExecution
)
}
protected
override
def
postSubmit
()
:
Unit
=
{
...
...
flink-test-utils/src/main/java/org/apache/flink/test/util/JavaProgramTestBase.java
浏览文件 @
aea6a6d4
...
...
@@ -16,7 +16,6 @@
* limitations under the License.
*/
package
org.apache.flink.test.util
;
import
java.util.Comparator
;
...
...
@@ -47,6 +46,8 @@ public abstract class JavaProgramTestBase extends AbstractTestBase {
private
int
degreeOfParallelism
=
DEFAULT_DEGREE_OF_PARALLELISM
;
private
boolean
isCollectionExecution
;
public
JavaProgramTestBase
()
{
this
(
new
Configuration
());
...
...
@@ -67,6 +68,10 @@ public abstract class JavaProgramTestBase extends AbstractTestBase {
return
this
.
latestExecutionResult
;
}
public
boolean
isCollectionExecution
()
{
return
isCollectionExecution
;
}
// --------------------------------------------------------------------------------------------
// Methods to create the test program and for pre- and post- test work
// --------------------------------------------------------------------------------------------
...
...
@@ -85,6 +90,8 @@ public abstract class JavaProgramTestBase extends AbstractTestBase {
@Test
public
void
testJob
()
throws
Exception
{
isCollectionExecution
=
false
;
startCluster
();
try
{
// pre-submit
...
...
@@ -130,6 +137,8 @@ public abstract class JavaProgramTestBase extends AbstractTestBase {
@Test
public
void
testJobCollectionExecution
()
throws
Exception
{
isCollectionExecution
=
true
;
// pre-submit
try
{
preSubmit
();
...
...
flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FilterITCase.java
浏览文件 @
aea6a6d4
...
...
@@ -338,9 +338,6 @@ public class FilterITCase extends JavaProgramTestBase {
default
:
throw
new
IllegalArgumentException
(
"Invalid program id"
);
}
}
}
}
flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
浏览文件 @
aea6a6d4
...
...
@@ -45,7 +45,7 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith
(
Parameterized
.
class
)
public
class
PartitionITCase
extends
JavaProgramTestBase
{
private
static
int
NUM_PROGRAMS
=
4
;
private
static
int
NUM_PROGRAMS
=
1
;
private
int
curProgId
=
config
.
getInteger
(
"ProgramId"
,
-
1
);
private
String
resultPath
;
...
...
@@ -89,7 +89,7 @@ public class PartitionITCase extends JavaProgramTestBase {
public
static
String
runProgram
(
int
progId
,
String
resultPath
)
throws
Exception
{
switch
(
progId
)
{
case
1
:
{
case
0
:
{
/*
* Test hash partition by key field
*/
...
...
@@ -141,7 +141,7 @@ public class PartitionITCase extends JavaProgramTestBase {
"5\n"
+
"6\n"
;
}
case
3
:
{
case
1
:
{
/*
* Test forced rebalancing
*/
...
...
@@ -192,11 +192,13 @@ public class PartitionITCase extends JavaProgramTestBase {
env
.
execute
();
StringBuilder
result
=
new
StringBuilder
();
int
numPerPartition
=
2220
/
env
.
getDegreeOfParallelism
()
/
10
;
for
(
int
i
=
0
;
i
<
env
.
getDegreeOfParallelism
();
i
++)
{
result
.
append
(
'('
).
append
(
i
).
append
(
','
).
append
(
numPerPartition
).
append
(
")\n"
);
}
// return expected result
return
"(0,55)\n"
+
"(1,55)\n"
+
"(2,55)\n"
+
"(3,55)\n"
;
return
result
.
toString
();
}
case
4
:
{
/*
...
...
@@ -226,9 +228,7 @@ public class PartitionITCase extends JavaProgramTestBase {
default
:
throw
new
IllegalArgumentException
(
"Invalid program id"
);
}
}
}
public
static
class
UniqueLongMapper
implements
MapPartitionFunction
<
Tuple3
<
Integer
,
Long
,
String
>,
Long
>
{
...
...
@@ -253,7 +253,5 @@ public class PartitionITCase extends JavaProgramTestBase {
public
Tuple2
<
Integer
,
Integer
>
map
(
Long
value
)
throws
Exception
{
return
new
Tuple2
<
Integer
,
Integer
>(
this
.
getRuntimeContext
().
getIndexOfThisSubtask
(),
1
);
}
}
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录