Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
95d035ab
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
95d035ab
编写于
9月 09, 2015
作者:
S
Stephan Ewen
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[FLINK-2619] [tests] Fix failing ExecutionGraphRestartTest and JobManagerRegistrationTest
上级
c9edd9a8
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
138 addition
and
118 deletion
+138
-118
flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
...g/apache/flink/runtime/executiongraph/ExecutionGraph.java
+18
-13
flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
...flink/runtime/executiongraph/ExecutionGraphTestUtils.java
+4
-0
flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.scala
...nk/runtime/executiongraph/ExecutionGraphRestartTest.scala
+23
-11
flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
...flink/runtime/jobmanager/JobManagerRegistrationTest.scala
+93
-94
未找到文件。
flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
浏览文件 @
95d035ab
...
...
@@ -857,20 +857,25 @@ public class ExecutionGraph implements Serializable {
else
if
(
current
==
JobStatus
.
FAILING
)
{
if
(
numberOfRetriesLeft
>
0
&&
transitionState
(
current
,
JobStatus
.
RESTARTING
))
{
numberOfRetriesLeft
--;
future
(
new
Callable
<
Object
>()
{
@Override
public
Object
call
()
throws
Exception
{
try
{
LOG
.
info
(
"Delaying retry of job execution for {} ms ..."
,
delayBeforeRetrying
);
Thread
.
sleep
(
delayBeforeRetrying
);
if
(
delayBeforeRetrying
>
0
)
{
future
(
new
Callable
<
Object
>()
{
@Override
public
Object
call
()
throws
Exception
{
try
{
LOG
.
info
(
"Delaying retry of job execution for {} ms ..."
,
delayBeforeRetrying
);
Thread
.
sleep
(
delayBeforeRetrying
);
}
catch
(
InterruptedException
e
){
// should only happen on shutdown
}
restart
();
return
null
;
}
catch
(
InterruptedException
e
){
// should only happen on shutdown
}
restart
();
return
null
;
}
},
executionContext
);
},
executionContext
);
}
else
{
restart
();
}
break
;
}
else
if
(
numberOfRetriesLeft
<=
0
&&
transitionState
(
current
,
JobStatus
.
FAILED
,
failureCause
))
{
...
...
flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java
浏览文件 @
95d035ab
...
...
@@ -112,7 +112,9 @@ public class ExecutionGraphTestUtils {
return
new
Instance
(
gateway
,
connection
,
new
InstanceID
(),
hardwareDescription
,
numberOfSlots
);
}
@SuppressWarnings
(
"serial"
)
public
static
class
SimpleActorGateway
extends
BaseTestingActorGateway
{
public
TaskDeploymentDescriptor
lastTDD
;
public
SimpleActorGateway
(
ExecutionContext
executionContext
){
...
...
@@ -139,7 +141,9 @@ public class ExecutionGraphTestUtils {
}
}
@SuppressWarnings
(
"serial"
)
public
static
class
SimpleFailingActorGateway
extends
BaseTestingActorGateway
{
public
SimpleFailingActorGateway
(
ExecutionContext
executionContext
)
{
super
(
executionContext
);
}
...
...
flink-runtime/src/test/scala/org/apache/flink/runtime/executiongraph/ExecutionGraphRestartTest.scala
浏览文件 @
95d035ab
...
...
@@ -26,10 +26,13 @@ import org.apache.flink.runtime.jobgraph.{JobStatus, JobGraph, JobVertex}
import
org.apache.flink.runtime.jobmanager.Tasks
import
org.apache.flink.runtime.jobmanager.scheduler.Scheduler
import
org.apache.flink.runtime.testingUtils.TestingUtils
import
org.junit.runner.RunWith
import
org.scalatest.junit.JUnitRunner
import
org.scalatest.
{
Matchers
,
WordSpecLike
}
import
scala.collection.JavaConverters._
@RunWith
(
classOf
[
JUnitRunner
])
class
ExecutionGraphRestartTest
extends
WordSpecLike
with
Matchers
{
...
...
@@ -39,7 +42,8 @@ class ExecutionGraphRestartTest extends WordSpecLike with Matchers {
"be manually restartable"
in
{
try
{
val
instance
=
ExecutionGraphTestUtils
.
getInstance
(
new
SimpleActorGateway
(
TestingUtils
.
directExecutionContext
))
new
SimpleActorGateway
(
TestingUtils
.
directExecutionContext
),
NUM_TASKS
)
val
scheduler
=
new
Scheduler
(
TestingUtils
.
defaultExecutionContext
)
scheduler
.
newInstanceAvailable
(
instance
)
...
...
@@ -65,14 +69,18 @@ class ExecutionGraphRestartTest extends WordSpecLike with Matchers {
eg
.
getState
should
equal
(
JobStatus
.
RUNNING
)
eg
.
getAllExecutionVertices
.
iterator
().
next
().
fail
(
new
Exception
(
"Test Exception"
))
for
(
vertex
<-
eg
.
getAllExecutionVertices
().
asScala
)
{
vertex
.
getCurrentExecutionAttempt
().
cancelingComplete
()
}
eg
.
getState
should
equal
(
JobStatus
.
FAILED
)
eg
.
restart
()
eg
.
getState
should
equal
(
JobStatus
.
RUNNING
)
import
collection.JavaConverters._
for
(
vertex
<-
eg
.
getAllExecutionVertices
.
asScala
)
{
vertex
.
execution
Finished
()
vertex
.
getCurrentExecutionAttempt
().
mark
Finished
()
}
eg
.
getState
should
equal
(
JobStatus
.
FINISHED
)
...
...
@@ -86,7 +94,8 @@ class ExecutionGraphRestartTest extends WordSpecLike with Matchers {
"restart itself automatically"
in
{
try
{
val
instance
=
ExecutionGraphTestUtils
.
getInstance
(
new
SimpleActorGateway
(
TestingUtils
.
directExecutionContext
))
new
SimpleActorGateway
(
TestingUtils
.
directExecutionContext
),
NUM_TASKS
)
val
scheduler
=
new
Scheduler
(
TestingUtils
.
defaultExecutionContext
)
scheduler
.
newInstanceAvailable
(
instance
)
...
...
@@ -112,15 +121,19 @@ class ExecutionGraphRestartTest extends WordSpecLike with Matchers {
eg
.
getState
should
equal
(
JobStatus
.
RUNNING
)
eg
.
getAllExecutionVertices
.
iterator
().
next
().
fail
(
new
Exception
(
"Test Exception"
))
eg
.
getState
should
equal
(
JobStatus
.
FAILING
)
for
(
vertex
<-
eg
.
getAllExecutionVertices
.
asScala
)
{
vertex
.
getCurrentExecutionAttempt
().
cancelingComplete
()
}
eg
.
getState
should
equal
(
JobStatus
.
RUNNING
)
import
collection.JavaConverters._
for
(
vertex
<-
eg
.
getAllExecutionVertices
.
asScala
)
{
vertex
.
execution
Finished
()
vertex
.
getCurrentExecutionAttempt
().
mark
Finished
()
}
eg
.
getState
should
equal
(
JobStatus
.
FINISHED
)
eg
.
getState
()
should
equal
(
JobStatus
.
FINISHED
)
}
catch
{
case
t
:
Throwable
=>
t
.
printStackTrace
()
...
...
@@ -128,5 +141,4 @@ class ExecutionGraphRestartTest extends WordSpecLike with Matchers {
}
}
}
}
flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/JobManagerRegistrationTest.scala
浏览文件 @
95d035ab
...
...
@@ -19,18 +19,21 @@
package
org.apache.flink.runtime.jobmanager
import
java.net.InetAddress
import
java.util.UUID
import
akka.actor._
import
akka.testkit.
{
ImplicitSender
,
TestKit
}
import
org.apache.flink.configuration.Configuration
import
org.apache.flink.runtime.StreamingMode
import
org.apache.flink.runtime.akka.AkkaUtils
import
org.apache.flink.runtime.instance.
{
HardwareDescription
,
InstanceConnectionInfo
,
InstanceID
}
import
org.apache.flink.runtime.instance._
import
org.apache.flink.runtime.jobmanager.JobManagerRegistrationTest.PlainForwardingActor
import
org.apache.flink.runtime.messages.JobManagerMessages.LeaderSessionMessage
import
org.apache.flink.runtime.messages.RegistrationMessages.
{
AcknowledgeRegistration
,
AlreadyRegistered
,
RegisterTaskManager
}
import
org.junit.Assert.
{
assertNotEquals
,
assertNotNull
}
import
org.junit.runner.RunWith
import
org.scalatest.junit.JUnitRunner
import
org.scalatest.
{
BeforeAndAfterAll
,
Matchers
,
WordSpecLike
}
...
...
@@ -56,123 +59,119 @@ ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
"assign a TaskManager a unique instance ID"
in
{
val
jm
=
startTestingJobManager
(
_system
)
val
tmDummy1
=
_system
.
actorOf
(
Props
(
classOf
[
JobManagerRegistrationTest.DummyActor
]))
val
tmDummy2
=
_system
.
actorOf
(
Props
(
classOf
[
JobManagerRegistrationTest.DummyActor
]))
try
{
val
connectionInfo1
=
new
InstanceConnectionInfo
(
InetAddress
.
getLocalHost
,
10000
)
val
connectionInfo2
=
new
InstanceConnectionInfo
(
InetAddress
.
getLocalHost
,
10001
)
val
hardwareDescription
=
HardwareDescription
.
extractFromSystem
(
10
)
val
leaderSessionID
=
UUID
.
randomUUID
()
var
id1
:
InstanceID
=
null
var
id2
:
InstanceID
=
null
// task manager 1
within
(
1
second
)
{
jm
.
tell
(
RegisterTaskManager
(
connectionInfo1
,
hardwareDescription
,
1
),
tmDummy1
)
val
response
=
receiveOne
(
1
second
)
response
match
{
case
LeaderSessionMessage
(
leaderSessionID
,
AcknowledgeRegistration
(
id
,
_
))
=>
id1
=
id
case
_
=>
fail
(
"Wrong response message: "
+
response
)
}
val
tm1
=
_system
.
actorOf
(
Props
(
new
PlainForwardingActor
(
testActor
)))
val
tm2
=
_system
.
actorOf
(
Props
(
new
PlainForwardingActor
(
testActor
)))
val
connectionInfo1
=
new
InstanceConnectionInfo
(
InetAddress
.
getLocalHost
,
10000
)
val
connectionInfo2
=
new
InstanceConnectionInfo
(
InetAddress
.
getLocalHost
,
10001
)
val
hardwareDescription
=
HardwareDescription
.
extractFromSystem
(
10
)
var
id1
:
InstanceID
=
null
var
id2
:
InstanceID
=
null
// task manager 1
within
(
1
second
)
{
jm
.
tell
(
RegisterTaskManager
(
connectionInfo1
,
hardwareDescription
,
1
),
new
AkkaActorGateway
(
tm1
,
null
))
val
response
=
expectMsgType
[
LeaderSessionMessage
]
response
match
{
case
LeaderSessionMessage
(
_
,
AcknowledgeRegistration
(
id
,
_
))
=>
id1
=
id
case
_
=>
fail
(
"Wrong response message: "
+
response
)
}
}
// task manager 2
within
(
1
second
)
{
jm
.
tell
(
RegisterTaskManager
(
connectionInfo2
,
hardwareDescription
,
1
),
tmDummy2
)
val
response
=
receiveOne
(
1
second
)
response
match
{
case
LeaderSessionMessage
(
leaderSessionID
,
AcknowledgeRegistration
(
id
,
_
))
=>
id2
=
id
case
_
=>
fail
(
"Wrong response message: "
+
response
)
}
// task manager 2
within
(
1
second
)
{
jm
.
tell
(
RegisterTaskManager
(
connectionInfo2
,
hardwareDescription
,
1
),
new
AkkaActorGateway
(
tm2
,
null
))
val
response
=
expectMsgType
[
LeaderSessionMessage
]
response
match
{
case
LeaderSessionMessage
(
leaderSessionID
,
AcknowledgeRegistration
(
id
,
_
))
=>
id2
=
id
case
_
=>
fail
(
"Wrong response message: "
+
response
)
}
assertNotNull
(
id1
)
assertNotNull
(
id2
)
assertNotEquals
(
id1
,
id2
)
}
finally
{
tmDummy1
!
Kill
tmDummy2
!
Kill
jm
!
Kill
}
assertNotNull
(
id1
)
assertNotNull
(
id2
)
assertNotEquals
(
id1
,
id2
)
}
"handle repeated registration calls"
in
{
val
jm
=
startTestingJobManager
(
_system
)
val
tmDummy
=
_system
.
actorOf
(
Props
(
classOf
[
JobManagerRegistrationTest.DummyActor
]))
try
{
val
connectionInfo
=
new
InstanceConnectionInfo
(
InetAddress
.
getLocalHost
,
1
)
val
hardwareDescription
=
HardwareDescription
.
extractFromSystem
(
10
)
within
(
1
second
)
{
jm
.
tell
(
RegisterTaskManager
(
connectionInfo
,
hardwareDescription
,
1
),
tmDummy
)
jm
.
tell
(
RegisterTaskManager
(
connectionInfo
,
hardwareDescription
,
1
),
tmDummy
)
jm
.
tell
(
RegisterTaskManager
(
connectionInfo
,
hardwareDescription
,
1
),
tmDummy
)
expectMsgType
[
AcknowledgeRegistration
]
expectMsgType
[
AlreadyRegistered
]
expectMsgType
[
AlreadyRegistered
]
val
selfGateway
=
new
AkkaActorGateway
(
testActor
,
null
)
val
connectionInfo
=
new
InstanceConnectionInfo
(
InetAddress
.
getLocalHost
,
1
)
val
hardwareDescription
=
HardwareDescription
.
extractFromSystem
(
10
)
within
(
5
second
)
{
jm
.
tell
(
RegisterTaskManager
(
connectionInfo
,
hardwareDescription
,
1
),
selfGateway
)
jm
.
tell
(
RegisterTaskManager
(
connectionInfo
,
hardwareDescription
,
1
),
selfGateway
)
jm
.
tell
(
RegisterTaskManager
(
connectionInfo
,
hardwareDescription
,
1
),
selfGateway
)
expectMsgType
[
LeaderSessionMessage
]
match
{
case
LeaderSessionMessage
(
null
,
AcknowledgeRegistration
(
_
,
_
))
=>
case
m
=>
fail
(
"Wrong message type: "
+
m
)
}
expectMsgType
[
LeaderSessionMessage
]
match
{
case
LeaderSessionMessage
(
null
,
AlreadyRegistered
(
_
,
_
))
=>
case
m
=>
fail
(
"Wrong message type: "
+
m
)
}
expectMsgType
[
LeaderSessionMessage
]
match
{
case
LeaderSessionMessage
(
null
,
AlreadyRegistered
(
_
,
_
))
=>
case
m
=>
fail
(
"Wrong message type: "
+
m
)
}
}
finally
{
tmDummy
!
Kill
jm
!
Kill
}
}
}
private
def
startTestingJobManager
(
system
:
ActorSystem
)
:
Actor
Ref
=
{
private
def
startTestingJobManager
(
system
:
ActorSystem
)
:
Actor
Gateway
=
{
val
(
jm
:
ActorRef
,
_
)
=
JobManager
.
startJobManagerActors
(
new
Configuration
(),
_system
,
None
,
None
,
StreamingMode
.
BATCH_ONLY
)
jm
new
AkkaActorGateway
(
jm
,
null
)
}
}
object
JobManagerRegistrationTest
{
/** Simply dummy actor that swallows all messages */
class
DummyActor
extends
Actor
{
class
PlainForwardingActor
(
private
val
target
:
ActorRef
)
extends
Actor
{
override
def
receive
:
Receive
=
{
case
_
=>
case
message
=>
target
.
forward
(
message
)
}
}
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录