Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
58b9a377
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
58b9a377
编写于
6月 04, 2015
作者:
R
Robert Metzger
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[FLINK-2087] Add streaming mode switch to YARN
This closes #788
上级
85c55dcb
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
53 addition
and
4 deletion
+53
-4
docs/setup/yarn_setup.md
docs/setup/yarn_setup.md
+1
-0
flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java
...ain/java/org/apache/flink/client/FlinkYarnSessionCli.java
+8
-0
flink-runtime/src/main/java/org/apache/flink/runtime/yarn/AbstractFlinkYarnClient.java
...rg/apache/flink/runtime/yarn/AbstractFlinkYarnClient.java
+6
-0
flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
...la/org/apache/flink/runtime/taskmanager/TaskManager.scala
+1
-1
flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
.../apache/flink/streaming/connectors/kafka/KafkaITCase.java
+2
-0
flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java
.../src/main/java/org/apache/flink/yarn/FlinkYarnClient.java
+8
-0
flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala
.../main/scala/org/apache/flink/yarn/ApplicationMaster.scala
+15
-1
flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMasterActor.scala
.../scala/org/apache/flink/yarn/ApplicationMasterActor.scala
+12
-2
未找到文件。
docs/setup/yarn_setup.md
浏览文件 @
58b9a377
...
...
@@ -99,6 +99,7 @@ Usage:
-q
,--query Display available YARN resources
(
memory, cores
)
-qu
,--queue <arg> Specify YARN queue.
-s
,--slots <arg> Number of slots per TaskManager
-st
,--streaming Start Flink
in
streaming mode
-tm
,--taskManagerMemory <arg> Memory per TaskManager Container
[
in
MB]
~~~
...
...
flink-clients/src/main/java/org/apache/flink/client/FlinkYarnSessionCli.java
浏览文件 @
58b9a377
...
...
@@ -73,6 +73,7 @@ public class FlinkYarnSessionCli {
private
final
Option
CONTAINER
;
private
final
Option
SLOTS
;
private
final
Option
DETACHED
;
private
final
Option
STREAMING
;
/**
* Dynamic properties allow the user to specify additional configuration values with -D, such as
...
...
@@ -95,6 +96,7 @@ public class FlinkYarnSessionCli {
SLOTS
=
new
Option
(
shortPrefix
+
"s"
,
longPrefix
+
"slots"
,
true
,
"Number of slots per TaskManager"
);
DYNAMIC_PROPERTIES
=
new
Option
(
shortPrefix
+
"D"
,
true
,
"Dynamic properties"
);
DETACHED
=
new
Option
(
shortPrefix
+
"d"
,
longPrefix
+
"detached"
,
false
,
"Start detached"
);
STREAMING
=
new
Option
(
shortPrefix
+
"st"
,
longPrefix
+
"streaming"
,
false
,
"Start Flink in streaming mode"
);
}
public
AbstractFlinkYarnClient
createFlinkYarnClient
(
CommandLine
cmd
)
{
...
...
@@ -214,6 +216,10 @@ public class FlinkYarnSessionCli {
detachedMode
=
true
;
flinkYarnClient
.
setDetachedMode
(
detachedMode
);
}
if
(
cmd
.
hasOption
(
STREAMING
.
getOpt
()))
{
flinkYarnClient
.
setStreamingMode
(
true
);
}
return
flinkYarnClient
;
}
...
...
@@ -237,6 +243,7 @@ public class FlinkYarnSessionCli {
opt
.
addOption
(
SLOTS
);
opt
.
addOption
(
DYNAMIC_PROPERTIES
);
opt
.
addOption
(
DETACHED
);
opt
.
addOption
(
STREAMING
);
formatter
.
printHelp
(
" "
,
opt
);
}
...
...
@@ -342,6 +349,7 @@ public class FlinkYarnSessionCli {
options
.
addOption
(
SLOTS
);
options
.
addOption
(
DYNAMIC_PROPERTIES
);
options
.
addOption
(
DETACHED
);
options
.
addOption
(
STREAMING
);
}
public
int
run
(
String
[]
args
)
{
...
...
flink-runtime/src/main/java/org/apache/flink/runtime/yarn/AbstractFlinkYarnClient.java
浏览文件 @
58b9a377
...
...
@@ -132,4 +132,10 @@ public abstract class AbstractFlinkYarnClient {
* directory in HDFS that contains the jar files and configuration which is shipped to all the containers.
*/
public
abstract
String
getSessionFilesDir
();
/**
* Instruct Flink to start in streaming mode
* @param streamingMode
*/
public
abstract
void
setStreamingMode
(
boolean
streamingMode
);
}
flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
浏览文件 @
58b9a377
...
...
@@ -1248,7 +1248,7 @@ object TaskManager {
streamingMode
:
StreamingMode
,
taskManagerClass
:
Class
[
_
<:
TaskManager
])
:
Unit
=
{
LOG
.
info
(
"Starting TaskManager
"
)
LOG
.
info
(
s
"Starting TaskManager in streaming mode $streamingMode
"
)
// Bring up the TaskManager actor system first, bind it to the given address.
...
...
flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
浏览文件 @
58b9a377
...
...
@@ -74,6 +74,7 @@ import org.junit.AfterClass;
import
org.junit.Assert
;
import
org.junit.BeforeClass
;
import
org.junit.ClassRule
;
import
org.junit.Ignore
;
import
org.junit.Test
;
import
org.junit.rules.TemporaryFolder
;
import
org.slf4j.Logger
;
...
...
@@ -281,6 +282,7 @@ public class KafkaITCase {
*
*/
@Test
@Ignore
public
void
testPersistentSourceWithOffsetUpdates
()
throws
Exception
{
LOG
.
info
(
"Starting testPersistentSourceWithOffsetUpdates()"
);
...
...
flink-yarn/src/main/java/org/apache/flink/yarn/FlinkYarnClient.java
浏览文件 @
58b9a377
...
...
@@ -93,6 +93,7 @@ public class FlinkYarnClient extends AbstractFlinkYarnClient {
public
static
final
String
ENV_CLIENT_USERNAME
=
"_CLIENT_USERNAME"
;
public
static
final
String
ENV_SLOTS
=
"_SLOTS"
;
public
static
final
String
ENV_DETACHED
=
"_DETACHED"
;
public
static
final
String
ENV_STREAMING_MODE
=
"_STREAMING_MODE"
;
public
static
final
String
ENV_DYNAMIC_PROPERTIES
=
"_DYNAMIC_PROPERTIES"
;
...
...
@@ -140,6 +141,7 @@ public class FlinkYarnClient extends AbstractFlinkYarnClient {
private
org
.
apache
.
flink
.
configuration
.
Configuration
flinkConfiguration
;
private
boolean
detached
;
private
boolean
streamingMode
;
public
FlinkYarnClient
()
{
...
...
@@ -576,6 +578,7 @@ public class FlinkYarnClient extends AbstractFlinkYarnClient {
appMasterEnv
.
put
(
FlinkYarnClient
.
ENV_CLIENT_USERNAME
,
UserGroupInformation
.
getCurrentUser
().
getShortUserName
());
appMasterEnv
.
put
(
FlinkYarnClient
.
ENV_SLOTS
,
String
.
valueOf
(
slots
));
appMasterEnv
.
put
(
FlinkYarnClient
.
ENV_DETACHED
,
String
.
valueOf
(
detached
));
appMasterEnv
.
put
(
FlinkYarnClient
.
ENV_STREAMING_MODE
,
String
.
valueOf
(
streamingMode
));
if
(
dynamicPropertiesEncoded
!=
null
)
{
appMasterEnv
.
put
(
FlinkYarnClient
.
ENV_DYNAMIC_PROPERTIES
,
dynamicPropertiesEncoded
);
...
...
@@ -726,6 +729,11 @@ public class FlinkYarnClient extends AbstractFlinkYarnClient {
return
sessionFilesDir
.
toString
();
}
@Override
public
void
setStreamingMode
(
boolean
streamingMode
)
{
this
.
streamingMode
=
streamingMode
;
}
public
static
class
YarnDeploymentException
extends
RuntimeException
{
public
YarnDeploymentException
()
{
}
...
...
flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala
浏览文件 @
58b9a377
...
...
@@ -56,7 +56,7 @@ object ApplicationMaster {
EnvironmentInformation
.
checkJavaVersion
()
org
.
apache
.
flink
.
runtime
.
util
.
SignalHandler
.
register
(
LOG
.
logger
)
va
l
streamingMode
=
StreamingMode
.
BATCH_ONLY
va
r
streamingMode
=
StreamingMode
.
BATCH_ONLY
val
ugi
=
UserGroupInformation
.
createRemoteUser
(
yarnClientUsername
)
...
...
@@ -84,6 +84,11 @@ object ApplicationMaster {
val
logDirs
=
env
.
get
(
Environment
.
LOG_DIRS
.
key
())
if
(
hasStreamingMode
(
env
))
{
LOG
.
info
(
"Starting ApplicationMaster/JobManager in streaming mode"
)
streamingMode
=
StreamingMode
.
STREAMING
}
// Note that we use the "ownHostname" given by YARN here, to make sure
// we use the hostnames given by YARN consistently throughout akka.
// for akka "localhost" and "localhost.localdomain" are different actors.
...
...
@@ -246,4 +251,13 @@ object ApplicationMaster {
(
configuration
,
jobManagerSystem
,
jobManager
,
archiver
)
}
def
hasStreamingMode
(
env
:
java.util.Map
[
String
,
String
])
:
Boolean
=
{
val
sModeString
=
env
.
get
(
FlinkYarnClient
.
ENV_STREAMING_MODE
)
if
(
sModeString
!=
null
)
{
return
sModeString
.
toBoolean
}
false
}
}
flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMasterActor.scala
浏览文件 @
58b9a377
...
...
@@ -461,8 +461,10 @@ trait ApplicationMasterActor extends ActorLogMessages {
runningContainers
=
0
failedContainers
=
0
val
hs
=
ApplicationMaster
.
hasStreamingMode
(
env
)
containerLaunchContext
=
Some
(
createContainerLaunchContext
(
heapLimit
,
hasLogback
,
hasLog4j
,
yarnClientUsername
,
conf
,
taskManagerLocalResources
))
yarnClientUsername
,
conf
,
taskManagerLocalResources
,
hs
))
context
.
system
.
scheduler
.
scheduleOnce
(
FAST_YARN_HEARTBEAT_DELAY
,
self
,
HeartbeatWithYarn
)
}
recover
{
...
...
@@ -499,7 +501,8 @@ trait ApplicationMasterActor extends ActorLogMessages {
private
def
createContainerLaunchContext
(
heapLimit
:
Int
,
hasLogback
:
Boolean
,
hasLog4j
:
Boolean
,
yarnClientUsername
:
String
,
yarnConf
:
Configuration
,
taskManagerLocalResources
:
Map
[
String
,
LocalResource
])
:
taskManagerLocalResources
:
Map
[
String
,
LocalResource
],
streamingMode
:
Boolean
)
:
ContainerLaunchContext
=
{
log
.
info
(
"Create container launch context."
)
val
ctx
=
Records
.
newRecord
(
classOf
[
ContainerLaunchContext
])
...
...
@@ -525,6 +528,13 @@ trait ApplicationMasterActor extends ActorLogMessages {
s
"${ApplicationConstants.LOG_DIR_EXPANSION_VAR}/taskmanager-stdout.log 2> "
+
s
"${ApplicationConstants.LOG_DIR_EXPANSION_VAR}/taskmanager-stderr.log"
tmCommand
++=
" --streamingMode"
if
(
streamingMode
)
{
tmCommand
++=
" streaming"
}
else
{
tmCommand
++=
" batch"
}
ctx
.
setCommands
(
Collections
.
singletonList
(
tmCommand
.
toString
()))
log
.
info
(
s
"Starting TM with command=${tmCommand.toString()}"
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录