Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
doujutun3207
flink
提交
f6051277
F
flink
项目概览
doujutun3207
/
flink
与 Fork 源项目一致
从无法访问的项目Fork
通知
24
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
F
flink
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
f6051277
编写于
10月 10, 2014
作者:
R
Robert Metzger
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[YARN] Fix issue with port-offsetting.
上级
d749c24d
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
41 addition
and
16 deletion
+41
-16
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/Client.java
...link-yarn/src/main/java/org/apache/flink/yarn/Client.java
+2
-2
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/ClientMasterControl.java
.../main/java/org/apache/flink/yarn/ClientMasterControl.java
+4
-5
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java
...flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java
+19
-0
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/appMaster/ApplicationMaster.java
...va/org/apache/flink/yarn/appMaster/ApplicationMaster.java
+8
-7
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/rpc/YARNClientMasterProtocol.java
...a/org/apache/flink/yarn/rpc/YARNClientMasterProtocol.java
+1
-2
flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobManager.java
.../java/org/apache/flink/runtime/jobmanager/JobManager.java
+7
-0
未找到文件。
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/Client.java
浏览文件 @
f6051277
...
...
@@ -450,7 +450,7 @@ public class Client {
*/
int
appNumber
=
appId
.
getId
();
jmPort
+=
appNumber
;
jmPort
=
Utils
.
offsetPort
(
jmPort
,
appNumber
)
;
// Setup jar for ApplicationMaster
LocalResource
appMasterJar
=
Records
.
newRecord
(
LocalResource
.
class
);
...
...
@@ -492,7 +492,7 @@ public class Client {
fs
.
close
();
int
amRPCPort
=
GlobalConfiguration
.
getInteger
(
ConfigConstants
.
YARN_AM_PRC_PORT
,
ConfigConstants
.
DEFAULT_YARN_AM_RPC_PORT
);
amRPCPort
+=
appNumber
;
amRPCPort
=
Utils
.
offsetPort
(
amRPCPort
,
appNumber
)
;
// Setup CLASSPATH for ApplicationMaster
Map
<
String
,
String
>
appMasterEnv
=
new
HashMap
<
String
,
String
>();
Utils
.
setupEnv
(
conf
,
appMasterEnv
);
...
...
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/ClientMasterControl.java
浏览文件 @
f6051277
...
...
@@ -106,13 +106,12 @@ public class ClientMasterControl extends Thread {
}
}
public
boolean
shutdownAM
()
{
public
void
shutdownAM
()
{
try
{
boolean
result
=
cmp
.
shutdownAM
().
getValue
();
return
result
;
cmp
.
shutdownAM
();
}
catch
(
Throwable
e
)
{
LOG
.
warn
(
"Error shutting down the application master"
,
e
);
return
false
;
// the old RPC service is unable to shut down itself. So the java.io.EOFException is expected here.
LOG
.
debug
(
"This exception is expected"
,
e
)
;
}
}
...
...
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/Utils.java
浏览文件 @
f6051277
...
...
@@ -266,4 +266,23 @@ public class Utils {
environment
.
put
(
StringInterner
.
weakIntern
(
variable
),
StringInterner
.
weakIntern
(
val
));
}
/**
* Valid ports are 1024 - 65535.
* We offset the incoming port by the applicationId to avoid port collisions if YARN allocates two ApplicationMasters
* on the same physical hardware
*/
public
static
int
offsetPort
(
int
port
,
int
appId
)
{
if
(
port
>
65535
)
{
LOG
.
warn
(
"The specified YARN RPC port ("
+
port
+
") is above the maximum possible port 65535."
+
"Setting it to "
+
64535
);
port
=
64535
;
}
if
(
port
+
(
appId
%
1000
)
>
65535
)
{
LOG
.
warn
(
"The specified YARN RPC port ("
+
port
+
") is, when offsetted by the ApplicationID ("
+
appId
+
") above "
+
"the maximum possible port 65535. Setting it to "
+
64535
);
port
=
port
-
1000
;
}
return
port
+
(
appId
%
1000
);
}
}
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/appMaster/ApplicationMaster.java
浏览文件 @
f6051277
...
...
@@ -18,7 +18,6 @@
package
org.apache.flink.yarn.appMaster
;
import
java.io.BufferedReader
;
import
java.io.BufferedWriter
;
import
java.io.File
;
...
...
@@ -44,7 +43,6 @@ import org.apache.flink.runtime.ipc.RPC;
import
org.apache.flink.runtime.ipc.RPC.Server
;
import
org.apache.flink.runtime.jobmanager.JobManager
;
import
org.apache.flink.runtime.util.SerializableArrayList
;
import
org.apache.flink.types.BooleanValue
;
import
org.apache.flink.util.StringUtils
;
import
org.apache.flink.yarn.Client
;
import
org.apache.flink.yarn.Utils
;
...
...
@@ -220,12 +218,13 @@ public class ApplicationMaster implements YARNClientMasterProtocol {
// determine JobManager port
int
port
=
GlobalConfiguration
.
getInteger
(
ConfigConstants
.
JOB_MANAGER_IPC_PORT_KEY
,
-
1
);
if
(
port
!=
-
1
)
{
port
+=
appNumber
;
port
=
Utils
.
offsetPort
(
port
,
appNumber
)
;
}
else
{
LOG
.
warn
(
"JobManager port is unknown"
);
}
this
.
jobManagerPort
=
port
;
this
.
jobManagerWebPort
=
GlobalConfiguration
.
getInteger
(
ConfigConstants
.
JOB_MANAGER_WEB_PORT_KEY
,
ConfigConstants
.
DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT
)+
appNumber
;
int
jmWebPort
=
GlobalConfiguration
.
getInteger
(
ConfigConstants
.
JOB_MANAGER_WEB_PORT_KEY
,
ConfigConstants
.
DEFAULT_JOB_MANAGER_WEB_FRONTEND_PORT
);
this
.
jobManagerWebPort
=
Utils
.
offsetPort
(
jmWebPort
,
appNumber
);
}
private
void
setFailed
(
boolean
failed
)
{
...
...
@@ -506,7 +505,7 @@ public class ApplicationMaster implements YARNClientMasterProtocol {
@Override
public
BooleanValue
shutdownAM
()
throws
Exception
{
public
void
shutdownAM
()
throws
Exception
{
LOG
.
info
(
"Client requested shutdown of AM"
);
FinalApplicationStatus
finalStatus
=
FinalApplicationStatus
.
SUCCEEDED
;
String
finalMessage
=
""
;
...
...
@@ -518,7 +517,6 @@ public class ApplicationMaster implements YARNClientMasterProtocol {
}
rmClient
.
unregisterApplicationMaster
(
finalStatus
,
finalMessage
,
""
);
this
.
close
();
return
new
BooleanValue
(
true
);
}
private
void
close
()
throws
Exception
{
...
...
@@ -527,12 +525,15 @@ public class ApplicationMaster implements YARNClientMasterProtocol {
nmClient
.
close
();
rmClient
.
close
();
if
(!
isFailed
)
{
//
amRpcServer.stop();
amRpcServer
.
stop
();
}
else
{
LOG
.
warn
(
"Can not close AM RPC connection since the AM is in failed state"
);
}
}
else
{
LOG
.
warn
(
"The AM has already been closed before"
);
}
this
.
isClosed
=
true
;
System
.
exit
(
0
);
// kill it hard.
}
@Override
...
...
flink-addons/flink-yarn/src/main/java/org/apache/flink/yarn/rpc/YARNClientMasterProtocol.java
浏览文件 @
f6051277
...
...
@@ -26,7 +26,6 @@ import org.apache.flink.core.io.IOReadableWritable;
import
org.apache.flink.core.memory.DataInputView
;
import
org.apache.flink.core.memory.DataOutputView
;
import
org.apache.flink.core.protocols.VersionedProtocol
;
import
org.apache.flink.types.BooleanValue
;
/**
...
...
@@ -71,7 +70,7 @@ public interface YARNClientMasterProtocol extends VersionedProtocol {
ApplicationMasterStatus
getAppplicationMasterStatus
();
BooleanValue
shutdownAM
()
throws
Exception
;
void
shutdownAM
()
throws
Exception
;
List
<
Message
>
getMessages
();
...
...
flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobManager.java
浏览文件 @
f6051277
...
...
@@ -279,6 +279,13 @@ public class JobManager implements ExtendedManagementProtocol, InputSplitProvide
this
.
scheduler
.
shutdown
();
}
if
(
this
.
server
!=
null
)
{
try
{
this
.
server
.
stop
();
}
catch
(
Exception
e1
)
{
throw
new
RuntimeException
(
"Error shtopping the web-info-server."
,
e1
);
}
}
this
.
isShutDown
=
true
;
LOG
.
debug
(
"Shutdown of job manager completed"
);
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录