Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
lhongjum2003
DolphinScheduler
提交
d0ca95ea
DolphinScheduler
项目概览
lhongjum2003
/
DolphinScheduler
与 Fork 源项目一致
Fork自
apache / DolphinScheduler
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d0ca95ea
编写于
6月 08, 2019
作者:
leon-baoliang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
change the failover process.
上级
d7110c66
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
52 addition
and
31 deletion
+52
-31
escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java
...duler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java
+1
-1
escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java
...rc/main/java/cn/escheduler/server/utils/ProcessUtils.java
+2
-3
escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java
...src/main/java/cn/escheduler/server/zk/ZKMasterClient.java
+49
-27
未找到文件。
escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java
浏览文件 @
d0ca95ea
...
...
@@ -58,6 +58,7 @@ public class ProcessDao extends AbstractBaseDao {
private
final
int
[]
stateArray
=
new
int
[]{
ExecutionStatus
.
SUBMITTED_SUCCESS
.
ordinal
(),
ExecutionStatus
.
RUNNING_EXEUTION
.
ordinal
(),
ExecutionStatus
.
READY_PAUSE
.
ordinal
(),
ExecutionStatus
.
NEED_FAULT_TOLERANCE
.
ordinal
(),
ExecutionStatus
.
READY_STOP
.
ordinal
()};
@Autowired
...
...
@@ -1569,7 +1570,6 @@ public class ProcessDao extends AbstractBaseDao {
for
(
ProcessInstance
processInstance:
processInstanceList
){
processNeedFailoverProcessInstances
(
processInstance
);
}
}
@Transactional
(
value
=
"TransactionManager"
,
rollbackFor
=
Exception
.
class
)
...
...
escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java
浏览文件 @
d0ca95ea
...
...
@@ -273,9 +273,8 @@ public class ProcessUtils {
/**
* find logs and kill yarn tasks
* @param taskInstance
* @throws IOException
*/
public
static
void
killYarnJob
(
TaskInstance
taskInstance
)
throws
Exception
{
public
static
void
killYarnJob
(
TaskInstance
taskInstance
)
{
try
{
Thread
.
sleep
(
Constants
.
SLEEP_TIME_MILLIS
);
LogClient
logClient
=
new
LogClient
(
taskInstance
.
getHost
(),
Constants
.
RPC_PORT
);
...
...
@@ -295,7 +294,7 @@ public class ProcessUtils {
}
catch
(
Exception
e
)
{
logger
.
error
(
"kill yarn job failed : "
+
e
.
getMessage
(),
e
);
throw
new
RuntimeException
(
"kill yarn job fail"
);
//
throw new RuntimeException("kill yarn job fail");
}
}
}
escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java
浏览文件 @
d0ca95ea
...
...
@@ -30,6 +30,7 @@ import cn.escheduler.dao.model.ProcessInstance;
import
cn.escheduler.dao.model.TaskInstance
;
import
cn.escheduler.server.ResInfo
;
import
cn.escheduler.server.utils.ProcessUtils
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.curator.framework.CuratorFramework
;
import
org.apache.curator.framework.imps.CuratorFrameworkState
;
import
org.apache.curator.framework.recipes.cache.PathChildrenCache
;
...
...
@@ -279,17 +280,9 @@ public class ZKMasterClient extends AbstractZKClient {
for
(
int
i
=
0
;
i
<
Constants
.
ESCHEDULER_WARN_TIMES_FAILOVER
;
i
++)
{
alertDao
.
sendServerStopedAlert
(
1
,
masterHost
,
"Master-Server"
);
}
logger
.
info
(
"start master failover ..."
);
List
<
ProcessInstance
>
needFailoverProcessInstanceList
=
processDao
.
queryNeedFailoverProcessInstances
(
masterHost
);
//updateProcessInstance host is null and insert into command
for
(
ProcessInstance
processInstance
:
needFailoverProcessInstanceList
){
processDao
.
processNeedFailoverProcessInstances
(
processInstance
);
if
(
StringUtils
.
isNotEmpty
(
masterHost
)){
FailoverMaster
(
masterHost
);
}
logger
.
info
(
"master failover end"
);
}
catch
(
Exception
e
){
logger
.
error
(
"master failover failed : "
+
e
.
getMessage
(),
e
);
}
finally
{
...
...
@@ -331,6 +324,8 @@ public class ZKMasterClient extends AbstractZKClient {
}
/**
* monitor worker
*/
...
...
@@ -369,23 +364,9 @@ public class ZKMasterClient extends AbstractZKClient {
alertDao
.
sendServerStopedAlert
(
1
,
workerHost
,
"Worker-Server"
);
}
logger
.
info
(
"start worker failover ..."
);
List
<
TaskInstance
>
needFailoverTaskInstanceList
=
processDao
.
queryNeedFailoverTaskInstances
(
workerHost
);
for
(
TaskInstance
taskInstance
:
needFailoverTaskInstanceList
){
ProcessInstance
instance
=
processDao
.
findProcessInstanceDetailById
(
taskInstance
.
getProcessInstanceId
());
if
(
instance
!=
null
){
taskInstance
.
setProcessInstance
(
instance
);
}
// only kill yarn job if exists , the local thread has exited
ProcessUtils
.
killYarnJob
(
taskInstance
);
}
//updateProcessInstance state value is NEED_FAULT_TOLERANCE
processDao
.
updateNeedFailoverTaskInstances
(
workerHost
);
logger
.
info
(
"worker failover end"
);
if
(
StringUtils
.
isNotEmpty
(
workerHost
)){
FailoverWorker
(
workerHost
);
}
}
catch
(
Exception
e
){
logger
.
error
(
"worker failover failed : "
+
e
.
getMessage
(),
e
);
}
...
...
@@ -476,6 +457,46 @@ public class ZKMasterClient extends AbstractZKClient {
}
/**
* failover worker tasks
* 1. kill yarn job if there are yarn jobs in tasks.
* 2. change task state from running to need failover.
* @param workerHost
*/
private
void
FailoverWorker
(
String
workerHost
){
logger
.
info
(
"start worker[{}] failover ..."
,
workerHost
);
List
<
TaskInstance
>
needFailoverTaskInstanceList
=
processDao
.
queryNeedFailoverTaskInstances
(
workerHost
);
for
(
TaskInstance
taskInstance
:
needFailoverTaskInstanceList
){
ProcessInstance
instance
=
processDao
.
findProcessInstanceDetailById
(
taskInstance
.
getProcessInstanceId
());
if
(
instance
!=
null
){
taskInstance
.
setProcessInstance
(
instance
);
}
// only kill yarn job if exists , the local thread has exited
ProcessUtils
.
killYarnJob
(
taskInstance
);
}
//updateProcessInstance state value is NEED_FAULT_TOLERANCE
processDao
.
updateNeedFailoverTaskInstances
(
workerHost
);
logger
.
info
(
"end worker[{}] failover ..."
,
workerHost
);
}
/**
* failover master tasks
* @param masterHost
*/
private
void
FailoverMaster
(
String
masterHost
)
{
logger
.
info
(
"start master failover ..."
);
List
<
ProcessInstance
>
needFailoverProcessInstanceList
=
processDao
.
queryNeedFailoverProcessInstances
(
masterHost
);
//updateProcessInstance host is null and insert into command
for
(
ProcessInstance
processInstance
:
needFailoverProcessInstanceList
){
processDao
.
processNeedFailoverProcessInstances
(
processInstance
);
}
logger
.
info
(
"master failover end"
);
}
/**
* get host ip
...
...
@@ -488,6 +509,7 @@ public class ZKMasterClient extends AbstractZKClient {
if
(
startIndex
>=
endIndex
){
logger
.
error
(
"parse ip error"
);
return
""
;
}
return
path
.
substring
(
startIndex
,
endIndex
);
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录