Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
apache
DolphinScheduler
提交
492b318b
DolphinScheduler
项目概览
apache
/
DolphinScheduler
上一次同步 1 年多
通知
704
Star
9572
Fork
3514
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
未验证
提交
492b318b
编写于
3月 26, 2021
作者:
S
Shiwen Cheng
提交者:
GitHub
3月 26, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[Fix][Server] Fix clear task execute path is related to master (#5123)
上级
a8b47e1d
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
43 addition
and
67 deletion
+43
-67
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
...a/org/apache/dolphinscheduler/common/utils/FileUtils.java
+2
-21
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
...g/apache/dolphinscheduler/common/utils/FileUtilsTest.java
+1
-3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
...lphinscheduler/server/master/runner/MasterExecThread.java
+0
-30
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
...heduler/server/worker/processor/TaskExecuteProcessor.java
+1
-1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
...phinscheduler/server/worker/runner/TaskExecuteThread.java
+38
-7
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
.../dolphinscheduler/server/master/MasterExecThreadTest.java
+1
-5
未找到文件。
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
浏览文件 @
492b318b
...
...
@@ -121,27 +121,8 @@ public class FileUtils {
* @return directory of process execution
*/
public
static
String
getProcessExecDir
(
int
projectId
,
int
processDefineId
,
int
processInstanceId
,
int
taskInstanceId
)
{
String
fileName
=
String
.
format
(
"%s/exec/process/%s/%s/%s/%s"
,
DATA_BASEDIR
,
Integer
.
toString
(
projectId
),
Integer
.
toString
(
processDefineId
),
Integer
.
toString
(
processInstanceId
),
Integer
.
toString
(
taskInstanceId
));
File
file
=
new
File
(
fileName
);
if
(!
file
.
getParentFile
().
exists
())
{
file
.
getParentFile
().
mkdirs
();
}
return
fileName
;
}
/**
* directory of process instances
*
* @param projectId project id
* @param processDefineId process definition id
* @param processInstanceId process instance id
* @return directory of process instances
*/
public
static
String
getProcessExecDir
(
int
projectId
,
int
processDefineId
,
int
processInstanceId
)
{
String
fileName
=
String
.
format
(
"%s/exec/process/%s/%s/%s"
,
DATA_BASEDIR
,
Integer
.
toString
(
projectId
),
Integer
.
toString
(
processDefineId
),
Integer
.
toString
(
processInstanceId
));
String
fileName
=
String
.
format
(
"%s/exec/process/%d/%d/%d/%d"
,
DATA_BASEDIR
,
projectId
,
processDefineId
,
processInstanceId
,
taskInstanceId
);
File
file
=
new
File
(
fileName
);
if
(!
file
.
getParentFile
().
exists
())
{
file
.
getParentFile
().
mkdirs
();
...
...
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
浏览文件 @
492b318b
...
...
@@ -60,10 +60,8 @@ public class FileUtilsTest {
@Test
public
void
testGetProcessExecDir
()
{
String
dir
=
FileUtils
.
getProcessExecDir
(
1
,
2
,
3
,
4
);
String
dir
=
FileUtils
.
getProcessExecDir
(
1
,
2
,
3
,
4
);
Assert
.
assertEquals
(
"/tmp/dolphinscheduler/exec/process/1/2/3/4"
,
dir
);
dir
=
FileUtils
.
getProcessExecDir
(
1
,
2
,
3
);
Assert
.
assertEquals
(
"/tmp/dolphinscheduler/exec/process/1/2/3"
,
dir
);
}
@Test
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
浏览文件 @
492b318b
...
...
@@ -42,7 +42,6 @@ import org.apache.dolphinscheduler.common.process.Property;
import
org.apache.dolphinscheduler.common.thread.Stopper
;
import
org.apache.dolphinscheduler.common.thread.ThreadUtils
;
import
org.apache.dolphinscheduler.common.utils.CollectionUtils
;
import
org.apache.dolphinscheduler.common.utils.CommonUtils
;
import
org.apache.dolphinscheduler.common.utils.DateUtils
;
import
org.apache.dolphinscheduler.common.utils.JSONUtils
;
import
org.apache.dolphinscheduler.common.utils.OSUtils
;
...
...
@@ -61,16 +60,11 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import
org.apache.dolphinscheduler.service.quartz.cron.CronUtils
;
import
org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue
;
import
org.apache.commons.io.FileUtils
;
import
java.io.File
;
import
java.io.IOException
;
import
java.text.ParseException
;
import
java.util.ArrayList
;
import
java.util.Arrays
;
import
java.util.Collection
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.Map
;
...
...
@@ -78,7 +72,6 @@ import java.util.Set;
import
java.util.concurrent.ConcurrentHashMap
;
import
java.util.concurrent.ExecutorService
;
import
java.util.concurrent.Future
;
import
java.util.stream.Collectors
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -236,8 +229,6 @@ public class MasterExecThread implements Runnable {
processService
.
updateProcessInstance
(
processInstance
);
}
finally
{
taskExecService
.
shutdown
();
// post handle
postHandle
();
}
}
...
...
@@ -427,27 +418,6 @@ public class MasterExecThread implements Runnable {
}
}
/**
* process post handle
*/
private
void
postHandle
()
{
logger
.
info
(
"develop mode is: {}"
,
CommonUtils
.
isDevelopMode
());
if
(!
CommonUtils
.
isDevelopMode
())
{
// get exec dir
String
execLocalPath
=
org
.
apache
.
dolphinscheduler
.
common
.
utils
.
FileUtils
.
getProcessExecDir
(
processInstance
.
getProcessDefinition
().
getProjectId
(),
processInstance
.
getProcessDefinitionId
(),
processInstance
.
getId
());
try
{
FileUtils
.
deleteDirectory
(
new
File
(
execLocalPath
));
}
catch
(
IOException
e
)
{
logger
.
error
(
"delete exec dir failed "
,
e
);
}
}
}
/**
* submit task to execute
*
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
浏览文件 @
492b318b
...
...
@@ -141,7 +141,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
// local execute path
String
execLocalPath
=
getExecLocalPath
(
taskExecutionContext
);
logger
.
info
(
"task instance
local execute path : {}
"
,
execLocalPath
);
logger
.
info
(
"task instance
local execute path : {}
"
,
execLocalPath
);
taskExecutionContext
.
setExecutePath
(
execLocalPath
);
FileUtils
.
taskLoggerThreadLocal
.
set
(
taskLogger
);
...
...
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
浏览文件 @
492b318b
...
...
@@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import
org.apache.dolphinscheduler.common.utils.JSONUtils
;
import
org.apache.dolphinscheduler.common.utils.OSUtils
;
import
org.apache.dolphinscheduler.common.utils.RetryerUtils
;
import
org.apache.dolphinscheduler.common.utils.StringUtils
;
import
org.apache.dolphinscheduler.remote.command.Command
;
import
org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand
;
import
org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand
;
...
...
@@ -45,6 +46,7 @@ import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import
org.apache.commons.collections.MapUtils
;
import
java.io.File
;
import
java.io.IOException
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.List
;
...
...
@@ -71,17 +73,17 @@ public class TaskExecuteThread implements Runnable, Delayed {
private
final
Logger
logger
=
LoggerFactory
.
getLogger
(
TaskExecuteThread
.
class
);
/**
*
task instance
* task instance
*/
private
TaskExecutionContext
taskExecutionContext
;
/**
*
abstract task
* abstract task
*/
private
AbstractTask
task
;
/**
*
task callback service
* task callback service
*/
private
TaskCallbackService
taskCallbackService
;
...
...
@@ -185,9 +187,38 @@ public class TaskExecuteThread implements Runnable, Delayed {
responseCommand
.
setAppIds
(
task
.
getAppIds
());
}
finally
{
taskExecutionContextCacheManager
.
removeByTaskInstanceId
(
taskExecutionContext
.
getTaskInstanceId
());
ResponceCache
.
get
().
cache
(
taskExecutionContext
.
getTaskInstanceId
(),
responseCommand
.
convert2Command
(),
Event
.
RESULT
);
ResponceCache
.
get
().
cache
(
taskExecutionContext
.
getTaskInstanceId
(),
responseCommand
.
convert2Command
(),
Event
.
RESULT
);
taskCallbackService
.
sendResult
(
taskExecutionContext
.
getTaskInstanceId
(),
responseCommand
.
convert2Command
());
clearTaskExecPath
();
}
}
/**
* when task finish, clear execute path.
*/
private
void
clearTaskExecPath
()
{
logger
.
info
(
"develop mode is: {}"
,
CommonUtils
.
isDevelopMode
());
if
(!
CommonUtils
.
isDevelopMode
())
{
// get exec dir
String
execLocalPath
=
taskExecutionContext
.
getExecutePath
();
if
(
StringUtils
.
isEmpty
(
execLocalPath
))
{
logger
.
warn
(
"task: {} exec local path is empty."
,
taskExecutionContext
.
getTaskName
());
return
;
}
if
(
"/"
.
equals
(
execLocalPath
))
{
logger
.
warn
(
"task: {} exec local path is '/', direct deletion is not allowed"
,
taskExecutionContext
.
getTaskName
());
return
;
}
try
{
org
.
apache
.
commons
.
io
.
FileUtils
.
deleteDirectory
(
new
File
(
execLocalPath
));
logger
.
info
(
"exec local path: {} cleared."
,
execLocalPath
);
}
catch
(
IOException
e
)
{
logger
.
error
(
"delete exec dir failed : {}"
,
e
.
getMessage
(),
e
);
}
}
}
...
...
@@ -196,7 +227,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
* @return
*/
private
Map
<
String
,
String
>
getGlobalParamsMap
()
{
Map
<
String
,
String
>
globalParamsMap
=
new
HashMap
<>(
16
);
Map
<
String
,
String
>
globalParamsMap
=
new
HashMap
<>(
16
);
// global params string
String
globalParamsStr
=
taskExecutionContext
.
getGlobalParams
();
...
...
@@ -241,7 +272,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
}
/**
*
kill task
* kill task
*/
public
void
kill
()
{
if
(
task
!=
null
)
{
...
...
@@ -261,7 +292,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
* @param logger
*/
private
void
downloadResource
(
String
execLocalPath
,
Map
<
String
,
String
>
projectRes
,
Map
<
String
,
String
>
projectRes
,
Logger
logger
)
throws
Exception
{
if
(
MapUtils
.
isEmpty
(
projectRes
))
{
return
;
...
...
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
浏览文件 @
492b318b
...
...
@@ -102,16 +102,12 @@ public class MasterExecThreadTest {
processDefinition
.
setGlobalParamList
(
Collections
.
EMPTY_LIST
);
Mockito
.
when
(
processInstance
.
getProcessDefinition
()).
thenReturn
(
processDefinition
);
masterExecThread
=
PowerMockito
.
spy
(
new
MasterExecThread
(
processInstance
,
processService
,
null
,
null
,
config
));
masterExecThread
=
PowerMockito
.
spy
(
new
MasterExecThread
(
processInstance
,
processService
,
null
,
null
,
config
));
// prepareProcess init dag
Field
dag
=
MasterExecThread
.
class
.
getDeclaredField
(
"dag"
);
dag
.
setAccessible
(
true
);
dag
.
set
(
masterExecThread
,
new
DAG
());
PowerMockito
.
doNothing
().
when
(
masterExecThread
,
"executeProcess"
);
PowerMockito
.
doNothing
().
when
(
masterExecThread
,
"postHandle"
);
PowerMockito
.
doNothing
().
when
(
masterExecThread
,
"prepareProcess"
);
PowerMockito
.
doNothing
().
when
(
masterExecThread
,
"runProcess"
);
PowerMockito
.
doNothing
().
when
(
masterExecThread
,
"endProcess"
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录