Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
淡淡忧伤的程序员
DolphinScheduler
提交
df5d9a58
DolphinScheduler
项目概览
淡淡忧伤的程序员
/
DolphinScheduler
与 Fork 源项目一致
Fork自
apache / DolphinScheduler
通知
48
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
df5d9a58
编写于
4月 16, 2019
作者:
Q
qiaozhanwei
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Python task specifies path execution
上级
dd6a103f
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
117 addition
and
18 deletion
+117
-18
escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
.../escheduler/server/worker/task/PythonCommandExecutor.java
+52
-18
escheduler-server/src/test/java/cn/escheduler/server/worker/EnvFileTest.java
...rc/test/java/cn/escheduler/server/worker/EnvFileTest.java
+65
-0
未找到文件。
escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
浏览文件 @
df5d9a58
...
@@ -19,9 +19,9 @@ package cn.escheduler.server.worker.task;
...
@@ -19,9 +19,9 @@ package cn.escheduler.server.worker.task;
import
cn.escheduler.common.utils.FileUtils
;
import
cn.escheduler.common.utils.FileUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.io.File
;
import
java.io.*
;
import
java.io.IOException
;
import
java.nio.charset.StandardCharsets
;
import
java.nio.charset.StandardCharsets
;
import
java.nio.file.Files
;
import
java.nio.file.Files
;
import
java.nio.file.Paths
;
import
java.nio.file.Paths
;
...
@@ -34,6 +34,8 @@ import java.util.function.Consumer;
...
@@ -34,6 +34,8 @@ import java.util.function.Consumer;
*/
*/
public
class
PythonCommandExecutor
extends
AbstractCommandExecutor
{
public
class
PythonCommandExecutor
extends
AbstractCommandExecutor
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
PythonCommandExecutor
.
class
);
public
static
final
String
PYTHON
=
"python"
;
public
static
final
String
PYTHON
=
"python"
;
...
@@ -63,27 +65,13 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
...
@@ -63,27 +65,13 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
*/
*/
@Override
@Override
protected
void
createCommandFileIfNotExists
(
String
execCommand
,
String
commandFile
)
throws
IOException
{
protected
void
createCommandFileIfNotExists
(
String
execCommand
,
String
commandFile
)
throws
IOException
{
logger
.
info
(
"
proxy user
:{}, work dir:{}"
,
tenantCode
,
taskDir
);
logger
.
info
(
"
tenant
:{}, work dir:{}"
,
tenantCode
,
taskDir
);
if
(!
Files
.
exists
(
Paths
.
get
(
commandFile
)))
{
if
(!
Files
.
exists
(
Paths
.
get
(
commandFile
)))
{
logger
.
info
(
"generate command file:{}"
,
commandFile
);
logger
.
info
(
"generate command file:{}"
,
commandFile
);
StringBuilder
sb
=
new
StringBuilder
(
200
);
StringBuilder
sb
=
new
StringBuilder
(
200
);
sb
.
append
(
"#-*- encoding=utf8 -*-\n"
);
sb
.
append
(
"#-*- encoding=utf8 -*-\n"
);
sb
.
append
(
"import os,sys\n"
);
sb
.
append
(
"BASEDIR = os.path.dirname(os.path.realpath(__file__))\n"
);
sb
.
append
(
"os.chdir(BASEDIR)\n"
);
if
(
StringUtils
.
isNotEmpty
(
envFile
))
{
String
[]
envArray
=
envFile
.
split
(
"\\."
);
if
(
envArray
.
length
==
2
){
String
path
=
envArray
[
0
];
logger
.
info
(
"path:"
+
path
);
int
index
=
path
.
lastIndexOf
(
"/"
);
sb
.
append
(
String
.
format
(
"sys.path.append('%s')\n"
,
path
.
substring
(
0
,
index
)));
sb
.
append
(
String
.
format
(
"import %s\n"
,
path
.
substring
(
index
+
1
)));
}
}
sb
.
append
(
"\n\n"
);
sb
.
append
(
"\n\n"
);
sb
.
append
(
String
.
format
(
"import py_%s_node\n"
,
taskAppId
));
sb
.
append
(
String
.
format
(
"import py_%s_node\n"
,
taskAppId
));
...
@@ -96,7 +84,12 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
...
@@ -96,7 +84,12 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
@Override
@Override
protected
String
commandType
()
{
protected
String
commandType
()
{
return
PYTHON
;
String
envPath
=
System
.
getProperty
(
"user.dir"
)+
"/conf/env/.escheduler_env.sh"
;
String
pythonHome
=
getPythonHome
(
envPath
);
if
(
StringUtils
.
isEmpty
(
pythonHome
)){
return
PYTHON
;
}
return
pythonHome
;
}
}
@Override
@Override
...
@@ -109,4 +102,45 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
...
@@ -109,4 +102,45 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
return
true
;
return
true
;
}
}
/**
* get python home
* @param envPath
* @return
*/
private
static
String
getPythonHome
(
String
envPath
){
BufferedReader
br
=
null
;
String
line
=
null
;
StringBuilder
sb
=
new
StringBuilder
();
try
{
br
=
new
BufferedReader
(
new
InputStreamReader
(
new
FileInputStream
(
envPath
)));
while
((
line
=
br
.
readLine
())
!=
null
){
if
(
line
.
contains
(
"PYTHON_HOME"
)){
sb
.
append
(
line
);
break
;
}
}
String
result
=
sb
.
toString
();
if
(
org
.
apache
.
commons
.
lang
.
StringUtils
.
isEmpty
(
result
)){
return
null
;
}
String
[]
arrs
=
result
.
split
(
"="
);
if
(
arrs
.
length
==
2
){
return
arrs
[
1
];
}
}
catch
(
IOException
e
){
logger
.
error
(
"read file failed : "
+
e
.
getMessage
(),
e
);
}
finally
{
try
{
if
(
br
!=
null
){
br
.
close
();
}
}
catch
(
IOException
e
)
{
logger
.
error
(
e
.
getMessage
(),
e
);
}
}
return
null
;
}
}
}
escheduler-server/src/test/java/cn/escheduler/server/worker/EnvFileTest.java
0 → 100644
浏览文件 @
df5d9a58
package
cn.escheduler.server.worker
;
import
org.apache.commons.lang.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.io.BufferedReader
;
import
java.io.FileInputStream
;
import
java.io.IOException
;
import
java.io.InputStreamReader
;
/**
* Created by qiaozhanwei on 2019/4/15.
*/
public
class
EnvFileTest
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
EnvFileTest
.
class
);
public
static
void
main
(
String
[]
args
)
{
String
path
=
System
.
getProperty
(
"user.dir"
)+
"\\script\\env\\.escheduler_env.sh"
;
String
pythonHome
=
getPythonHome
(
path
);
logger
.
info
(
pythonHome
);
}
/**
* get python home
* @param path
* @return
*/
private
static
String
getPythonHome
(
String
path
){
BufferedReader
br
=
null
;
String
line
=
null
;
StringBuilder
sb
=
new
StringBuilder
();
try
{
br
=
new
BufferedReader
(
new
InputStreamReader
(
new
FileInputStream
(
path
)));
while
((
line
=
br
.
readLine
())
!=
null
){
if
(
line
.
contains
(
"PYTHON_HOME"
)){
sb
.
append
(
line
);
break
;
}
}
String
result
=
sb
.
toString
();
if
(
StringUtils
.
isEmpty
(
result
)){
return
null
;
}
String
[]
arrs
=
result
.
split
(
"="
);
if
(
arrs
.
length
==
2
){
return
arrs
[
1
];
}
}
catch
(
IOException
e
){
logger
.
error
(
"read file failed : "
+
e
.
getMessage
(),
e
);
}
finally
{
try
{
if
(
br
!=
null
){
br
.
close
();
}
}
catch
(
IOException
e
)
{
logger
.
error
(
e
.
getMessage
(),
e
);
}
}
return
null
;
}
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录