Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
apache
DolphinScheduler
提交
267df899
DolphinScheduler
项目概览
apache
/
DolphinScheduler
上一次同步 1 年多
通知
707
Star
9572
Fork
3514
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
267df899
编写于
4月 18, 2019
作者:
journey2018
提交者:
GitHub
4月 18, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #81 from qiaozhanwei/dev-20190415
dev merge
上级
9df5d43e
cd60898a
变更
16
隐藏空白更改
内联
并排
Showing
16 changed file
with
137 addition
and
71 deletion
+137
-71
escheduler-alert/pom.xml
escheduler-alert/pom.xml
+1
-1
escheduler-api/pom.xml
escheduler-api/pom.xml
+1
-1
escheduler-api/src/main/java/cn/escheduler/api/quartz/ProcessScheduleJob.java
...ain/java/cn/escheduler/api/quartz/ProcessScheduleJob.java
+1
-1
escheduler-common/pom.xml
escheduler-common/pom.xml
+1
-1
escheduler-common/src/main/java/cn/escheduler/common/Constants.java
...-common/src/main/java/cn/escheduler/common/Constants.java
+7
-23
escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java
...src/main/java/cn/escheduler/common/utils/CommonUtils.java
+0
-7
escheduler-common/src/main/resources/common/common.properties
...eduler-common/src/main/resources/common/common.properties
+0
-1
escheduler-dao/pom.xml
escheduler-dao/pom.xml
+1
-1
escheduler-rpc/pom.xml
escheduler-rpc/pom.xml
+1
-1
escheduler-server/pom.xml
escheduler-server/pom.xml
+1
-1
escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java
...a/cn/escheduler/server/worker/runner/FetchTaskThread.java
+1
-1
escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
.../escheduler/server/worker/task/PythonCommandExecutor.java
+55
-18
escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java
...a/cn/escheduler/server/worker/task/python/PythonTask.java
+1
-1
escheduler-server/src/test/java/cn/escheduler/server/worker/EnvFileTest.java
...rc/test/java/cn/escheduler/server/worker/EnvFileTest.java
+65
-0
pom.xml
pom.xml
+1
-1
script/env/escheduler_env.py
script/env/escheduler_env.py
+0
-12
未找到文件。
escheduler-alert/pom.xml
浏览文件 @
267df899
...
...
@@ -4,7 +4,7 @@
<parent>
<groupId>
cn.analysys
</groupId>
<artifactId>
escheduler
</artifactId>
<version>
1.0.
0
-SNAPSHOT
</version>
<version>
1.0.
1
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-alert
</artifactId>
<packaging>
jar
</packaging>
...
...
escheduler-api/pom.xml
浏览文件 @
267df899
...
...
@@ -3,7 +3,7 @@
<parent>
<groupId>
cn.analysys
</groupId>
<artifactId>
escheduler
</artifactId>
<version>
1.0.
0
-SNAPSHOT
</version>
<version>
1.0.
1
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-api
</artifactId>
<packaging>
jar
</packaging>
...
...
escheduler-api/src/main/java/cn/escheduler/api/quartz/ProcessScheduleJob.java
浏览文件 @
267df899
...
...
@@ -125,7 +125,7 @@ public class ProcessScheduleJob implements Job {
}
Command
command
=
new
Command
();
command
.
setCommandType
(
CommandType
.
S
TART_PROCESS
);
command
.
setCommandType
(
CommandType
.
S
CHEDULER
);
command
.
setExecutorId
(
schedule
.
getUserId
());
command
.
setFailureStrategy
(
schedule
.
getFailureStrategy
());
command
.
setProcessDefinitionId
(
schedule
.
getProcessDefinitionId
());
...
...
escheduler-common/pom.xml
浏览文件 @
267df899
...
...
@@ -4,7 +4,7 @@
<parent>
<artifactId>
escheduler
</artifactId>
<groupId>
cn.analysys
</groupId>
<version>
1.0.
0
-SNAPSHOT
</version>
<version>
1.0.
1
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-common
</artifactId>
<name>
escheduler-common
</name>
...
...
escheduler-common/src/main/java/cn/escheduler/common/Constants.java
浏览文件 @
267df899
...
...
@@ -70,25 +70,6 @@ public final class Constants {
*/
public
static
final
String
YARN_APPLICATION_STATUS_ADDRESS
=
"yarn.application.status.address"
;
/**
* spring.redis.maxIdle
*/
public
static
final
String
SPRING_REDIS_MAXIDLE
=
"spring.redis.maxIdle"
;
/**
* spring.redis.maxTotal
*/
public
static
final
String
SPRING_REDIS_MAXTOTAL
=
"spring.redis.maxTotal"
;
/**
* spring.redis.host
*/
public
static
final
String
SPRING_REDIS_HOST
=
"spring.redis.host"
;
/**
* spring.redis.port
*/
public
static
final
String
SPRING_REDIS_PORT
=
"spring.redis.port"
;
/**
* hdfs configuration
...
...
@@ -117,9 +98,14 @@ public final class Constants {
public
static
final
String
ESCHEDULER_ENV_PATH
=
"escheduler.env.path"
;
/**
* escheduler.env.
py
* escheduler.env.
sh
*/
public
static
final
String
ESCHEDULER_ENV_PY
=
"escheduler.env.py"
;
public
static
final
String
ESCHEDULER_ENV_SH
=
".escheduler_env.sh"
;
/**
* python home
*/
public
static
final
String
PYTHON_HOME
=
"PYTHON_HOME"
;
/**
* resource.view.suffixs
...
...
@@ -255,8 +241,6 @@ public final class Constants {
public
static
final
String
SCHEDULER_QUEUE_IMPL
=
"escheduler.queue.impl"
;
public
static
final
String
SCHEDULER_QUEUE_REDIS_IMPL
=
"redis"
;
/**
* date format of yyyy-MM-dd HH:mm:ss
...
...
escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java
浏览文件 @
267df899
...
...
@@ -46,13 +46,6 @@ public class CommonUtils {
return
envPath
;
}
/**
* @return get the path of Python system environment variables
*/
public
static
String
getPythonSystemEnvPath
()
{
return
getString
(
ESCHEDULER_ENV_PY
);
}
/**
* @return get queue implementation name
*/
...
...
escheduler-common/src/main/resources/common/common.properties
浏览文件 @
267df899
...
...
@@ -18,7 +18,6 @@ hdfs.startup.state=true
# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions
escheduler.env.path
=
/opt/.escheduler_env.sh
escheduler.env.py
=
/opt/escheduler_env.py
#resource.view.suffixs
resource.view.suffixs
=
txt,log,sh,conf,cfg,py,java,sql,hql,xml
...
...
escheduler-dao/pom.xml
浏览文件 @
267df899
...
...
@@ -4,7 +4,7 @@
<parent>
<groupId>
cn.analysys
</groupId>
<artifactId>
escheduler
</artifactId>
<version>
1.0.
0
-SNAPSHOT
</version>
<version>
1.0.
1
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-dao
</artifactId>
<name>
escheduler-dao
</name>
...
...
escheduler-rpc/pom.xml
浏览文件 @
267df899
...
...
@@ -4,7 +4,7 @@
<parent>
<artifactId>
escheduler
</artifactId>
<groupId>
cn.analysys
</groupId>
<version>
1.0.
0
-SNAPSHOT
</version>
<version>
1.0.
1
-SNAPSHOT
</version>
</parent>
<modelVersion>
4.0.0
</modelVersion>
...
...
escheduler-server/pom.xml
浏览文件 @
267df899
...
...
@@ -3,7 +3,7 @@
<parent>
<artifactId>
escheduler
</artifactId>
<groupId>
cn.analysys
</groupId>
<version>
1.0.
0
-SNAPSHOT
</version>
<version>
1.0.
1
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-server
</artifactId>
<name>
escheduler-server
</name>
...
...
escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java
浏览文件 @
267df899
...
...
@@ -172,7 +172,7 @@ public class FetchTaskThread implements Runnable{
FileUtils
.
createWorkDirAndUserIfAbsent
(
execLocalPath
,
processInstance
.
getTenantCode
(),
logger
);
logger
.
info
(
"task : {} ready to submit to task scheduler thread"
,
taskId
);
// submit task
workerExecService
.
submit
(
new
TaskScheduleThread
(
taskInstance
,
processDao
));
}
...
...
escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
浏览文件 @
267df899
...
...
@@ -16,12 +16,13 @@
*/
package
cn.escheduler.server.worker.task
;
import
cn.escheduler.common.Constants
;
import
cn.escheduler.common.utils.FileUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.io.File
;
import
java.io.IOException
;
import
java.io.*
;
import
java.nio.charset.StandardCharsets
;
import
java.nio.file.Files
;
import
java.nio.file.Paths
;
...
...
@@ -34,6 +35,8 @@ import java.util.function.Consumer;
*/
public
class
PythonCommandExecutor
extends
AbstractCommandExecutor
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
PythonCommandExecutor
.
class
);
public
static
final
String
PYTHON
=
"python"
;
...
...
@@ -63,27 +66,13 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
*/
@Override
protected
void
createCommandFileIfNotExists
(
String
execCommand
,
String
commandFile
)
throws
IOException
{
logger
.
info
(
"
proxy user
:{}, work dir:{}"
,
tenantCode
,
taskDir
);
logger
.
info
(
"
tenant
:{}, work dir:{}"
,
tenantCode
,
taskDir
);
if
(!
Files
.
exists
(
Paths
.
get
(
commandFile
)))
{
logger
.
info
(
"generate command file:{}"
,
commandFile
);
StringBuilder
sb
=
new
StringBuilder
(
200
);
sb
.
append
(
"#-*- encoding=utf8 -*-\n"
);
sb
.
append
(
"import os,sys\n"
);
sb
.
append
(
"BASEDIR = os.path.dirname(os.path.realpath(__file__))\n"
);
sb
.
append
(
"os.chdir(BASEDIR)\n"
);
if
(
StringUtils
.
isNotEmpty
(
envFile
))
{
String
[]
envArray
=
envFile
.
split
(
"\\."
);
if
(
envArray
.
length
==
2
){
String
path
=
envArray
[
0
];
logger
.
info
(
"path:"
+
path
);
int
index
=
path
.
lastIndexOf
(
"/"
);
sb
.
append
(
String
.
format
(
"sys.path.append('%s')\n"
,
path
.
substring
(
0
,
index
)));
sb
.
append
(
String
.
format
(
"import %s\n"
,
path
.
substring
(
index
+
1
)));
}
}
sb
.
append
(
"\n\n"
);
sb
.
append
(
String
.
format
(
"import py_%s_node\n"
,
taskAppId
));
...
...
@@ -96,7 +85,14 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
@Override
protected
String
commandType
()
{
return
PYTHON
;
String
envPath
=
System
.
getProperty
(
"user.dir"
)
+
Constants
.
SINGLE_SLASH
+
"conf"
+
Constants
.
SINGLE_SLASH
+
"env"
+
Constants
.
SINGLE_SLASH
+
Constants
.
ESCHEDULER_ENV_SH
;
String
pythonHome
=
getPythonHome
(
envPath
);
if
(
StringUtils
.
isEmpty
(
pythonHome
)){
return
PYTHON
;
}
return
pythonHome
;
}
@Override
...
...
@@ -109,4 +105,45 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
return
true
;
}
/**
* get python home
* @param envPath
* @return
*/
private
static
String
getPythonHome
(
String
envPath
){
BufferedReader
br
=
null
;
String
line
=
null
;
StringBuilder
sb
=
new
StringBuilder
();
try
{
br
=
new
BufferedReader
(
new
InputStreamReader
(
new
FileInputStream
(
envPath
)));
while
((
line
=
br
.
readLine
())
!=
null
){
if
(
line
.
contains
(
Constants
.
PYTHON_HOME
)){
sb
.
append
(
line
);
break
;
}
}
String
result
=
sb
.
toString
();
if
(
org
.
apache
.
commons
.
lang
.
StringUtils
.
isEmpty
(
result
)){
return
null
;
}
String
[]
arrs
=
result
.
split
(
"="
);
if
(
arrs
.
length
==
2
){
return
arrs
[
1
];
}
}
catch
(
IOException
e
){
logger
.
error
(
"read file failed : "
+
e
.
getMessage
(),
e
);
}
finally
{
try
{
if
(
br
!=
null
){
br
.
close
();
}
}
catch
(
IOException
e
)
{
logger
.
error
(
e
.
getMessage
(),
e
);
}
}
return
null
;
}
}
escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java
浏览文件 @
267df899
...
...
@@ -72,7 +72,7 @@ public class PythonTask extends AbstractTask {
this
.
pythonProcessTask
=
new
PythonCommandExecutor
(
this
::
logHandle
,
taskProps
.
getTaskDir
(),
taskProps
.
getTaskAppId
(),
taskProps
.
getTenantCode
(),
CommonUtils
.
getPythonSystemEnvPath
()
,
taskProps
.
getTaskStartTime
(),
taskProps
.
getTenantCode
(),
null
,
taskProps
.
getTaskStartTime
(),
taskProps
.
getTaskTimeout
(),
logger
);
this
.
processDao
=
DaoFactory
.
getDaoInstance
(
ProcessDao
.
class
);
}
...
...
escheduler-server/src/test/java/cn/escheduler/server/worker/EnvFileTest.java
0 → 100644
浏览文件 @
267df899
package
cn.escheduler.server.worker
;
import
org.apache.commons.lang.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.io.BufferedReader
;
import
java.io.FileInputStream
;
import
java.io.IOException
;
import
java.io.InputStreamReader
;
/**
* Created by qiaozhanwei on 2019/4/15.
*/
public
class
EnvFileTest
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
EnvFileTest
.
class
);
public
static
void
main
(
String
[]
args
)
{
String
path
=
System
.
getProperty
(
"user.dir"
)+
"\\script\\env\\.escheduler_env.sh"
;
String
pythonHome
=
getPythonHome
(
path
);
logger
.
info
(
pythonHome
);
}
/**
* get python home
* @param path
* @return
*/
private
static
String
getPythonHome
(
String
path
){
BufferedReader
br
=
null
;
String
line
=
null
;
StringBuilder
sb
=
new
StringBuilder
();
try
{
br
=
new
BufferedReader
(
new
InputStreamReader
(
new
FileInputStream
(
path
)));
while
((
line
=
br
.
readLine
())
!=
null
){
if
(
line
.
contains
(
"PYTHON_HOME"
)){
sb
.
append
(
line
);
break
;
}
}
String
result
=
sb
.
toString
();
if
(
StringUtils
.
isEmpty
(
result
)){
return
null
;
}
String
[]
arrs
=
result
.
split
(
"="
);
if
(
arrs
.
length
==
2
){
return
arrs
[
1
];
}
}
catch
(
IOException
e
){
logger
.
error
(
"read file failed : "
+
e
.
getMessage
(),
e
);
}
finally
{
try
{
if
(
br
!=
null
){
br
.
close
();
}
}
catch
(
IOException
e
)
{
logger
.
error
(
e
.
getMessage
(),
e
);
}
}
return
null
;
}
}
pom.xml
浏览文件 @
267df899
...
...
@@ -3,7 +3,7 @@
<modelVersion>
4.0.0
</modelVersion>
<groupId>
cn.analysys
</groupId>
<artifactId>
escheduler
</artifactId>
<version>
1.0.
0
-SNAPSHOT
</version>
<version>
1.0.
1
-SNAPSHOT
</version>
<packaging>
pom
</packaging>
<name>
escheduler
</name>
<url>
http://maven.apache.org
</url>
...
...
script/env/escheduler_env.py
已删除
100644 → 0
浏览文件 @
9df5d43e
import
os
HADOOP_HOME
=
"/opt/soft/hadoop"
SPARK_HOME1
=
"/opt/soft/spark1"
SPARK_HOME2
=
"/opt/soft/spark2"
PYTHON_HOME
=
"/opt/soft/python"
JAVA_HOME
=
"/opt/soft/java"
HIVE_HOME
=
"/opt/soft/hive"
PATH
=
os
.
environ
[
'PATH'
]
PATH
=
"%s/bin:%s/bin:%s/bin:%s/bin:%s/bin:%s/bin:%s"
%
(
HIVE_HOME
,
HADOOP_HOME
,
SPARK_HOME1
,
SPARK_HOME2
,
JAVA_HOME
,
PYTHON_HOME
,
PATH
)
os
.
putenv
(
'PATH'
,
'%s'
%
PATH
)
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录