Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
#法外狂徒张三
DolphinScheduler
提交
9f975e0a
DolphinScheduler
项目概览
#法外狂徒张三
/
DolphinScheduler
与 Fork 源项目一致
Fork自
apache / DolphinScheduler
通知
12
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9f975e0a
编写于
7月 09, 2019
作者:
L
lidongdai
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'upstream/dev-1.1.0' into dev-1.1.0
上级
92e182e6
e2f73b71
变更
16
隐藏空白更改
内联
并排
Showing
16 changed file
with
75 addition
and
45 deletion
+75
-45
escheduler-alert/pom.xml
escheduler-alert/pom.xml
+1
-1
escheduler-api/pom.xml
escheduler-api/pom.xml
+6
-3
escheduler-common/pom.xml
escheduler-common/pom.xml
+1
-1
escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java
...main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java
+23
-3
escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java
...st/java/cn/escheduler/common/queue/TaskQueueImplTest.java
+8
-6
escheduler-dao/pom.xml
escheduler-dao/pom.xml
+5
-1
escheduler-rpc/pom.xml
escheduler-rpc/pom.xml
+1
-1
escheduler-server/pom.xml
escheduler-server/pom.xml
+1
-1
escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
.../escheduler/server/worker/task/PythonCommandExecutor.java
+5
-4
escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java
...a/cn/escheduler/server/worker/task/python/PythonTask.java
+15
-14
escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue
...c/js/conf/home/pages/monitor/pages/servers/statistics.vue
+1
-1
escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
...s/projects/pages/definition/pages/list/_source/timing.vue
+0
-5
escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue
...nf/home/pages/security/pages/users/_source/createUser.vue
+6
-2
install.sh
install.sh
+1
-0
pom.xml
pom.xml
+1
-1
sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql
sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql
+0
-1
未找到文件。
escheduler-alert/pom.xml
浏览文件 @
9f975e0a
...
...
@@ -4,7 +4,7 @@
<parent>
<groupId>
cn.analysys
</groupId>
<artifactId>
escheduler
</artifactId>
<version>
1.
0.4
-SNAPSHOT
</version>
<version>
1.
1.0
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-alert
</artifactId>
<packaging>
jar
</packaging>
...
...
escheduler-api/pom.xml
浏览文件 @
9f975e0a
<project
xmlns=
"http://maven.apache.org/POM/4.0.0"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation=
"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
>
<project
xmlns=
"http://maven.apache.org/POM/4.0.0"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation=
"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
>
<modelVersion>
4.0.0
</modelVersion>
<parent>
<groupId>
cn.analysys
</groupId>
<artifactId>
escheduler
</artifactId>
<version>
1.
0.4
-SNAPSHOT
</version>
<version>
1.
1.0
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-api
</artifactId>
<packaging>
jar
</packaging>
...
...
@@ -48,6 +47,10 @@
<groupId>
org.springframework.boot
</groupId>
<artifactId>
spring-boot-starter-tomcat
</artifactId>
</exclusion>
<exclusion>
<artifactId>
log4j-to-slf4j
</artifactId>
<groupId>
org.apache.logging.log4j
</groupId>
</exclusion>
</exclusions>
</dependency>
...
...
escheduler-common/pom.xml
浏览文件 @
9f975e0a
...
...
@@ -4,7 +4,7 @@
<parent>
<artifactId>
escheduler
</artifactId>
<groupId>
cn.analysys
</groupId>
<version>
1.
0.4
-SNAPSHOT
</version>
<version>
1.
1.0
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-common
</artifactId>
<name>
escheduler-common
</name>
...
...
escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java
浏览文件 @
9f975e0a
...
...
@@ -151,7 +151,27 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
int
size
=
list
.
size
();
Set
<
String
>
taskTreeSet
=
new
TreeSet
<>();
Set
<
String
>
taskTreeSet
=
new
TreeSet
<>(
new
Comparator
<
String
>()
{
@Override
public
int
compare
(
String
o1
,
String
o2
)
{
String
s1
=
o1
;
String
s2
=
o2
;
String
[]
s1Array
=
s1
.
split
(
Constants
.
UNDERLINE
);
if
(
s1Array
.
length
>
4
){
// warning: if this length > 5, need to be changed
s1
=
s1
.
substring
(
0
,
s1
.
lastIndexOf
(
Constants
.
UNDERLINE
)
);
}
String
[]
s2Array
=
s2
.
split
(
Constants
.
UNDERLINE
);
if
(
s2Array
.
length
>
4
){
// warning: if this length > 5, need to be changed
s2
=
s2
.
substring
(
0
,
s2
.
lastIndexOf
(
Constants
.
UNDERLINE
)
);
}
return
s1
.
compareTo
(
s2
);
}
});
for
(
int
i
=
0
;
i
<
size
;
i
++)
{
...
...
@@ -173,8 +193,8 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
continue
;
}
}
formatTask
+=
Constants
.
UNDERLINE
+
taskDetailArrs
[
4
];
}
taskTreeSet
.
add
(
formatTask
);
}
...
...
@@ -229,7 +249,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue {
int
taskId
=
Integer
.
parseInt
(
taskArray
[
3
]);
StringBuilder
sb
=
new
StringBuilder
(
50
);
String
destTask
=
String
.
format
(
"%s_%s_%s_%s"
,
taskArray
[
0
],
processInstanceId
,
taskArray
[
3
],
taskId
);
String
destTask
=
String
.
format
(
"%s_%s_%s_%s"
,
taskArray
[
0
],
processInstanceId
,
taskArray
[
2
],
taskId
);
sb
.
append
(
destTask
);
...
...
escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java
浏览文件 @
9f975e0a
...
...
@@ -17,6 +17,8 @@
package
cn.escheduler.common.queue
;
import
cn.escheduler.common.Constants
;
import
cn.escheduler.common.utils.IpUtils
;
import
cn.escheduler.common.utils.OSUtils
;
import
org.junit.After
;
import
org.junit.Assert
;
import
org.junit.Before
;
...
...
@@ -58,31 +60,31 @@ public class TaskQueueImplTest {
@Test
public
void
testAdd
(){
//add
tasksQueue
.
add
(
Constants
.
SCHEDULER_TASKS_QUEUE
,
"1_
1_1_1_2130706433,3232236775
"
);
tasksQueue
.
add
(
Constants
.
SCHEDULER_TASKS_QUEUE
,
"1_
0_1_1_-1
"
);
tasksQueue
.
add
(
Constants
.
SCHEDULER_TASKS_QUEUE
,
"0_1_1_1_2130706433,3232236775"
);
tasksQueue
.
add
(
Constants
.
SCHEDULER_TASKS_QUEUE
,
"1_1_0_1_2130706433,3232236775
"
);
tasksQueue
.
add
(
Constants
.
SCHEDULER_TASKS_QUEUE
,
"1_1_0_1_2130706433,3232236775
,"
+
IpUtils
.
ipToLong
(
OSUtils
.
getHost
())
);
tasksQueue
.
add
(
Constants
.
SCHEDULER_TASKS_QUEUE
,
"1_2_1_1_2130706433,3232236775"
);
List
<
String
>
tasks
=
tasksQueue
.
poll
(
Constants
.
SCHEDULER_TASKS_QUEUE
,
1
);
if
(
tasks
.
size
()
<
0
){
if
(
tasks
.
size
()
<
=
0
){
return
;
}
//pop
String
node1
=
tasks
.
get
(
0
);
assertEquals
(
node1
,
"
0_0000000001_1_000000000
1"
);
assertEquals
(
node1
,
"
1_0_1_1_-
1"
);
tasks
=
tasksQueue
.
poll
(
Constants
.
SCHEDULER_TASKS_QUEUE
,
1
);
if
(
tasks
.
size
()
<
0
){
if
(
tasks
.
size
()
<
=
0
){
return
;
}
String
node2
=
tasks
.
get
(
0
);
assertEquals
(
node2
,
"0_0000000001_1_0000000001"
);
}
...
...
escheduler-dao/pom.xml
浏览文件 @
9f975e0a
...
...
@@ -4,7 +4,7 @@
<parent>
<groupId>
cn.analysys
</groupId>
<artifactId>
escheduler
</artifactId>
<version>
1.
0.4
-SNAPSHOT
</version>
<version>
1.
1.0
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-dao
</artifactId>
<name>
escheduler-dao
</name>
...
...
@@ -37,6 +37,10 @@
<groupId>
org.apache.tomcat
</groupId>
<artifactId>
tomcat-jdbc
</artifactId>
</exclusion>
<exclusion>
<artifactId>
log4j-to-slf4j
</artifactId>
<groupId>
org.apache.logging.log4j
</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
...
...
escheduler-rpc/pom.xml
浏览文件 @
9f975e0a
...
...
@@ -4,7 +4,7 @@
<parent>
<artifactId>
escheduler
</artifactId>
<groupId>
cn.analysys
</groupId>
<version>
1.
0.4
-SNAPSHOT
</version>
<version>
1.
1.0
-SNAPSHOT
</version>
</parent>
<modelVersion>
4.0.0
</modelVersion>
...
...
escheduler-server/pom.xml
浏览文件 @
9f975e0a
...
...
@@ -3,7 +3,7 @@
<parent>
<artifactId>
escheduler
</artifactId>
<groupId>
cn.analysys
</groupId>
<version>
1.
0.4
-SNAPSHOT
</version>
<version>
1.
1.0
-SNAPSHOT
</version>
</parent>
<artifactId>
escheduler-server
</artifactId>
<name>
escheduler-server
</name>
...
...
escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java
浏览文件 @
9f975e0a
...
...
@@ -18,6 +18,7 @@ package cn.escheduler.server.worker.task;
import
cn.escheduler.common.Constants
;
import
cn.escheduler.common.utils.FileUtils
;
import
cn.escheduler.common.utils.PropertyUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -71,11 +72,11 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
if
(!
Files
.
exists
(
Paths
.
get
(
commandFile
)))
{
logger
.
info
(
"generate command file:{}"
,
commandFile
);
StringBuilder
sb
=
new
StringBuilder
(
200
);
StringBuilder
sb
=
new
StringBuilder
();
sb
.
append
(
"#-*- encoding=utf8 -*-\n"
);
sb
.
append
(
"\n\n"
);
sb
.
append
(
String
.
format
(
"import py_%s_node\n"
,
taskAppId
)
);
sb
.
append
(
execCommand
);
logger
.
info
(
sb
.
toString
());
// write data to file
...
...
@@ -86,8 +87,8 @@ public class PythonCommandExecutor extends AbstractCommandExecutor {
@Override
protected
String
commandType
()
{
String
envPath
=
System
.
getProperty
(
"user.dir"
)
+
Constants
.
SINGLE_SLASH
+
"conf"
+
Constants
.
SINGLE_SLASH
+
"env"
+
Constants
.
SINGLE_SLASH
+
Constants
.
ESCHEDULER_ENV_SH
;
String
envPath
=
PropertyUtils
.
getString
(
Constants
.
ESCHEDULER_ENV_PATH
);
String
pythonHome
=
getPythonHome
(
envPath
);
if
(
StringUtils
.
isEmpty
(
pythonHome
)){
return
PYTHON
;
...
...
escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java
浏览文件 @
9f975e0a
...
...
@@ -112,14 +112,14 @@ public class PythonTask extends AbstractTask {
*/
private
String
buildCommand
()
throws
Exception
{
// generate scripts
String
fileName
=
String
.
format
(
"%s/py_%s_node.py"
,
taskDir
,
taskProps
.
getTaskAppId
());
Path
path
=
new
File
(
fileName
).
toPath
();
//
String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId());
//
Path path = new File(fileName).toPath();
if
(
Files
.
exists
(
path
))
{
return
fileName
;
}
//
if (Files.exists(path)) {
//
return fileName;
//
}
String
rawScript
=
pythonParameters
.
getRawScript
().
replaceAll
(
"\\r\\n"
,
"\n"
);
...
...
@@ -140,19 +140,20 @@ public class PythonTask extends AbstractTask {
}
pythonParameters
.
setRawScript
(
rawScript
);
//
pythonParameters.setRawScript(rawScript);
logger
.
info
(
"raw script : {}"
,
pythonParameters
.
getRawScript
());
logger
.
info
(
"task dir : {}"
,
taskDir
);
Set
<
PosixFilePermission
>
perms
=
PosixFilePermissions
.
fromString
(
"rwxr-xr-x"
);
FileAttribute
<
Set
<
PosixFilePermission
>>
attr
=
PosixFilePermissions
.
asFileAttribute
(
perms
);
Files
.
createFile
(
path
,
attr
);
Files
.
write
(
path
,
pythonParameters
.
getRawScript
().
getBytes
(),
StandardOpenOption
.
APPEND
);
return
fileName
;
// Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxr-xr-x");
// FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
//
// Files.createFile(path, attr);
//
// Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
//
// return fileName;
return
rawScript
;
}
@Override
...
...
escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue
浏览文件 @
9f975e0a
...
...
@@ -16,7 +16,7 @@
<div
class=
"col-md-3"
>
<div
class=
"text-num-model text"
>
<div
class=
"title"
>
<span
>
{{
$t
(
'
failure command number
'
)
}}
}
</span>
<span
>
{{
$t
(
'
failure command number
'
)
}}
</span>
</div>
<div
class=
"value-p"
>
<b
:style=
"
{color:color[1]}">
{{
commandCountData
.
errorCount
}}
</b>
...
...
escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
浏览文件 @
9f975e0a
...
...
@@ -254,11 +254,6 @@
this
.
store
.
dispatch
(
api
,
searchParams
).
then
(
res
=>
{
this
.
previewTimes
=
res
if
(
this
.
previewTimes
.
length
)
{
resolve
()
}
else
{
reject
(
new
Error
(
0
))
}
})
}
},
...
...
escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue
浏览文件 @
9f975e0a
...
...
@@ -131,7 +131,8 @@
}
},
_verification
()
{
let
regEmail
=
/^
([
a-zA-Z0-9
]
+
[
_|
\-
|
\.]?)
*
[
a-zA-Z0-9
]
+@
([
a-zA-Z0-9
]
+
[
_|
\-
|
\.]?)
*
[
a-zA-Z0-9
]
+
\.[
a-zA-Z
]{2,3}
$/
// eslint-disable-line
let
regEmail
=
/^
([
a-zA-Z0-9
]
+
[
_|
\-
|
\.]?)
*
[
a-zA-Z0-9
]
+@
([
a-zA-Z0-9
]
+
[
_|
\-
|
\.]?)
*
[
a-zA-Z0-9
]
+
\.[
a-zA-Z
]{2,}
$/
// eslint-disable-line
// Mobile phone number regular
let
regPhone
=
/^1
(
3|4|5|6|7|8
)\d{9}
$/
;
// eslint-disable-line
...
...
@@ -184,7 +185,10 @@
_getTenantList
()
{
return
new
Promise
((
resolve
,
reject
)
=>
{
this
.
store
.
dispatch
(
'
security/getTenantList
'
).
then
(
res
=>
{
this
.
tenantList
=
_
.
map
(
res
,
v
=>
{
let
arr
=
_
.
filter
(
res
,
(
o
)
=>
{
return
o
.
id
!==
-
1
})
this
.
tenantList
=
_
.
map
(
arr
,
v
=>
{
return
{
id
:
v
.
id
,
code
:
v
.
tenantName
...
...
install.sh
浏览文件 @
9f975e0a
...
...
@@ -144,6 +144,7 @@ singleYarnIp="ark1"
hdfsPath
=
"/escheduler"
# 拥有在hdfs根路径/下创建目录权限的用户
# 注意:如果开启了kerberos,则直接hdfsRootUser=,就可以
hdfsRootUser
=
"hdfs"
# common 配置
...
...
pom.xml
浏览文件 @
9f975e0a
...
...
@@ -3,7 +3,7 @@
<modelVersion>
4.0.0
</modelVersion>
<groupId>
cn.analysys
</groupId>
<artifactId>
escheduler
</artifactId>
<version>
1.
0.4
-SNAPSHOT
</version>
<version>
1.
1.0
-SNAPSHOT
</version>
<packaging>
pom
</packaging>
<name>
escheduler
</name>
<url>
http://maven.apache.org
</url>
...
...
sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql
浏览文件 @
9f975e0a
INSERT
INTO
`t_escheduler_version`
(
`version`
)
VALUES
(
'1.1.0'
);
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录