Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
deeplearning
DolphinScheduler
提交
c53f5bcc
DolphinScheduler
项目概览
deeplearning
/
DolphinScheduler
与 Fork 源项目一致
Fork自
apache / DolphinScheduler
通知
24
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
c53f5bcc
编写于
7月 02, 2019
作者:
journey2018
提交者:
GitHub
7月 02, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #507 from qiaozhanwei/dev-1.1.0
Kerberos integrated
上级
8464c3e0
c9c91a45
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
37 addition
and
20 deletion
+37
-20
escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java
...va/cn/escheduler/api/controller/DataSourceController.java
+2
-1
escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java
...ain/java/cn/escheduler/api/service/DataSourceService.java
+3
-2
escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java
...api/src/main/java/cn/escheduler/api/utils/CheckUtils.java
+0
-12
escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java
...src/main/java/cn/escheduler/common/utils/CommonUtils.java
+11
-0
escheduler-common/src/main/resources/common/common.properties
...eduler-common/src/main/resources/common/common.properties
+2
-2
escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java
...a/cn/escheduler/server/worker/runner/FetchTaskThread.java
+4
-1
escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java
...in/java/cn/escheduler/server/worker/task/sql/SqlTask.java
+15
-2
未找到文件。
escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java
浏览文件 @
c53f5bcc
...
...
@@ -23,6 +23,7 @@ import cn.escheduler.api.utils.Constants;
import
cn.escheduler.api.utils.Result
;
import
cn.escheduler.common.enums.DbType
;
import
cn.escheduler.common.enums.ResUploadType
;
import
cn.escheduler.common.utils.CommonUtils
;
import
cn.escheduler.common.utils.ParameterUtils
;
import
cn.escheduler.common.utils.PropertyUtils
;
import
cn.escheduler.dao.model.User
;
...
...
@@ -455,7 +456,7 @@ public class DataSourceController extends BaseController {
logger
.
info
(
"login user {},get kerberos startup state : {}"
,
loginUser
.
getUserName
());
try
{
// if upload resource is HDFS and kerberos startup is true , else false
return
success
(
Status
.
SUCCESS
.
getMsg
(),
C
heck
Utils
.
getKerberosStartupState
());
return
success
(
Status
.
SUCCESS
.
getMsg
(),
C
ommon
Utils
.
getKerberosStartupState
());
}
catch
(
Exception
e
){
logger
.
error
(
KERBEROS_STARTUP_STATE
.
getMsg
(),
e
);
return
error
(
Status
.
KERBEROS_STARTUP_STATE
.
getCode
(),
Status
.
KERBEROS_STARTUP_STATE
.
getMsg
());
...
...
escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java
浏览文件 @
c53f5bcc
...
...
@@ -25,6 +25,7 @@ import cn.escheduler.common.enums.DbType;
import
cn.escheduler.common.enums.ResUploadType
;
import
cn.escheduler.common.enums.UserType
;
import
cn.escheduler.common.job.db.*
;
import
cn.escheduler.common.utils.CommonUtils
;
import
cn.escheduler.common.utils.PropertyUtils
;
import
cn.escheduler.dao.mapper.DataSourceMapper
;
import
cn.escheduler.dao.mapper.DatasourceUserMapper
;
...
...
@@ -381,7 +382,7 @@ public class DataSourceService extends BaseService{
break
;
case
HIVE:
case
SPARK:
if
(
C
heck
Utils
.
getKerberosStartupState
())
{
if
(
C
ommon
Utils
.
getKerberosStartupState
())
{
System
.
setProperty
(
cn
.
escheduler
.
common
.
Constants
.
JAVA_SECURITY_KRB5_CONF
,
getString
(
cn
.
escheduler
.
common
.
Constants
.
JAVA_SECURITY_KRB5_CONF_PATH
));
Configuration
configuration
=
new
Configuration
();
...
...
@@ -477,7 +478,7 @@ public class DataSourceService extends BaseService{
String
address
=
buildAddress
(
type
,
host
,
port
);
String
jdbcUrl
=
address
+
"/"
+
database
;
if
(
C
heck
Utils
.
getKerberosStartupState
()
&&
if
(
C
ommon
Utils
.
getKerberosStartupState
()
&&
(
type
==
DbType
.
HIVE
||
type
==
DbType
.
SPARK
)){
jdbcUrl
+=
";principal="
+
principal
;
}
...
...
escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java
浏览文件 @
c53f5bcc
...
...
@@ -160,16 +160,4 @@ public class CheckUtils {
return
pattern
.
matcher
(
str
).
matches
();
}
/**
* if upload resource is HDFS and kerberos startup is true , else false
* @return
*/
public
static
boolean
getKerberosStartupState
(){
String
resUploadStartupType
=
PropertyUtils
.
getString
(
cn
.
escheduler
.
common
.
Constants
.
RES_UPLOAD_STARTUP_TYPE
);
ResUploadType
resUploadType
=
ResUploadType
.
valueOf
(
resUploadStartupType
);
Boolean
kerberosStartupState
=
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE
);
return
resUploadType
==
ResUploadType
.
HDFS
&&
kerberosStartupState
;
}
}
escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java
浏览文件 @
c53f5bcc
...
...
@@ -17,6 +17,7 @@
package
cn.escheduler.common.utils
;
import
cn.escheduler.common.Constants
;
import
cn.escheduler.common.enums.ResUploadType
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -63,4 +64,14 @@ public class CommonUtils {
/**
* if upload resource is HDFS and kerberos startup is true , else false
* @return
*/
public
static
boolean
getKerberosStartupState
(){
String
resUploadStartupType
=
PropertyUtils
.
getString
(
cn
.
escheduler
.
common
.
Constants
.
RES_UPLOAD_STARTUP_TYPE
);
ResUploadType
resUploadType
=
ResUploadType
.
valueOf
(
resUploadStartupType
);
Boolean
kerberosStartupState
=
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE
);
return
resUploadType
==
ResUploadType
.
HDFS
&&
kerberosStartupState
;
}
}
escheduler-common/src/main/resources/common/common.properties
浏览文件 @
c53f5bcc
...
...
@@ -26,10 +26,10 @@ hadoop.security.authentication.startup.state=false
java.security.krb5.conf.path
=
/opt/krb5.conf
# loginUserFromKeytab user
login.user.keytab.username
=
"hdfs-mycluster@ESZ.COM"
login.user.keytab.username
=
hdfs-mycluster@ESZ.COM
# loginUserFromKeytab path
login.user.keytab.path
=
"/opt/hdfs.headless.keytab"
login.user.keytab.path
=
/opt/hdfs.headless.keytab
# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions
escheduler.env.path
=
/opt/.escheduler_env.sh
...
...
escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java
浏览文件 @
c53f5bcc
...
...
@@ -211,9 +211,12 @@ public class FetchTaskThread implements Runnable{
// set task execute path
taskInstance
.
setExecutePath
(
execLocalPath
);
Tenant
tenant
=
processDao
.
getTenantForProcess
(
processInstance
.
getTenantId
(),
processDefine
.
getUserId
());
// check and create Linux users
FileUtils
.
createWorkDirAndUserIfAbsent
(
execLocalPath
,
processInstance
.
getTenantCode
(),
logger
);
tenant
.
getTenantCode
(),
logger
);
logger
.
info
(
"task : {} ready to submit to task scheduler thread"
,
taskId
);
// submit task
...
...
escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java
浏览文件 @
c53f5bcc
...
...
@@ -29,6 +29,7 @@ import cn.escheduler.common.task.sql.SqlBinds;
import
cn.escheduler.common.task.sql.SqlParameters
;
import
cn.escheduler.common.task.sql.SqlType
;
import
cn.escheduler.common.utils.CollectionUtils
;
import
cn.escheduler.common.utils.CommonUtils
;
import
cn.escheduler.common.utils.ParameterUtils
;
import
cn.escheduler.dao.AlertDao
;
import
cn.escheduler.dao.DaoFactory
;
...
...
@@ -43,6 +44,8 @@ import com.alibaba.fastjson.JSONObject;
import
com.alibaba.fastjson.serializer.SerializerFeature
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.commons.lang3.EnumUtils
;
import
org.apache.hadoop.conf.Configuration
;
import
org.apache.hadoop.security.UserGroupInformation
;
import
org.slf4j.Logger
;
import
java.sql.*
;
...
...
@@ -51,6 +54,8 @@ import java.util.regex.Matcher;
import
java.util.regex.Pattern
;
import
java.util.stream.Collectors
;
import
static
cn
.
escheduler
.
common
.
utils
.
PropertyUtils
.
getString
;
/**
* sql task
*/
...
...
@@ -228,7 +233,15 @@ public class SqlTask extends AbstractTask {
List
<
String
>
createFuncs
){
Connection
connection
=
null
;
try
{
if
(
CommonUtils
.
getKerberosStartupState
())
{
System
.
setProperty
(
cn
.
escheduler
.
common
.
Constants
.
JAVA_SECURITY_KRB5_CONF
,
getString
(
cn
.
escheduler
.
common
.
Constants
.
JAVA_SECURITY_KRB5_CONF_PATH
));
Configuration
configuration
=
new
Configuration
();
configuration
.
set
(
cn
.
escheduler
.
common
.
Constants
.
HADOOP_SECURITY_AUTHENTICATION
,
"kerberos"
);
UserGroupInformation
.
setConfiguration
(
configuration
);
UserGroupInformation
.
loginUserFromKeytab
(
getString
(
cn
.
escheduler
.
common
.
Constants
.
LOGIN_USER_KEY_TAB_USERNAME
),
getString
(
cn
.
escheduler
.
common
.
Constants
.
LOGIN_USER_KEY_TAB_PATH
));
}
if
(
DbType
.
HIVE
.
name
().
equals
(
sqlParameters
.
getType
()))
{
Properties
paramProp
=
new
Properties
();
paramProp
.
setProperty
(
"user"
,
baseDataSource
.
getUser
());
...
...
@@ -278,7 +291,7 @@ public class SqlTask extends AbstractTask {
array
.
add
(
mapOfColValues
);
}
logger
.
info
(
"execute sql : {}"
,
JSONObject
.
toJSONString
(
array
,
SerializerFeature
.
WriteMapNullValue
));
logger
.
debug
(
"execute sql : {}"
,
JSONObject
.
toJSONString
(
array
,
SerializerFeature
.
WriteMapNullValue
));
// send as an attachment
if
(
StringUtils
.
isEmpty
(
sqlParameters
.
getShowType
()))
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录