Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
善良的小乔
DolphinScheduler
提交
9135b142
DolphinScheduler
项目概览
善良的小乔
/
DolphinScheduler
与 Fork 源项目一致
Fork自
apache / DolphinScheduler
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9135b142
编写于
6月 24, 2019
作者:
journey2018
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add resource upload s3
上级
947f3ade
变更
11
隐藏空白更改
内联
并排
Showing
11 changed file
with
168 addition
and
85 deletion
+168
-85
escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java
...main/java/cn/escheduler/api/service/ResourcesService.java
+20
-20
escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java
...rc/main/java/cn/escheduler/api/service/TenantService.java
+2
-2
escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java
...c/main/java/cn/escheduler/api/service/UdfFuncService.java
+6
-6
escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java
...src/main/java/cn/escheduler/api/service/UsersService.java
+3
-3
escheduler-common/src/main/java/cn/escheduler/common/Constants.java
...-common/src/main/java/cn/escheduler/common/Constants.java
+19
-2
escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java
...c/main/java/cn/escheduler/common/enums/ResUploadType.java
+29
-0
escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java
...src/main/java/cn/escheduler/common/utils/HadoopUtils.java
+51
-36
escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java
...c/main/java/cn/escheduler/common/utils/PropertyUtils.java
+10
-4
escheduler-common/src/main/resources/common/common.properties
...eduler-common/src/main/resources/common/common.properties
+2
-2
escheduler-common/src/main/resources/common/hadoop/hadoop.properties
...common/src/main/resources/common/hadoop/hadoop.properties
+11
-1
install.sh
install.sh
+15
-9
未找到文件。
escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java
浏览文件 @
9135b142
...
...
@@ -86,8 +86,8 @@ public class ResourcesService extends BaseService {
Result
result
=
new
Result
();
// if hdfs not startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
putMsg
(
result
,
Status
.
HDFS_NOT_STARTUP
);
return
result
;
}
...
...
@@ -185,9 +185,9 @@ public class ResourcesService extends BaseService {
ResourceType
type
)
{
Result
result
=
new
Result
();
// if
hdfs not
startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
// if
resource upload
startup
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
putMsg
(
result
,
Status
.
HDFS_NOT_STARTUP
);
return
result
;
}
...
...
@@ -386,9 +386,9 @@ public class ResourcesService extends BaseService {
public
Result
delete
(
User
loginUser
,
int
resourceId
)
throws
Exception
{
Result
result
=
new
Result
();
// if
hdfs not
startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
// if
resource upload
startup
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
putMsg
(
result
,
Status
.
HDFS_NOT_STARTUP
);
return
result
;
}
...
...
@@ -449,9 +449,9 @@ public class ResourcesService extends BaseService {
public
Result
readResource
(
int
resourceId
,
int
skipLineNum
,
int
limit
)
{
Result
result
=
new
Result
();
// if
hdfs not
startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
// if
resource upload
startup
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
putMsg
(
result
,
Status
.
HDFS_NOT_STARTUP
);
return
result
;
}
...
...
@@ -510,9 +510,9 @@ public class ResourcesService extends BaseService {
@Transactional
(
value
=
"TransactionManager"
,
rollbackFor
=
Exception
.
class
)
public
Result
onlineCreateResource
(
User
loginUser
,
ResourceType
type
,
String
fileName
,
String
fileSuffix
,
String
desc
,
String
content
)
{
Result
result
=
new
Result
();
// if
hdfs not
startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
// if
resource upload
startup
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
putMsg
(
result
,
Status
.
HDFS_NOT_STARTUP
);
return
result
;
}
...
...
@@ -573,9 +573,9 @@ public class ResourcesService extends BaseService {
public
Result
updateResourceContent
(
int
resourceId
,
String
content
)
{
Result
result
=
new
Result
();
// if
hdfs not
startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
// if
resource upload
startup
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
putMsg
(
result
,
Status
.
HDFS_NOT_STARTUP
);
return
result
;
}
...
...
@@ -663,9 +663,9 @@ public class ResourcesService extends BaseService {
* @return
*/
public
org
.
springframework
.
core
.
io
.
Resource
downloadResource
(
int
resourceId
)
throws
Exception
{
// if
hdfs not
startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
// if
resource upload
startup
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
throw
new
RuntimeException
(
"hdfs not startup"
);
}
...
...
escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java
浏览文件 @
9135b142
...
...
@@ -96,7 +96,7 @@ public class TenantService extends BaseService{
tenantMapper
.
insert
(
tenant
);
// if hdfs startup
if
(
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
if
(
PropertyUtils
.
get
ResUploadStartupState
(
)){
String
resourcePath
=
HadoopUtils
.
getHdfsDataBasePath
()
+
"/"
+
tenantCode
+
"/resources"
;
String
udfsPath
=
HadoopUtils
.
getHdfsUdfDir
(
tenantCode
);
/**
...
...
@@ -178,7 +178,7 @@ public class TenantService extends BaseService{
Tenant
newTenant
=
tenantMapper
.
queryByTenantCode
(
tenantCode
);
if
(
newTenant
==
null
){
// if hdfs startup
if
(
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
if
(
PropertyUtils
.
get
ResUploadStartupState
(
)){
String
resourcePath
=
HadoopUtils
.
getHdfsDataBasePath
()
+
"/"
+
tenantCode
+
"/resources"
;
String
udfsPath
=
HadoopUtils
.
getHdfsUdfDir
(
tenantCode
);
//init hdfs resource
...
...
escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java
浏览文件 @
9135b142
...
...
@@ -80,9 +80,9 @@ public class UdfFuncService extends BaseService{
int
resourceId
)
{
Result
result
=
new
Result
();
// if
hdfs not
startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
// if
resource upload
startup
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
putMsg
(
result
,
Status
.
HDFS_NOT_STARTUP
);
return
result
;
}
...
...
@@ -167,9 +167,9 @@ public class UdfFuncService extends BaseService{
// verify udfFunc is exist
UdfFunc
udf
=
udfFuncMapper
.
queryUdfById
(
udfFuncId
);
// if
hdfs not
startup
if
(!
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
logger
.
error
(
"
hdfs startup state: {}"
,
PropertyUtils
.
getBoolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
));
// if
resource upload
startup
if
(!
PropertyUtils
.
get
ResUploadStartupState
(
)){
logger
.
error
(
"
resource upload startup state: {}"
,
PropertyUtils
.
getResUploadStartupState
(
));
putMsg
(
result
,
Status
.
HDFS_NOT_STARTUP
);
return
result
;
}
...
...
escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java
浏览文件 @
9135b142
...
...
@@ -125,7 +125,7 @@ public class UsersService extends BaseService {
Tenant
tenant
=
tenantMapper
.
queryById
(
tenantId
);
// if hdfs startup
if
(
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
if
(
PropertyUtils
.
get
ResUploadStartupState
(
)){
String
userPath
=
HadoopUtils
.
getHdfsDataBasePath
()
+
"/"
+
tenant
.
getTenantCode
()
+
"/home/"
+
user
.
getId
();
HadoopUtils
.
getInstance
().
mkdir
(
userPath
);
...
...
@@ -245,7 +245,7 @@ public class UsersService extends BaseService {
Tenant
newTenant
=
tenantMapper
.
queryById
(
tenantId
);
if
(
newTenant
!=
null
)
{
// if hdfs startup
if
(
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
if
(
PropertyUtils
.
get
ResUploadStartupState
(
)){
String
newTenantCode
=
newTenant
.
getTenantCode
();
String
oldResourcePath
=
HadoopUtils
.
getHdfsDataBasePath
()
+
"/"
+
oldTenant
.
getTenantCode
()
+
"/resources"
;
String
oldUdfsPath
=
HadoopUtils
.
getHdfsUdfDir
(
oldTenant
.
getTenantCode
());
...
...
@@ -308,7 +308,7 @@ public class UsersService extends BaseService {
User
user
=
userMapper
.
queryTenantCodeByUserId
(
id
);
if
(
PropertyUtils
.
get
Boolean
(
cn
.
escheduler
.
common
.
Constants
.
HDFS_STARTUP_STATE
)){
if
(
PropertyUtils
.
get
ResUploadStartupState
(
)){
String
userPath
=
HadoopUtils
.
getHdfsDataBasePath
()
+
"/"
+
user
.
getTenantCode
()
+
"/home/"
+
id
;
HadoopUtils
.
getInstance
().
delete
(
userPath
,
true
);
...
...
escheduler-common/src/main/java/cn/escheduler/common/Constants.java
浏览文件 @
9135b142
...
...
@@ -60,6 +60,23 @@ public final class Constants {
*/
public
static
final
String
FS_DEFAULTFS
=
"fs.defaultFS"
;
/**
* fs s3a endpoint
*/
public
static
final
String
FS_S3A_ENDPOINT
=
"fs.s3a.endpoint"
;
/**
* fs s3a access key
*/
public
static
final
String
FS_S3A_ACCESS_KEY
=
"fs.s3a.access.key"
;
/**
* fs s3a secret key
*/
public
static
final
String
FS_S3A_SECRET_KEY
=
"fs.s3a.secret.key"
;
/**
* yarn.resourcemanager.ha.rm.idsfs.defaultFS
*/
...
...
@@ -123,9 +140,9 @@ public final class Constants {
public
static
final
String
DEVELOPMENT_STATE
=
"development.state"
;
/**
*
hdfs.startup.stat
e
*
res.upload.startup.typ
e
*/
public
static
final
String
HDFS_STARTUP_STATE
=
"hdfs.startup.stat
e"
;
public
static
final
String
RES_UPLOAD_STARTUP_TYPE
=
"res.upload.startup.typ
e"
;
/**
* zookeeper quorum
...
...
escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java
0 → 100644
浏览文件 @
9135b142
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
cn.escheduler.common.enums
;
/**
* data base types
*/
public
enum
ResUploadType
{
/**
* 0 hdfs
* 1 s3
* 2 none
*/
HDFS
,
S3
,
NONE
}
escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java
浏览文件 @
9135b142
...
...
@@ -18,6 +18,7 @@ package cn.escheduler.common.utils;
import
cn.escheduler.common.Constants
;
import
cn.escheduler.common.enums.ExecutionStatus
;
import
cn.escheduler.common.enums.ResUploadType
;
import
com.alibaba.fastjson.JSON
;
import
com.alibaba.fastjson.JSONException
;
import
com.alibaba.fastjson.JSONObject
;
...
...
@@ -40,6 +41,7 @@ import java.util.stream.Stream;
import
static
cn
.
escheduler
.
common
.
Constants
.*;
import
static
cn
.
escheduler
.
common
.
utils
.
PropertyUtils
.*;
import
static
cn
.
escheduler
.
common
.
utils
.
PropertyUtils
.
getString
;
/**
* hadoop utils
...
...
@@ -94,48 +96,61 @@ public class HadoopUtils implements Closeable {
try
{
configuration
=
new
Configuration
();
if
(
getBoolean
(
Constants
.
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE
)){
System
.
setProperty
(
Constants
.
JAVA_SECURITY_KRB5_CONF
,
getString
(
Constants
.
JAVA_SECURITY_KRB5_CONF_PATH
));
configuration
.
set
(
Constants
.
HADOOP_SECURITY_AUTHENTICATION
,
"kerberos"
);
UserGroupInformation
.
setConfiguration
(
configuration
);
UserGroupInformation
.
loginUserFromKeytab
(
getString
(
Constants
.
LOGIN_USER_KEY_TAB_USERNAME
),
getString
(
Constants
.
LOGIN_USER_KEY_TAB_PATH
));
}
String
resUploadStartupType
=
PropertyUtils
.
getString
(
Constants
.
RES_UPLOAD_STARTUP_TYPE
);
ResUploadType
resUploadType
=
ResUploadType
.
valueOf
(
resUploadStartupType
);
if
(
resUploadType
==
ResUploadType
.
HDFS
){
if
(
getBoolean
(
Constants
.
HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE
)){
System
.
setProperty
(
Constants
.
JAVA_SECURITY_KRB5_CONF
,
getString
(
Constants
.
JAVA_SECURITY_KRB5_CONF_PATH
));
configuration
.
set
(
Constants
.
HADOOP_SECURITY_AUTHENTICATION
,
"kerberos"
);
UserGroupInformation
.
setConfiguration
(
configuration
);
UserGroupInformation
.
loginUserFromKeytab
(
getString
(
Constants
.
LOGIN_USER_KEY_TAB_USERNAME
),
getString
(
Constants
.
LOGIN_USER_KEY_TAB_PATH
));
}
String
defaultFS
=
configuration
.
get
(
FS_DEFAULTFS
);
//first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file
// the default is the local file system
if
(
defaultFS
.
startsWith
(
"file"
)){
String
defaultFSProp
=
getString
(
FS_DEFAULTFS
);
if
(
StringUtils
.
isNotBlank
(
defaultFSProp
)){
Map
<
String
,
String
>
fsRelatedProps
=
getPrefixedProperties
(
"fs."
);
configuration
.
set
(
FS_DEFAULTFS
,
defaultFSProp
);
fsRelatedProps
.
entrySet
().
stream
().
forEach
(
entry
->
configuration
.
set
(
entry
.
getKey
(),
entry
.
getValue
()));
String
defaultFS
=
configuration
.
get
(
FS_DEFAULTFS
);
//first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file
// the default is the local file system
if
(
defaultFS
.
startsWith
(
"file"
)){
String
defaultFSProp
=
getString
(
FS_DEFAULTFS
);
if
(
StringUtils
.
isNotBlank
(
defaultFSProp
)){
Map
<
String
,
String
>
fsRelatedProps
=
getPrefixedProperties
(
"fs."
);
configuration
.
set
(
FS_DEFAULTFS
,
defaultFSProp
);
fsRelatedProps
.
entrySet
().
stream
().
forEach
(
entry
->
configuration
.
set
(
entry
.
getKey
(),
entry
.
getValue
()));
}
else
{
logger
.
error
(
"property:{} can not to be empty, please set!"
);
throw
new
RuntimeException
(
"property:{} can not to be empty, please set!"
);
}
}
else
{
logger
.
error
(
"property:{} can not to be empty, please set!"
);
throw
new
RuntimeException
(
"property:{} can not to be empty, please set!"
);
logger
.
info
(
"get property:{} -> {}, from core-site.xml hdfs-site.xml "
,
FS_DEFAULTFS
,
defaultFS
);
}
}
else
{
logger
.
info
(
"get property:{} -> {}, from core-site.xml hdfs-site.xml "
,
FS_DEFAULTFS
,
defaultFS
);
}
if
(
fs
==
null
)
{
if
(
StringUtils
.
isNotEmpty
(
hdfsUser
)){
//UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser());
UserGroupInformation
ugi
=
UserGroupInformation
.
createRemoteUser
(
hdfsUser
);
ugi
.
doAs
(
new
PrivilegedExceptionAction
<
Boolean
>()
{
@Override
public
Boolean
run
()
throws
Exception
{
fs
=
FileSystem
.
get
(
configuration
);
return
true
;
}
});
}
else
{
logger
.
warn
(
"hdfs.root.user is not set value!"
);
fs
=
FileSystem
.
get
(
configuration
);
if
(
fs
==
null
)
{
if
(
StringUtils
.
isNotEmpty
(
hdfsUser
)){
//UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser());
UserGroupInformation
ugi
=
UserGroupInformation
.
createRemoteUser
(
hdfsUser
);
ugi
.
doAs
(
new
PrivilegedExceptionAction
<
Boolean
>()
{
@Override
public
Boolean
run
()
throws
Exception
{
fs
=
FileSystem
.
get
(
configuration
);
return
true
;
}
});
}
else
{
logger
.
warn
(
"hdfs.root.user is not set value!"
);
fs
=
FileSystem
.
get
(
configuration
);
}
}
}
else
if
(
resUploadType
==
ResUploadType
.
S3
){
configuration
.
set
(
FS_DEFAULTFS
,
getString
(
FS_DEFAULTFS
));
configuration
.
set
(
FS_S3A_ENDPOINT
,
getString
(
FS_S3A_ENDPOINT
));
configuration
.
set
(
FS_S3A_ACCESS_KEY
,
getString
(
FS_S3A_ACCESS_KEY
));
configuration
.
set
(
FS_S3A_SECRET_KEY
,
getString
(
FS_S3A_SECRET_KEY
));
fs
=
FileSystem
.
get
(
configuration
);
}
String
rmHaIds
=
getString
(
YARN_RESOURCEMANAGER_HA_RM_IDS
);
String
appAddress
=
getString
(
Constants
.
YARN_APPLICATION_STATUS_ADDRESS
);
if
(!
StringUtils
.
isEmpty
(
rmHaIds
))
{
...
...
escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java
浏览文件 @
9135b142
...
...
@@ -16,6 +16,8 @@
*/
package
cn.escheduler.common.utils
;
import
cn.escheduler.common.Constants
;
import
cn.escheduler.common.enums.ResUploadType
;
import
org.apache.commons.io.IOUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -65,11 +67,15 @@ public class PropertyUtils {
}
}
/*
public static PropertyUtils getInstance(){
return propertyUtils;
/**
* judge whether resource upload startup
* @return
*/
public
static
Boolean
getResUploadStartupState
(){
String
resUploadStartupType
=
PropertyUtils
.
getString
(
Constants
.
RES_UPLOAD_STARTUP_TYPE
);
ResUploadType
resUploadType
=
ResUploadType
.
valueOf
(
resUploadStartupType
);
return
resUploadType
==
ResUploadType
.
HDFS
||
resUploadType
==
ResUploadType
.
S3
;
}
*/
/**
* get property value
...
...
escheduler-common/src/main/resources/common/common.properties
浏览文件 @
9135b142
...
...
@@ -16,8 +16,8 @@ hdfs.root.user=hdfs
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/escheduler" is recommended
data.store2hdfs.basepath
=
/escheduler
#
whether hdfs starts
hdfs.startup.state
=
false
#
resource upload startup type : HDFS,S3,NONE
res.upload.startup.type
=
NONE
# whether kerberos starts
hadoop.security.authentication.startup.state
=
false
...
...
escheduler-common/src/main/resources/common/hadoop/hadoop.properties
浏览文件 @
9135b142
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory
# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml
# to the conf directory,support s3,for example : s3a://escheduler
fs.defaultFS
=
hdfs://mycluster:8020
# s3 need,s3 endpoint
fs.s3a.endpoint
=
http://192.168.199.91:9010
# s3 need,s3 access key
fs.s3a.access.key
=
A3DXS30FO22544RE
# s3 need,s3 secret key
fs.s3a.secret.key
=
OloCLq3n+8+sdPHUhJ21XrSxTC+JK
#resourcemanager ha note this need ips , this empty if single
yarn.resourcemanager.ha.rm.ids
=
192.168.xx.xx,192.168.xx.xx
...
...
install.sh
浏览文件 @
9135b142
...
...
@@ -110,14 +110,17 @@ xlsFilePath="/tmp/xls"
#是否启动监控自启动脚本
monitorServerState
=
"false"
# hadoop 配置
# 是否启动hdfs,如果启动则为true,需要配置以下hadoop相关参数;
# 不启动设置为false,如果为false,以下配置不需要修改
# 特别注意:如果启动hdfs,需要自行创建hdfs根路径,也就是install.sh中的 hdfsPath
hdfsStartupSate
=
"false"
# 资源中心上传选择存储方式:HDFS,S3,NONE
resUploadStartupType
=
"NONE"
# namenode地址,支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下
namenodeFs
=
"hdfs://mycluster:8020"
# 如果resUploadStartupType为HDFS,defaultFS写namenode地址,支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下
# 如果是S3,则写S3地址,比如说:s3a://escheduler,注意,一定要创建根目录/escheduler
defaultFS
=
"hdfs://mycluster:8020"
# 如果配置了S3,则需要有以下配置
s3Endpoint
=
"http://192.168.199.91:9010"
s3AccessKey
=
"A3DXS30FO22544RE"
s3SecretKey
=
"OloCLq3n+8+sdPHUhJ21XrSxTC+JK"
# resourcemanager HA配置,如果是单resourcemanager,这里为空即可
yarnHaIps
=
"192.168.xx.xx,192.168.xx.xx"
...
...
@@ -273,7 +276,10 @@ sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.us
sed
-i
${
txt
}
"s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=
${
mysqlPassword
}
#g"
conf/quartz.properties
sed
-i
${
txt
}
"s#fs.defaultFS.*#fs.defaultFS=
${
namenodeFs
}
#g"
conf/common/hadoop/hadoop.properties
sed
-i
${
txt
}
"s#fs.defaultFS.*#fs.defaultFS=
${
defaultFS
}
#g"
conf/common/hadoop/hadoop.properties
sed
-i
${
txt
}
"s#fs.s3a.endpoint.*#fs.s3a.endpoint=
${
s3Endpoint
}
#g"
conf/common/hadoop/hadoop.properties
sed
-i
${
txt
}
"s#fs.s3a.access.key.*#fs.s3a.access.key=
${
s3AccessKey
}
#g"
conf/common/hadoop/hadoop.properties
sed
-i
${
txt
}
"s#fs.s3a.secret.key.*#fs.s3a.secret.key=
${
s3SecretKey
}
#g"
conf/common/hadoop/hadoop.properties
sed
-i
${
txt
}
"s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=
${
yarnHaIps
}
#g"
conf/common/hadoop/hadoop.properties
sed
-i
${
txt
}
"s#yarn.application.status.address.*#yarn.application.status.address=http://
${
singleYarnIp
}
:8088/ws/v1/cluster/apps/%s#g"
conf/common/hadoop/hadoop.properties
...
...
@@ -283,7 +289,7 @@ sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downl
sed
-i
${
txt
}
"s#process.exec.basepath.*#process.exec.basepath=
${
execPath
}
#g"
conf/common/common.properties
sed
-i
${
txt
}
"s#hdfs.root.user.*#hdfs.root.user=
${
hdfsRootUser
}
#g"
conf/common/common.properties
sed
-i
${
txt
}
"s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=
${
hdfsPath
}
#g"
conf/common/common.properties
sed
-i
${
txt
}
"s#
hdfs.startup.state.*#hdfs.startup.state=
${
hdfsStartupSat
e
}
#g"
conf/common/common.properties
sed
-i
${
txt
}
"s#
res.upload.startup.type.*#res.upload.startup.type=
${
resUploadStartupTyp
e
}
#g"
conf/common/common.properties
sed
-i
${
txt
}
"s#escheduler.env.path.*#escheduler.env.path=
${
shellEnvPath
}
#g"
conf/common/common.properties
sed
-i
${
txt
}
"s#resource.view.suffixs.*#resource.view.suffixs=
${
resSuffixs
}
#g"
conf/common/common.properties
sed
-i
${
txt
}
"s#development.state.*#development.state=
${
devState
}
#g"
conf/common/common.properties
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录