Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
pentaLiker
DolphinScheduler
提交
b3e87041
DolphinScheduler
项目概览
pentaLiker
/
DolphinScheduler
与 Fork 源项目一致
Fork自
apache / DolphinScheduler
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
DolphinScheduler
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
b3e87041
编写于
12月 01, 2021
作者:
K
Kirs
提交者:
GitHub
12月 01, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[2.0.0-chery-pick #7003]Fix shell tools for database schema cannot run (#7087)
PR #7003 ISSUE #6998
上级
40e00585
变更
11
显示空白变更内容
内联
并排
Showing
11 changed file
with
223 addition
and
190 deletion
+223
-190
docker/build/hooks/build
docker/build/hooks/build
+9
-33
docker/build/startup-init-conf.sh
docker/build/startup-init-conf.sh
+2
-0
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
...phinscheduler/dao/datasource/SpringConnectionFactory.java
+8
-147
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java
.../org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java
+118
-0
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
...a/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
+4
-1
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
...est/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
+1
-0
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java
.../apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java
+60
-0
script/create-dolphinscheduler.sh
script/create-dolphinscheduler.sh
+5
-1
script/dolphinscheduler-daemon.sh
script/dolphinscheduler-daemon.sh
+9
-7
script/scp-hosts.sh
script/scp-hosts.sh
+2
-0
script/upgrade-dolphinscheduler.sh
script/upgrade-dolphinscheduler.sh
+5
-1
未找到文件。
docker/build/hooks/build
浏览文件 @
b3e87041
...
...
@@ -18,41 +18,17 @@
set
-
e
echo
"------ dolphinscheduler start - build -------"
printenv
ROOT_DIR
=
$
(
dirname
"$0"
)
/
..
/
..
/
..
MVN
=
"$ROOT_DIR"
/
mvnw
VERSION
=
$
(
"$MVN"
-
q
-
DforceStdout
-
N
org
.
apache
.
maven
.
plugins
:
maven
-
help
-
plugin
:
3.2
.
0
:
evaluate
-
Dexpression
=
project
.
version
)
if
[
-
z
"${VERSION}"
]
then
echo
"set default environment variable [VERSION]"
export
VERSION
=
$
(
cat
$
(
pwd
)
/
pom
.
xml
|
grep
'<version>'
-
m
1
|
awk
'{print $1}'
|
sed
's/<version>//'
|
sed
's/<\/version>//'
)
fi
DOCKER_REPO
=
$
{
DOCKER_REPO
:
-
"apache/dolphinscheduler"
}
TAG
=
$
{
TAG
:
-
"$VERSION"
}
if
[
"${DOCKER_REPO}x"
=
"x"
]
then
echo
"set default environment variable [DOCKER_REPO]"
export
DOCKER_REPO
=
'apache/dolphinscheduler'
fi
echo
"Building Docker image as: $DOCKER_REPO:$TAG"
echo
"Version: $VERSION"
echo
"Repo: $DOCKER_REPO"
"$MVN"
-
B
clean
package
-
Prelease
-
Dmaven
.
test
.
skip
=
true
-
Dhttp
.
keepAlive
=
false
-
Dmaven
.
wagon
.
http
.
pool
=
false
-
Dmaven
.
wagon
.
httpconnectionManager
.
ttlSeconds
=
120
echo
-
e
"Current Directory is $(pwd)
\n
"
cp
"$ROOT_DIR"
/
dolphinscheduler
-
dist
/
target
/
apache
-
dolphinscheduler
-
$
VERSION
-
bin
.
tar
.
gz
"$ROOT_DIR"
/
docker
/
build
/
# maven package(Project Directory)
echo
-
e
"./mvnw -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120"
.
/
mvnw
-
B
clean
package
-
Prelease
-
Dmaven
.
test
.
skip
=
true
-
Dhttp
.
keepAlive
=
false
-
Dmaven
.
wagon
.
http
.
pool
=
false
-
Dmaven
.
wagon
.
httpconnectionManager
.
ttlSeconds
=
120
# mv dolphinscheduler-bin.tar.gz file to docker/build directory
echo
-
e
"mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/
\n
"
mv
$
(
pwd
)
/
dolphinscheduler
-
dist
/
target
/
apache
-
dolphinscheduler
-
$
{
VERSION
}
-
bin
.
tar
.
gz
$
(
pwd
)
/
docker
/
build
/
# docker build
BUILD_COMMAND
=
"docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/"
echo
-
e
"$BUILD_COMMAND
\n
"
if
(
docker
info
2
>
/
dev
/
null
|
grep
-
i
"ERROR"
);
then
sudo
$
BUILD_COMMAND
else
$
BUILD_COMMAND
fi
echo
"------ dolphinscheduler end - build -------"
docker
build
--
build
-
arg
VERSION
=
$
VERSION
-
t
$
DOCKER_REPO
:
$
TAG
"$ROOT_DIR"
/
docker
/
build
/
docker/build/startup-init-conf.sh
浏览文件 @
b3e87041
...
...
@@ -37,6 +37,8 @@ echo "init env variables"
[
"h2"
==
${
DATABASE_TYPE
}
]
||
export
SPRING_DATASOURCE_URL
=
"jdbc:
${
DATABASE_TYPE
}
://
${
DATABASE_HOST
}
:
${
DATABASE_PORT
}
/
${
DATABASE_DATABASE
}
?
${
DATABASE_PARAMS
}
"
[
"h2"
==
${
DATABASE_TYPE
}
]
||
export
SPRING_DATASOURCE_USERNAME
=
${
DATABASE_USERNAME
}
[
"h2"
==
${
DATABASE_TYPE
}
]
||
export
SPRING_DATASOURCE_PASSWORD
=
${
DATABASE_PASSWORD
}
# export SPRING_DATASOURCE_DRIVER_CLASS_NAME=${DATABASE_DRIVER}
#============================================================================
# Registry
...
...
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
浏览文件 @
b3e87041
...
...
@@ -17,160 +17,38 @@
package
org.apache.dolphinscheduler.dao.datasource
;
import
static
org
.
apache
.
dolphinscheduler
.
common
.
Constants
.
DATASOURCE_PROPERTIES
;
import
org.apache.dolphinscheduler.common.Constants
;
import
org.apache.dolphinscheduler.common.enums.ProfileType
;
import
org.apache.dolphinscheduler.common.utils.PropertyUtils
;
import
org.apache.commons.lang.StringUtils
;
import
com.baomidou.mybatisplus.annotation.IdType
;
import
com.baomidou.mybatisplus.core.MybatisConfiguration
;
import
com.baomidou.mybatisplus.core.config.GlobalConfig
;
import
com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor
;
import
com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean
;
import
org.apache.ibatis.mapping.DatabaseIdProvider
;
import
org.apache.ibatis.mapping.VendorDatabaseIdProvider
;
import
org.apache.ibatis.session.SqlSession
;
import
org.apache.ibatis.session.SqlSessionFactory
;
import
org.apache.ibatis.type.JdbcType
;
import
java.sql.SQLException
;
import
java.util.Arrays
;
import
java.util.List
;
import
java.util.Properties
;
import
java.util.concurrent.atomic.AtomicBoolean
;
import
java.util.stream.Collectors
;
import
javax.annotation.PostConstruct
;
import
javax.sql.DataSource
;
import
org.h2.Driver
;
import
org.mybatis.spring.SqlSessionTemplate
;
import
org.mybatis.spring.annotation.MapperScan
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.context.annotation.Bean
;
import
org.springframework.context.annotation.Configuration
;
import
org.springframework.core.env.Environment
;
import
org.springframework.core.io.FileSystemResourceLoader
;
import
org.springframework.core.io.support.PathMatchingResourcePatternResolver
;
import
org.springframework.core.io.support.ResourcePatternResolver
;
import
org.springframework.jdbc.datasource.DataSourceTransactionManager
;
import
org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder
;
import
org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType
;
import
com.baomidou.mybatisplus.annotation.IdType
;
import
com.baomidou.mybatisplus.core.MybatisConfiguration
;
import
com.baomidou.mybatisplus.core.config.GlobalConfig
;
import
com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor
;
import
com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean
;
import
com.google.common.collect.Lists
;
import
com.zaxxer.hikari.HikariDataSource
;
import
javax.sql.DataSource
;
import
java.util.Properties
;
/**
* data source connection factory
*/
@Configuration
@MapperScan
(
"org.apache.dolphinscheduler.*.mapper"
)
public
class
SpringConnectionFactory
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
SpringConnectionFactory
.
class
);
@Autowired
private
Environment
environment
;
private
static
final
AtomicBoolean
H2_INITIALIZED
=
new
AtomicBoolean
(
false
);
@PostConstruct
public
void
init
()
{
String
datasourceProfile
=
getSpringActiveProfile
().
stream
()
.
filter
(
ProfileType
.
DATASOURCE_PROFILE
::
contains
)
.
findFirst
()
.
orElse
(
""
);
if
(
StringUtils
.
isEmpty
(
datasourceProfile
)
||
ProfileType
.
MYSQL
.
equals
(
datasourceProfile
))
{
// default load datasource.properties
PropertyUtils
.
loadPropertyFile
(
DATASOURCE_PROPERTIES
.
replace
(
"-%s"
,
""
));
}
else
{
// load datasource-{spring.profiles.active}.properties
PropertyUtils
.
loadPropertyFile
(
String
.
format
(
DATASOURCE_PROPERTIES
,
datasourceProfile
));
}
}
/**
* pagination interceptor
*
* @return pagination interceptor
*/
@Bean
public
PaginationInterceptor
paginationInterceptor
()
{
return
new
PaginationInterceptor
();
}
/**
* get the data source
*
* @return dataSource
*/
@Bean
(
destroyMethod
=
""
,
name
=
"datasource"
)
public
DataSource
dataSource
()
throws
SQLException
{
String
driverClassName
=
PropertyUtils
.
getString
(
Constants
.
SPRING_DATASOURCE_DRIVER_CLASS_NAME
);
if
(
Driver
.
class
.
getName
().
equals
(
driverClassName
))
{
initializeH2Datasource
();
}
HikariDataSource
dataSource
=
new
HikariDataSource
();
dataSource
.
setDriverClassName
(
driverClassName
);
dataSource
.
setJdbcUrl
(
PropertyUtils
.
getString
(
Constants
.
SPRING_DATASOURCE_URL
));
dataSource
.
setUsername
(
PropertyUtils
.
getString
(
Constants
.
SPRING_DATASOURCE_USERNAME
));
dataSource
.
setPassword
(
PropertyUtils
.
getString
(
Constants
.
SPRING_DATASOURCE_PASSWORD
));
dataSource
.
setConnectionTestQuery
(
PropertyUtils
.
getString
(
Constants
.
SPRING_DATASOURCE_VALIDATION_QUERY
,
"SELECT 1"
));
dataSource
.
setMinimumIdle
(
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_MIN_IDLE
,
5
));
dataSource
.
setMaximumPoolSize
(
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_MAX_ACTIVE
,
50
));
dataSource
.
setConnectionTimeout
(
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_CONNECTION_TIMEOUT
,
30000
));
dataSource
.
setIdleTimeout
(
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_IDLE_TIMEOUT
,
600000
));
dataSource
.
setMaxLifetime
(
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_MAX_LIFE_TIME
,
1800000
));
dataSource
.
setValidationTimeout
(
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_VALIDATION_TIMEOUT
,
5000
));
dataSource
.
setLeakDetectionThreshold
(
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_LEAK_DETECTION_THRESHOLD
,
0
));
dataSource
.
setInitializationFailTimeout
(
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_INITIALIZATION_FAIL_TIMEOUT
,
1
));
dataSource
.
setAutoCommit
(
PropertyUtils
.
getBoolean
(
Constants
.
SPRING_DATASOURCE_IS_AUTOCOMMIT
,
true
));
dataSource
.
addDataSourceProperty
(
Constants
.
CACHE_PREP_STMTS
,
PropertyUtils
.
getBoolean
(
Constants
.
SPRING_DATASOURCE_CACHE_PREP_STMTS
,
true
));
dataSource
.
addDataSourceProperty
(
Constants
.
PREP_STMT_CACHE_SIZE
,
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_PREP_STMT_CACHE_SIZE
,
250
));
dataSource
.
addDataSourceProperty
(
Constants
.
PREP_STMT_CACHE_SQL_LIMIT
,
PropertyUtils
.
getInt
(
Constants
.
SPRING_DATASOURCE_PREP_STMT_CACHE_SQL_LIMIT
,
2048
));
logger
.
info
(
"Initialize DataSource DataSource success"
);
return
dataSource
;
}
private
void
initializeH2Datasource
()
{
if
(
H2_INITIALIZED
.
compareAndSet
(
false
,
true
))
{
EmbeddedDatabaseBuilder
embeddedDatabaseBuilder
=
new
EmbeddedDatabaseBuilder
(
new
FileSystemResourceLoader
());
embeddedDatabaseBuilder
.
setType
(
EmbeddedDatabaseType
.
H2
)
.
setScriptEncoding
(
Constants
.
UTF_8
)
.
setName
(
"dolphinscheduler;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1"
)
.
addScript
(
PropertyUtils
.
getString
(
"spring.datasource.sql.schema"
,
"classpath:sql/dolphinscheduler_h2.sql"
))
.
build
();
logger
.
info
(
"Initialize H2 DataSource success"
);
}
}
/**
* * get transaction manager
*
* @return DataSourceTransactionManager
*/
@Bean
public
DataSourceTransactionManager
transactionManager
(
DataSource
dataSource
)
{
return
new
DataSourceTransactionManager
(
dataSource
);
}
/**
* * get sql session factory
*
* @return sqlSessionFactory
* @throws Exception sqlSessionFactory exception
*/
@Bean
public
SqlSessionFactory
sqlSessionFactory
(
DataSource
dataSource
)
throws
Exception
{
MybatisConfiguration
configuration
=
new
MybatisConfiguration
();
...
...
@@ -179,6 +57,7 @@ public class SpringConnectionFactory {
configuration
.
setCallSettersOnNulls
(
true
);
configuration
.
setJdbcTypeForNull
(
JdbcType
.
NULL
);
configuration
.
addInterceptor
(
paginationInterceptor
());
configuration
.
setGlobalConfig
(
new
GlobalConfig
().
setBanner
(
false
));
MybatisSqlSessionFactoryBean
sqlSessionFactoryBean
=
new
MybatisSqlSessionFactoryBean
();
sqlSessionFactoryBean
.
setConfiguration
(
configuration
);
sqlSessionFactoryBean
.
setDataSource
(
dataSource
);
...
...
@@ -196,11 +75,6 @@ public class SpringConnectionFactory {
return
sqlSessionFactoryBean
.
getObject
();
}
/**
* get sql session
*
* @return SqlSession
*/
@Bean
public
SqlSession
sqlSession
(
SqlSessionFactory
sqlSessionFactory
)
{
return
new
SqlSessionTemplate
(
sqlSessionFactory
);
...
...
@@ -216,17 +90,4 @@ public class SpringConnectionFactory {
databaseIdProvider
.
setProperties
(
properties
);
return
databaseIdProvider
;
}
/**
* Get spring active profile, which will be set by -Dspring.profiles.active=api, or in application.xml
*
* @return
*/
private
List
<
String
>
getSpringActiveProfile
()
{
if
(
environment
!=
null
)
{
return
Lists
.
newArrayList
(
environment
.
getActiveProfiles
());
}
String
property
=
System
.
getProperty
(
"spring.profiles.active"
,
""
);
return
Arrays
.
stream
(
property
.
split
(
","
)).
map
(
String:
:
trim
).
collect
(
Collectors
.
toList
());
}
}
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java
0 → 100644
浏览文件 @
b3e87041
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org.apache.dolphinscheduler.dao.upgrade
;
import
com.google.common.base.Strings
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.dolphinscheduler.common.utils.FileUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.core.io.ClassPathResource
;
import
java.io.File
;
import
java.io.FileNotFoundException
;
import
java.io.IOException
;
import
java.util.Arrays
;
import
java.util.Collections
;
import
java.util.List
;
import
java.util.stream.Collectors
;
/**
* Metadata related common classes
*/
public
class
SchemaUtils
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
SchemaUtils
.
class
);
private
SchemaUtils
()
{
throw
new
UnsupportedOperationException
(
"Construct SchemaUtils"
);
}
public
static
List
<
String
>
getAllSchemaList
()
throws
IOException
{
final
File
[]
schemaDirArr
=
new
ClassPathResource
(
"sql/upgrade"
).
getFile
().
listFiles
();
if
(
schemaDirArr
==
null
||
schemaDirArr
.
length
==
0
)
{
return
Collections
.
emptyList
();
}
return
Arrays
.
stream
(
schemaDirArr
).
map
(
File:
:
getName
).
sorted
((
o1
,
o2
)
->
{
try
{
String
version1
=
o1
.
split
(
"_"
)[
0
];
String
version2
=
o2
.
split
(
"_"
)[
0
];
if
(
version1
.
equals
(
version2
))
{
return
0
;
}
if
(
SchemaUtils
.
isAGreatVersion
(
version1
,
version2
))
{
return
1
;
}
return
-
1
;
}
catch
(
Exception
e
)
{
logger
.
error
(
e
.
getMessage
(),
e
);
throw
new
RuntimeException
(
e
);
}
}).
collect
(
Collectors
.
toList
());
}
/**
* Determine whether schemaVersion is higher than version
*
* @param schemaVersion schema version
* @param version version
* @return Determine whether schemaVersion is higher than version
*/
public
static
boolean
isAGreatVersion
(
String
schemaVersion
,
String
version
)
{
if
(
StringUtils
.
isEmpty
(
schemaVersion
)
||
StringUtils
.
isEmpty
(
version
))
{
throw
new
RuntimeException
(
"schemaVersion or version is empty"
);
}
String
[]
schemaVersionArr
=
schemaVersion
.
split
(
"\\."
);
String
[]
versionArr
=
version
.
split
(
"\\."
);
int
arrLength
=
Math
.
min
(
schemaVersionArr
.
length
,
versionArr
.
length
);
for
(
int
i
=
0
;
i
<
arrLength
;
i
++)
{
if
(
Integer
.
parseInt
(
schemaVersionArr
[
i
])
>
Integer
.
parseInt
(
versionArr
[
i
]))
{
return
true
;
}
else
if
(
Integer
.
parseInt
(
schemaVersionArr
[
i
])
<
Integer
.
parseInt
(
versionArr
[
i
]))
{
return
false
;
}
}
// If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number
return
schemaVersionArr
.
length
>
versionArr
.
length
;
}
/**
* Gets the current software version number of the system
*
* @return current software version
*/
public
static
String
getSoftVersion
()
throws
IOException
{
final
ClassPathResource
softVersionFile
=
new
ClassPathResource
(
"sql/soft_version"
);
String
softVersion
;
try
{
softVersion
=
FileUtils
.
readFile2Str
(
softVersionFile
.
getInputStream
());
softVersion
=
Strings
.
nullToEmpty
(
softVersion
).
replaceAll
(
"\\s+|\r|\n"
,
""
);
}
catch
(
FileNotFoundException
e
)
{
logger
.
error
(
e
.
getMessage
(),
e
);
throw
new
RuntimeException
(
"Failed to get the product version description file. The file could not be found"
,
e
);
}
return
softVersion
;
}
}
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
浏览文件 @
b3e87041
...
...
@@ -28,7 +28,10 @@ import org.apache.dolphinscheduler.common.Constants;
import
org.apache.dolphinscheduler.common.enums.*
;
import
org.apache.dolphinscheduler.common.process.ResourceInfo
;
import
org.apache.dolphinscheduler.common.task.TaskTimeoutParameter
;
import
org.apache.dolphinscheduler.common.utils.*
;
import
org.apache.dolphinscheduler.common.utils.CodeGenerateUtils
;
import
org.apache.dolphinscheduler.common.utils.ConnectionUtils
;
import
org.apache.dolphinscheduler.common.utils.JSONUtils
;
import
org.apache.dolphinscheduler.common.utils.ScriptRunner
;
import
org.apache.dolphinscheduler.dao.entity.ProcessDefinition
;
import
org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog
;
import
org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog
;
...
...
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
浏览文件 @
b3e87041
...
...
@@ -18,6 +18,7 @@
package
org.apache.dolphinscheduler.dao
;
import
org.apache.dolphinscheduler.common.enums.ProfileType
;
import
org.junit.runner.RunWith
;
import
org.springframework.boot.test.context.SpringBootTest
;
import
org.springframework.test.annotation.Rollback
;
...
...
dolphinscheduler-
common/src/test/java/org/apache/dolphinscheduler/common/utils
/SchemaUtilsTest.java
→
dolphinscheduler-
dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade
/SchemaUtilsTest.java
浏览文件 @
b3e87041
...
...
@@ -14,50 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org.apache.dolphinscheduler.common.utils
;
package
org.apache.dolphinscheduler.dao.upgrade
;
import
org.junit.Assert
;
import
org.junit.Test
;
import
org.junit.runner.RunWith
;
import
org.powermock.api.mockito.PowerMockito
;
import
org.powermock.core.classloader.annotations.PrepareForTest
;
import
org.powermock.modules.junit4.PowerMockRunner
;
import
org.slf4j.LoggerFactory
;
import
java.io.File
;
import
java.util.Arrays
;
import
java.util.List
;
@RunWith
(
PowerMockRunner
.
class
)
@PrepareForTest
({
LoggerFactory
.
class
,
FileUtils
.
class
})
public
class
SchemaUtilsTest
{
@Test
public
void
testReplaceBlank
()
{
Assert
.
assertEquals
(
"abc"
,
SchemaUtils
.
replaceBlank
(
" abc"
));
Assert
.
assertEquals
(
"abc"
,
SchemaUtils
.
replaceBlank
(
"abc "
));
Assert
.
assertEquals
(
"abc"
,
SchemaUtils
.
replaceBlank
(
"a b c"
));
Assert
.
assertEquals
(
"abc"
,
SchemaUtils
.
replaceBlank
(
"a b c"
));
Assert
.
assertEquals
(
""
,
SchemaUtils
.
replaceBlank
(
" "
));
Assert
.
assertEquals
(
""
,
SchemaUtils
.
replaceBlank
(
null
));
Assert
.
assertEquals
(
"我怕的你"
,
SchemaUtils
.
replaceBlank
(
"我怕的 你"
));
}
@Test
public
void
testGetSoftVersion
()
{
// file not found
try
{
SchemaUtils
.
getSoftVersion
();
}
catch
(
RuntimeException
e
)
{
Assert
.
assertEquals
(
"Failed to get the product version description file. The file could not be found"
,
e
.
getMessage
());
}
// file exists, fmt is invalid
FileUtils
.
writeContent2File
(
"32432423"
,
"sql/soft_version"
);
Assert
.
assertEquals
(
"32432423"
,
SchemaUtils
.
getSoftVersion
());
}
@Test
public
void
testIsAGreatVersion
()
{
// param is null
...
...
@@ -82,36 +46,15 @@ public class SchemaUtilsTest {
Assert
.
assertTrue
(
SchemaUtils
.
isAGreatVersion
(
"10.1.1"
,
"1.01.100"
));
try
{
SchemaUtils
.
isAGreatVersion
(
"10.1.1"
,
".1"
);
}
catch
(
Exception
e
)
{
Assert
.
assertNotNull
(
e
);
Assert
.
fail
(
"Should fail"
);
}
catch
(
Exception
ignored
)
{
// This is expected
}
try
{
SchemaUtils
.
isAGreatVersion
(
"a.1.1"
,
"b.1"
);
}
catch
(
Exception
e
)
{
Assert
.
assertNotNull
(
e
);
Assert
.
fail
(
"Should fail"
);
}
catch
(
Exception
ignored
)
{
// This is expected
}
}
@Test
public
void
testGetAllSchemaList
()
{
//normal
PowerMockito
.
mockStatic
(
FileUtils
.
class
);
File
[]
files
=
new
File
[
4
];
files
[
0
]
=
new
File
(
"sql/upgrade/1.2.0_schema"
);
files
[
1
]
=
new
File
(
"sql/upgrade/1.0.1_schema"
);
files
[
2
]
=
new
File
(
"sql/upgrade/1.0.2_schema"
);
files
[
3
]
=
new
File
(
"sql/upgrade/1.1.0_schema"
);
PowerMockito
.
when
(
FileUtils
.
getAllDir
(
"sql/upgrade"
)).
thenReturn
(
files
);
List
<
String
>
real
=
SchemaUtils
.
getAllSchemaList
();
List
<
String
>
expect
=
Arrays
.
asList
(
"1.0.1_schema"
,
"1.0.2_schema"
,
"1.1.0_schema"
,
"1.2.0_schema"
);
boolean
result
=
org
.
apache
.
commons
.
collections
.
CollectionUtils
.
isEqualCollection
(
real
,
expect
);
Assert
.
assertTrue
(
result
);
//normal
files
=
new
File
[
0
];
PowerMockito
.
when
(
FileUtils
.
getAllDir
(
"sql/upgrade"
)).
thenReturn
(
files
);
real
=
SchemaUtils
.
getAllSchemaList
();
Assert
.
assertNull
(
real
);
}
}
script/create-dolphinscheduler.sh
浏览文件 @
b3e87041
...
...
@@ -22,16 +22,20 @@ DOLPHINSCHEDULER_HOME=$BIN_DIR/..
export
JAVA_HOME
=
$JAVA_HOME
export
DATABASE_TYPE
=
${
DATABASE_TYPE
:-
"h2"
}
export
SPRING_PROFILES_ACTIVE
=
${
SPRING_PROFILES_ACTIVE
:-
"default"
}
export
SPRING_PROFILES_ACTIVE
=
"
${
SPRING_PROFILES_ACTIVE
}
,
${
DATABASE_TYPE
}
"
export
DOLPHINSCHEDULER_CONF_DIR
=
$DOLPHINSCHEDULER_HOME
/conf
export
DOLPHINSCHEDULER_LIB_JARS
=
$DOLPHINSCHEDULER_HOME
/lib/
*
export
DOLPHINSCHEDULER_SQL_DIR
=
$DOLPHINSCHEDULER_HOME
/sql
export
DOLPHINSCHEDULER_OPTS
=
"-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export
STOP_TIMEOUT
=
5
CLASS
=
org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler
exec_command
=
"
$DOLPHINSCHEDULER_OPTS
-classpath
$DOLPHINSCHEDULER_CONF_DIR
:
$DOLPHINSCHEDULER_LIB_JARS
$CLASS
"
exec_command
=
"
$DOLPHINSCHEDULER_OPTS
-classpath
$DOLPHINSCHEDULER_
SQL_DIR
:
$DOLPHINSCHEDULER_
CONF_DIR
:
$DOLPHINSCHEDULER_LIB_JARS
$CLASS
"
cd
$DOLPHINSCHEDULER_HOME
$JAVA_HOME
/bin/java
$exec_command
script/dolphinscheduler-daemon.sh
浏览文件 @
b3e87041
...
...
@@ -38,8 +38,6 @@ DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
source
/etc/profile
source
"
${
DOLPHINSCHEDULER_HOME
}
/conf/env/dolphinscheduler_env.sh"
export
JAVA_HOME
=
$JAVA_HOME
#export JAVA_HOME=/opt/soft/jdk
export
HOSTNAME
=
`
hostname
`
export
DOLPHINSCHEDULER_PID_DIR
=
$DOLPHINSCHEDULER_HOME
/pid
...
...
@@ -61,36 +59,40 @@ cd $DOLPHINSCHEDULER_HOME
export
DOLPHINSCHEDULER_OPTS
=
"-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:
$DOLPHINSCHEDULER_LOG_DIR
/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm
$DOLPHINSCHEDULER_OPTS
"
export
DATABASE_TYPE
=
${
DATABASE_TYPE
:-
"h2"
}
export
SPRING_PROFILES_ACTIVE
=
${
SPRING_PROFILES_ACTIVE
:-
"default"
}
if
[
"
$command
"
=
"api-server"
]
;
then
LOG_FILE
=
"-Dlogging.config=classpath:logback-api.xml
-Dspring.profiles.active=api
"
LOG_FILE
=
"-Dlogging.config=classpath:logback-api.xml"
CLASS
=
org.apache.dolphinscheduler.api.ApiApplicationServer
HEAP_OPTS
=
"-Xms1g -Xmx1g -Xmn512m"
export
DOLPHINSCHEDULER_OPTS
=
"
$HEAP_OPTS
$DOLPHINSCHEDULER_OPTS
$API_SERVER_OPTS
"
export
SPRING_PROFILES_ACTIVE
=
"
${
SPRING_PROFILES_ACTIVE
}
,api,
${
DATABASE_TYPE
}
"
elif
[
"
$command
"
=
"master-server"
]
;
then
LOG_FILE
=
"-Dlogging.config=classpath:logback-master.xml"
CLASS
=
org.apache.dolphinscheduler.server.master.MasterServer
HEAP_OPTS
=
"-Xms4g -Xmx4g -Xmn2g"
export
DOLPHINSCHEDULER_OPTS
=
"
$HEAP_OPTS
$DOLPHINSCHEDULER_OPTS
$MASTER_SERVER_OPTS
"
export
SPRING_PROFILES_ACTIVE
=
"
${
SPRING_PROFILES_ACTIVE
}
,master,
${
DATABASE_TYPE
}
"
elif
[
"
$command
"
=
"worker-server"
]
;
then
LOG_FILE
=
"-Dlogging.config=classpath:logback-worker.xml"
CLASS
=
org.apache.dolphinscheduler.server.worker.WorkerServer
HEAP_OPTS
=
"-Xms2g -Xmx2g -Xmn1g"
export
DOLPHINSCHEDULER_OPTS
=
"
$HEAP_OPTS
$DOLPHINSCHEDULER_OPTS
$WORKER_SERVER_OPTS
"
export
SPRING_PROFILES_ACTIVE
=
"
${
SPRING_PROFILES_ACTIVE
}
,worker,
${
DATABASE_TYPE
}
"
elif
[
"
$command
"
=
"alert-server"
]
;
then
LOG_FILE
=
"-Dlogback.configurationFile=conf/logback-alert.xml"
CLASS
=
org.apache.dolphinscheduler.alert.AlertServer
HEAP_OPTS
=
"-Xms1g -Xmx1g -Xmn512m"
export
DOLPHINSCHEDULER_OPTS
=
"
$HEAP_OPTS
$DOLPHINSCHEDULER_OPTS
$ALERT_SERVER_OPTS
"
export
SPRING_PROFILES_ACTIVE
=
"
${
SPRING_PROFILES_ACTIVE
}
,alert,
${
DATABASE_TYPE
}
"
elif
[
"
$command
"
=
"logger-server"
]
;
then
CLASS
=
org.apache.dolphinscheduler.server.log.LoggerServer
HEAP_OPTS
=
"-Xms1g -Xmx1g -Xmn512m"
export
DOLPHINSCHEDULER_OPTS
=
"
$HEAP_OPTS
$DOLPHINSCHEDULER_OPTS
$LOGGER_SERVER_OPTS
"
elif
[
"
$command
"
=
"zookeeper-server"
]
;
then
#note: this command just for getting a quick experience,not recommended for production. this operation will start a standalone zookeeper server
LOG_FILE
=
"-Dlogback.configurationFile=classpath:logback-zookeeper.xml"
CLASS
=
org.apache.dolphinscheduler.service.zk.ZKServer
elif
[
"
$command
"
=
"standalone-server"
]
;
then
CLASS
=
org.apache.dolphinscheduler.server.StandaloneServer
export
SPRING_PROFILES_ACTIVE
=
"
${
SPRING_PROFILES_ACTIVE
}
,standalone,
${
DATABASE_TYPE
}
"
else
echo
"Error: No command named '
$command
' was found."
exit
1
...
...
script/scp-hosts.sh
浏览文件 @
b3e87041
...
...
@@ -29,6 +29,7 @@ fi
declare
-A
workersGroupMap
=()
workersGroup
=(
${
workers
//,/
}
)
# shellcheck disable=SC2068
for
workerGroup
in
${
workersGroup
[@]
}
do
echo
$workerGroup
;
...
...
@@ -39,6 +40,7 @@ done
hostsArr
=(
${
ips
//,/
}
)
# shellcheck disable=SC2068
for
host
in
${
hostsArr
[@]
}
do
...
...
script/upgrade-dolphinscheduler.sh
浏览文件 @
b3e87041
...
...
@@ -22,16 +22,20 @@ DOLPHINSCHEDULER_HOME=$BIN_DIR/..
export
JAVA_HOME
=
$JAVA_HOME
export
DATABASE_TYPE
=
${
DATABASE_TYPE
:-
"h2"
}
export
SPRING_PROFILES_ACTIVE
=
${
SPRING_PROFILES_ACTIVE
:-
"default"
}
export
SPRING_PROFILES_ACTIVE
=
"
${
SPRING_PROFILES_ACTIVE
}
,
${
DATABASE_TYPE
}
"
export
DOLPHINSCHEDULER_CONF_DIR
=
$DOLPHINSCHEDULER_HOME
/conf
export
DOLPHINSCHEDULER_LIB_JARS
=
$DOLPHINSCHEDULER_HOME
/lib/
*
export
DOLPHINSCHEDULER_SQL_DIR
=
$DOLPHINSCHEDULER_HOME
/sql
export
DOLPHINSCHEDULER_OPTS
=
"-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export
STOP_TIMEOUT
=
5
CLASS
=
org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler
exec_command
=
"
$DOLPHINSCHEDULER_OPTS
-classpath
$DOLPHINSCHEDULER_CONF_DIR
:
$DOLPHINSCHEDULER_LIB_JARS
$CLASS
"
exec_command
=
"
$DOLPHINSCHEDULER_OPTS
-classpath
$DOLPHINSCHEDULER_
SQL_DIR
:
$DOLPHINSCHEDULER_
CONF_DIR
:
$DOLPHINSCHEDULER_LIB_JARS
$CLASS
"
cd
$DOLPHINSCHEDULER_HOME
$JAVA_HOME
/bin/java
$exec_command
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录