diff --git a/docker/build/hooks/build b/docker/build/hooks/build index 70ea260dea28049ae4f249fdb5996d80f8a0068d..1e6a96549b4757d7edc2a44da43397da51dd5eb6 100755 --- a/docker/build/hooks/build +++ b/docker/build/hooks/build @@ -18,41 +18,17 @@ set -e -echo "------ dolphinscheduler start - build -------" -printenv +ROOT_DIR=$(dirname "$0")/../../.. +MVN="$ROOT_DIR"/mvnw +VERSION=$("$MVN" -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version) -if [ -z "${VERSION}" ] -then - echo "set default environment variable [VERSION]" - export VERSION=$(cat $(pwd)/pom.xml | grep '' -m 1 | awk '{print $1}' | sed 's///' | sed 's/<\/version>//') -fi +DOCKER_REPO=${DOCKER_REPO:-"apache/dolphinscheduler"} +TAG=${TAG:-"$VERSION"} -if [ "${DOCKER_REPO}x" = "x" ] -then - echo "set default environment variable [DOCKER_REPO]" - export DOCKER_REPO='apache/dolphinscheduler' -fi +echo "Building Docker image as: $DOCKER_REPO:$TAG" -echo "Version: $VERSION" -echo "Repo: $DOCKER_REPO" +"$MVN" -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -echo -e "Current Directory is $(pwd)\n" +cp "$ROOT_DIR"/dolphinscheduler-dist/target/apache-dolphinscheduler-$VERSION-bin.tar.gz "$ROOT_DIR"/docker/build/ -# maven package(Project Directory) -echo -e "./mvnw -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120" -./mvnw -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - -# mv dolphinscheduler-bin.tar.gz file to docker/build directory -echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/\n" -mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/ - -# docker build -BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/" -echo -e "$BUILD_COMMAND\n" -if (docker info 2> /dev/null | grep -i "ERROR"); then - sudo $BUILD_COMMAND -else - $BUILD_COMMAND -fi - -echo "------ dolphinscheduler end - build -------" +docker build --build-arg VERSION=$VERSION -t $DOCKER_REPO:$TAG "$ROOT_DIR"/docker/build/ diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 32b71bfdb4874b0d38c571591159fed3cbf48576..ad61829074a0b94a2f5b2d64742b50c2690ba9d6 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -37,6 +37,8 @@ echo "init env variables" [ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_URL="jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}" [ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_USERNAME=${DATABASE_USERNAME} [ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_PASSWORD=${DATABASE_PASSWORD} +# export SPRING_DATASOURCE_DRIVER_CLASS_NAME=${DATABASE_DRIVER} + #============================================================================ # Registry diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java deleted file mode 100644 index b385523906e9a60d292a150a1bbf262e695488f6..0000000000000000000000000000000000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.utils; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.Arrays; -import java.util.List; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({ LoggerFactory.class, FileUtils.class }) -public class SchemaUtilsTest { - - @Test - public void testReplaceBlank() { - Assert.assertEquals("abc", SchemaUtils.replaceBlank(" abc")); - Assert.assertEquals("abc", SchemaUtils.replaceBlank("abc ")); - Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c")); - Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c")); - Assert.assertEquals("", SchemaUtils.replaceBlank(" ")); - Assert.assertEquals("", SchemaUtils.replaceBlank(null)); - Assert.assertEquals("我怕的你", SchemaUtils.replaceBlank("我怕的 你")); - } - - @Test - public void testGetSoftVersion() { - // file not found - try { - SchemaUtils.getSoftVersion(); - } catch (RuntimeException e) { - Assert.assertEquals("Failed to get the product version description file. The file could not be found", - e.getMessage()); - } - - // file exists, fmt is invalid - FileUtils.writeContent2File("32432423", "sql/soft_version"); - Assert.assertEquals("32432423", SchemaUtils.getSoftVersion()); - } - - @Test - public void testIsAGreatVersion() { - // param is null - try { - SchemaUtils.isAGreatVersion(null, null); - } catch (RuntimeException e) { - Assert.assertEquals("schemaVersion or version is empty", e.getMessage()); - } - - // param is "" - try { - SchemaUtils.isAGreatVersion("", ""); - } catch (RuntimeException e) { - Assert.assertEquals("schemaVersion or version is empty", e.getMessage()); - } - Assert.assertFalse(SchemaUtils.isAGreatVersion("1", "1")); - Assert.assertTrue(SchemaUtils.isAGreatVersion("2", "1")); - Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1")); - Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1.0.1")); - Assert.assertFalse(SchemaUtils.isAGreatVersion("1.1", "1.2")); - Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1.1", "1.1")); - Assert.assertTrue(SchemaUtils.isAGreatVersion("10.1.1", "1.01.100")); - try { - SchemaUtils.isAGreatVersion("10.1.1", ".1"); - } catch (Exception e) { - Assert.assertNotNull(e); - } - try { - SchemaUtils.isAGreatVersion("a.1.1", "b.1"); - } catch (Exception e) { - Assert.assertNotNull(e); - } - } - - @Test - public void testGetAllSchemaList() { - //normal - PowerMockito.mockStatic(FileUtils.class); - File[] files = new File[4]; - files[0] = new File("sql/upgrade/1.2.0_schema"); - files[1] = new File("sql/upgrade/1.0.1_schema"); - files[2] = new File("sql/upgrade/1.0.2_schema"); - files[3] = new File("sql/upgrade/1.1.0_schema"); - PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files); - List real = SchemaUtils.getAllSchemaList(); - List expect = Arrays.asList("1.0.1_schema", "1.0.2_schema", - "1.1.0_schema", "1.2.0_schema"); - boolean result = org.apache.commons.collections.CollectionUtils.isEqualCollection(real, expect); - Assert.assertTrue(result); - - //normal - files = new File[0]; - PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files); - real = SchemaUtils.getAllSchemaList(); - Assert.assertNull(real); - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java index 29cbefa97dc3d854a37ce4fa68c8bc9157d1fdca..e44ac85d59705c94244246c7540e9c5bf1a6fea1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java @@ -17,160 +17,38 @@ package org.apache.dolphinscheduler.dao.datasource; -import static org.apache.dolphinscheduler.common.Constants.DATASOURCE_PROPERTIES; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.ProfileType; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; - -import org.apache.commons.lang.StringUtils; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.config.GlobalConfig; +import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; +import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; import org.apache.ibatis.mapping.DatabaseIdProvider; import org.apache.ibatis.mapping.VendorDatabaseIdProvider; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.SqlSessionFactory; import org.apache.ibatis.type.JdbcType; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -import javax.annotation.PostConstruct; -import javax.sql.DataSource; - -import org.h2.Driver; import org.mybatis.spring.SqlSessionTemplate; -import org.mybatis.spring.annotation.MapperScan; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; -import org.springframework.core.io.FileSystemResourceLoader; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.ResourcePatternResolver; import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.core.MybatisConfiguration; -import com.baomidou.mybatisplus.core.config.GlobalConfig; -import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; -import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; -import com.google.common.collect.Lists; -import com.zaxxer.hikari.HikariDataSource; +import javax.sql.DataSource; +import java.util.Properties; -/** - * data source connection factory - */ @Configuration -@MapperScan("org.apache.dolphinscheduler.*.mapper") public class SpringConnectionFactory { - - private static final Logger logger = LoggerFactory.getLogger(SpringConnectionFactory.class); - - @Autowired - private Environment environment; - - private static final AtomicBoolean H2_INITIALIZED = new AtomicBoolean(false); - - @PostConstruct - public void init() { - String datasourceProfile = getSpringActiveProfile().stream() - .filter(ProfileType.DATASOURCE_PROFILE::contains) - .findFirst() - .orElse(""); - if (StringUtils.isEmpty(datasourceProfile) || ProfileType.MYSQL.equals(datasourceProfile)) { - // default load datasource.properties - PropertyUtils.loadPropertyFile(DATASOURCE_PROPERTIES.replace("-%s", "")); - } else { - // load datasource-{spring.profiles.active}.properties - PropertyUtils.loadPropertyFile(String.format(DATASOURCE_PROPERTIES, datasourceProfile)); - } - } - - /** - * pagination interceptor - * - * @return pagination interceptor - */ @Bean public PaginationInterceptor paginationInterceptor() { return new PaginationInterceptor(); } - /** - * get the data source - * - * @return dataSource - */ - @Bean(destroyMethod = "", name = "datasource") - public DataSource dataSource() throws SQLException { - String driverClassName = PropertyUtils.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME); - if (Driver.class.getName().equals(driverClassName)) { - initializeH2Datasource(); - } - - HikariDataSource dataSource = new HikariDataSource(); - - dataSource.setDriverClassName(driverClassName); - dataSource.setJdbcUrl(PropertyUtils.getString(Constants.SPRING_DATASOURCE_URL)); - dataSource.setUsername(PropertyUtils.getString(Constants.SPRING_DATASOURCE_USERNAME)); - dataSource.setPassword(PropertyUtils.getString(Constants.SPRING_DATASOURCE_PASSWORD)); - dataSource.setConnectionTestQuery(PropertyUtils.getString(Constants.SPRING_DATASOURCE_VALIDATION_QUERY, "SELECT 1")); - - dataSource.setMinimumIdle(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MIN_IDLE, 5)); - dataSource.setMaximumPoolSize(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50)); - dataSource.setConnectionTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_CONNECTION_TIMEOUT, 30000)); - dataSource.setIdleTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_IDLE_TIMEOUT, 600000)); - dataSource.setMaxLifetime(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_LIFE_TIME, 1800000)); - dataSource.setValidationTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_VALIDATION_TIMEOUT, 5000)); - dataSource.setLeakDetectionThreshold(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_LEAK_DETECTION_THRESHOLD, 0)); - dataSource.setInitializationFailTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_INITIALIZATION_FAIL_TIMEOUT, 1)); - - dataSource.setAutoCommit(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_IS_AUTOCOMMIT, true)); - - dataSource.addDataSourceProperty(Constants.CACHE_PREP_STMTS, PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_CACHE_PREP_STMTS, true)); - dataSource.addDataSourceProperty(Constants.PREP_STMT_CACHE_SIZE, PropertyUtils.getInt(Constants.SPRING_DATASOURCE_PREP_STMT_CACHE_SIZE, 250)); - dataSource.addDataSourceProperty(Constants.PREP_STMT_CACHE_SQL_LIMIT, PropertyUtils.getInt(Constants.SPRING_DATASOURCE_PREP_STMT_CACHE_SQL_LIMIT, 2048)); - - logger.info("Initialize DataSource DataSource success"); - return dataSource; - } - - private void initializeH2Datasource() { - if (H2_INITIALIZED.compareAndSet(false, true)) { - EmbeddedDatabaseBuilder embeddedDatabaseBuilder = new EmbeddedDatabaseBuilder(new FileSystemResourceLoader()); - embeddedDatabaseBuilder - .setType(EmbeddedDatabaseType.H2) - .setScriptEncoding(Constants.UTF_8) - .setName("dolphinscheduler;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1") - .addScript(PropertyUtils.getString("spring.datasource.sql.schema", "classpath:sql/dolphinscheduler_h2.sql")) - .build(); - logger.info("Initialize H2 DataSource success"); - } - } - - /** - * * get transaction manager - * - * @return DataSourceTransactionManager - */ @Bean public DataSourceTransactionManager transactionManager(DataSource dataSource) { return new DataSourceTransactionManager(dataSource); } - /** - * * get sql session factory - * - * @return sqlSessionFactory - * @throws Exception sqlSessionFactory exception - */ @Bean public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception { MybatisConfiguration configuration = new MybatisConfiguration(); @@ -179,6 +57,7 @@ public class SpringConnectionFactory { configuration.setCallSettersOnNulls(true); configuration.setJdbcTypeForNull(JdbcType.NULL); configuration.addInterceptor(paginationInterceptor()); + configuration.setGlobalConfig(new GlobalConfig().setBanner(false)); MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean(); sqlSessionFactoryBean.setConfiguration(configuration); sqlSessionFactoryBean.setDataSource(dataSource); @@ -196,11 +75,6 @@ public class SpringConnectionFactory { return sqlSessionFactoryBean.getObject(); } - /** - * get sql session - * - * @return SqlSession - */ @Bean public SqlSession sqlSession(SqlSessionFactory sqlSessionFactory) { return new SqlSessionTemplate(sqlSessionFactory); @@ -216,17 +90,4 @@ public class SpringConnectionFactory { databaseIdProvider.setProperties(properties); return databaseIdProvider; } - - /** - * Get spring active profile, which will be set by -Dspring.profiles.active=api, or in application.xml - * - * @return - */ - private List getSpringActiveProfile() { - if (environment != null) { - return Lists.newArrayList(environment.getActiveProfiles()); - } - String property = System.getProperty("spring.profiles.active", ""); - return Arrays.stream(property.split(",")).map(String::trim).collect(Collectors.toList()); - } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..83acb75c826058e0db7bdeda43d38ec64d5c04ea --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.upgrade; + +import com.google.common.base.Strings; +import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.ClassPathResource; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Metadata related common classes + */ +public class SchemaUtils { + private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); + + private SchemaUtils() { + throw new UnsupportedOperationException("Construct SchemaUtils"); + } + + public static List getAllSchemaList() throws IOException { + final File[] schemaDirArr = new ClassPathResource("sql/upgrade").getFile().listFiles(); + + if (schemaDirArr == null || schemaDirArr.length == 0) { + return Collections.emptyList(); + } + + return Arrays.stream(schemaDirArr).map(File::getName).sorted((o1, o2) -> { + try { + String version1 = o1.split("_")[0]; + String version2 = o2.split("_")[0]; + + if (version1.equals(version2)) { + return 0; + } + + if (SchemaUtils.isAGreatVersion(version1, version2)) { + return 1; + } + + return -1; + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); + } + }).collect(Collectors.toList()); + } + + /** + * Determine whether schemaVersion is higher than version + * + * @param schemaVersion schema version + * @param version version + * @return Determine whether schemaVersion is higher than version + */ + public static boolean isAGreatVersion(String schemaVersion, String version) { + if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { + throw new RuntimeException("schemaVersion or version is empty"); + } + + String[] schemaVersionArr = schemaVersion.split("\\."); + String[] versionArr = version.split("\\."); + int arrLength = Math.min(schemaVersionArr.length, versionArr.length); + for (int i = 0; i < arrLength; i++) { + if (Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) { + return true; + } else if (Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) { + return false; + } + } + + // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number + return schemaVersionArr.length > versionArr.length; + } + + /** + * Gets the current software version number of the system + * + * @return current software version + */ + public static String getSoftVersion() throws IOException { + final ClassPathResource softVersionFile = new ClassPathResource("sql/soft_version"); + String softVersion; + try { + softVersion = FileUtils.readFile2Str(softVersionFile.getInputStream()); + softVersion = Strings.nullToEmpty(softVersion).replaceAll("\\s+|\r|\n", ""); + } catch (FileNotFoundException e) { + logger.error(e.getMessage(), e); + throw new RuntimeException("Failed to get the product version description file. The file could not be found", e); + } + return softVersion; + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 2f9fb5d275b0d3a27c657eb695f327d5a83841f5..161f8b76497bb121484bd9eb8bbd54fea0946f7f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -28,7 +28,10 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; +import org.apache.dolphinscheduler.common.utils.ConnectionUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.ScriptRunner; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java index ddb87d5a4ec8d8e53fff42b4054f377cd51b0e0c..5838ad7874ff5d5a1c17f00788ebb9de78d14781 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.dao; import org.apache.dolphinscheduler.common.enums.ProfileType; + import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.Rollback; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java new file mode 100644 index 0000000000000000000000000000000000000000..b505709c2596349a93decabd935328abbe9c30f8 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.upgrade; + +import org.junit.Assert; +import org.junit.Test; + +public class SchemaUtilsTest { + + @Test + public void testIsAGreatVersion() { + // param is null + try { + SchemaUtils.isAGreatVersion(null, null); + } catch (RuntimeException e) { + Assert.assertEquals("schemaVersion or version is empty", e.getMessage()); + } + + // param is "" + try { + SchemaUtils.isAGreatVersion("", ""); + } catch (RuntimeException e) { + Assert.assertEquals("schemaVersion or version is empty", e.getMessage()); + } + Assert.assertFalse(SchemaUtils.isAGreatVersion("1", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("2", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1.0.1")); + Assert.assertFalse(SchemaUtils.isAGreatVersion("1.1", "1.2")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1.1", "1.1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("10.1.1", "1.01.100")); + try { + SchemaUtils.isAGreatVersion("10.1.1", ".1"); + Assert.fail("Should fail"); + } catch (Exception ignored) { + // This is expected + } + try { + SchemaUtils.isAGreatVersion("a.1.1", "b.1"); + Assert.fail("Should fail"); + } catch (Exception ignored) { + // This is expected + } + } +} diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh index 9ca9499431ed1b2077a258d7a47690f9f9379edc..8591277ad1b7b602f057422ecaf359abe5af37ee 100755 --- a/script/create-dolphinscheduler.sh +++ b/script/create-dolphinscheduler.sh @@ -22,16 +22,20 @@ DOLPHINSCHEDULER_HOME=$BIN_DIR/.. export JAVA_HOME=$JAVA_HOME +export DATABASE_TYPE=${DATABASE_TYPE:-"h2"} +export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"} +export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},${DATABASE_TYPE}" export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* +export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler -exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" +exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" cd $DOLPHINSCHEDULER_HOME $JAVA_HOME/bin/java $exec_command diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 41a7a1576cecc40ba8c4c4aba7f82100cafa56a7..64d6b8327c9186befbc628b9d3f029bc6bebeda0 100755 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -38,8 +38,6 @@ DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd` source /etc/profile source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh" -export JAVA_HOME=$JAVA_HOME -#export JAVA_HOME=/opt/soft/jdk export HOSTNAME=`hostname` export DOLPHINSCHEDULER_PID_DIR=$DOLPHINSCHEDULER_HOME/pid @@ -61,36 +59,40 @@ cd $DOLPHINSCHEDULER_HOME export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS" +export DATABASE_TYPE=${DATABASE_TYPE:-"h2"} +export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"} + if [ "$command" = "api-server" ]; then - LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api" + LOG_FILE="-Dlogging.config=classpath:logback-api.xml" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS" + export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},api,${DATABASE_TYPE}" elif [ "$command" = "master-server" ]; then LOG_FILE="-Dlogging.config=classpath:logback-master.xml" CLASS=org.apache.dolphinscheduler.server.master.MasterServer HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS" + export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},master,${DATABASE_TYPE}" elif [ "$command" = "worker-server" ]; then LOG_FILE="-Dlogging.config=classpath:logback-worker.xml" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS" + export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},worker,${DATABASE_TYPE}" elif [ "$command" = "alert-server" ]; then LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml" CLASS=org.apache.dolphinscheduler.alert.AlertServer HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS" + export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},alert,${DATABASE_TYPE}" elif [ "$command" = "logger-server" ]; then CLASS=org.apache.dolphinscheduler.server.log.LoggerServer HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS" -elif [ "$command" = "zookeeper-server" ]; then - #note: this command just for getting a quick experience,not recommended for production. this operation will start a standalone zookeeper server - LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml" - CLASS=org.apache.dolphinscheduler.service.zk.ZKServer elif [ "$command" = "standalone-server" ]; then CLASS=org.apache.dolphinscheduler.server.StandaloneServer + export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},standalone,${DATABASE_TYPE}" else echo "Error: No command named '$command' was found." exit 1 diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 8def4d1bae12c1c81002f1e1788434196e9839bc..da084468a60c2508dfd8a50f0915a2704e77c5b9 100755 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -29,6 +29,7 @@ fi declare -A workersGroupMap=() workersGroup=(${workers//,/ }) +# shellcheck disable=SC2068 for workerGroup in ${workersGroup[@]} do echo $workerGroup; @@ -39,6 +40,7 @@ done hostsArr=(${ips//,/ }) +# shellcheck disable=SC2068 for host in ${hostsArr[@]} do diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh index 42ba839d56a945e81e7e5fcb51a7e9fb13a34c65..a8ef4e7307bb7b7826fc9dcc7f8e490f287eb2ec 100755 --- a/script/upgrade-dolphinscheduler.sh +++ b/script/upgrade-dolphinscheduler.sh @@ -22,16 +22,20 @@ DOLPHINSCHEDULER_HOME=$BIN_DIR/.. export JAVA_HOME=$JAVA_HOME +export DATABASE_TYPE=${DATABASE_TYPE:-"h2"} +export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"} +export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},${DATABASE_TYPE}" export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* +export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler -exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" +exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" cd $DOLPHINSCHEDULER_HOME $JAVA_HOME/bin/java $exec_command