From 98ab5219ff74933f5ec6d51dab84e96aced59a51 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Mon, 6 Dec 2021 17:10:24 +0800 Subject: [PATCH] Fix SQL create / upgrade scripts cannot run (#7212) --- .../common/utils/SchemaUtils.java | 145 ------------------ .../dao/upgrade/DolphinSchedulerManager.java | 1 - .../dao/upgrade/UpgradeDao.java | 4 + .../postgresql/dolphinscheduler_ddl.sql | 3 +- .../mysql/dolphinscheduler_ddl.sql | 0 .../postgre/dolphinscheduler_ddl.sql | 2 +- script/create-dolphinscheduler.sh | 5 + script/upgrade-dolphinscheduler.sh | 5 + 8 files changed, 17 insertions(+), 148 deletions(-) delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java rename dolphinscheduler-dao/src/main/resources/sql/upgrade/{2.0.1-schema => 2.0.1_schema}/mysql/dolphinscheduler_ddl.sql (100%) rename dolphinscheduler-dao/src/main/resources/sql/upgrade/{2.0.1-schema => 2.0.1_schema}/postgre/dolphinscheduler_ddl.sql (99%) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java deleted file mode 100644 index 503cf8217..000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils; - -import org.apache.commons.lang.StringUtils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Metadata related common classes - */ -public class SchemaUtils { - - private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); - private static final Pattern p = Pattern.compile("\\s*|\t|\r|\n"); - - private SchemaUtils() { - throw new UnsupportedOperationException("Construct SchemaUtils"); - } - - /** - * Gets upgradable schemas for all upgrade directories - * - * @return all schema list - */ - public static List getAllSchemaList() { - List schemaDirList = new ArrayList<>(); - File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade"); - if (schemaDirArr == null || schemaDirArr.length == 0) { - return null; - } - - for (File file : schemaDirArr) { - schemaDirList.add(file.getName()); - } - - schemaDirList.sort((o1, o2) -> { - try { - String dir1 = String.valueOf(o1); - String dir2 = String.valueOf(o2); - String version1 = dir1.split("_")[0]; - String version2 = dir2.split("_")[0]; - if (version1.equals(version2)) { - return 0; - } - - if (SchemaUtils.isAGreatVersion(version1, version2)) { - return 1; - } - - return -1; - - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException(e); - } - }); - - return schemaDirList; - } - - /** - * Determine whether schemaVersion is higher than version - * - * @param schemaVersion schema version - * @param version version - * @return Determine whether schemaVersion is higher than version - */ - public static boolean isAGreatVersion(String schemaVersion, String version) { - if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { - throw new RuntimeException("schemaVersion or version is empty"); - } - - String[] schemaVersionArr = schemaVersion.split("\\."); - String[] versionArr = version.split("\\."); - int arrLength = Math.min(schemaVersionArr.length, versionArr.length); - for (int i = 0; i < arrLength; i++) { - if (Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) { - return true; - } else if (Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) { - return false; - } - } - - // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number - return schemaVersionArr.length > versionArr.length; - } - - /** - * Gets the current software version number of the system - * - * @return current software version - */ - public static String getSoftVersion() { - String softVersion; - try { - softVersion = FileUtils.readFile2Str(new FileInputStream("sql/soft_version")); - softVersion = replaceBlank(softVersion); - } catch (FileNotFoundException e) { - logger.error(e.getMessage(), e); - throw new RuntimeException("Failed to get the product version description file. The file could not be found", e); - } - return softVersion; - } - - /** - * Strips the string of space carriage returns and tabs - * - * @param str string - * @return string removed blank - */ - public static String replaceBlank(String str) { - String dest = ""; - if (str != null) { - - Matcher m = p.matcher(str); - dest = m.replaceAll(""); - } - return dest; - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java index 9de49a4c5..45186ba89 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.upgrade; -import org.apache.dolphinscheduler.common.utils.SchemaUtils; import org.apache.dolphinscheduler.spi.enums.DbType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 161f8b764..a0ebcd745 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -288,6 +288,10 @@ public abstract class UpgradeDao { private void upgradeDolphinSchedulerDML(String schemaDir) { String schemaVersion = schemaDir.split("_")[0]; Resource sqlFilePath = new ClassPathResource(String.format("sql/upgrade/%s/%s/dolphinscheduler_dml.sql", schemaDir, getDbType().name().toLowerCase())); + if (!sqlFilePath.exists()) { + logger.info("No dml file {}, returning", sqlFilePath); + return; + } logger.info("sqlSQLFilePath" + sqlFilePath); Connection conn = null; PreparedStatement pstmt = null; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql index fc6968978..1cd66c722 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +delimiter d// CREATE OR REPLACE FUNCTION public.dolphin_update_metadata( ) RETURNS character varying @@ -304,3 +304,4 @@ BEGIN END; $BODY$; +d// diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1-schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/mysql/dolphinscheduler_ddl.sql similarity index 100% rename from dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1-schema/mysql/dolphinscheduler_ddl.sql rename to dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/mysql/dolphinscheduler_ddl.sql diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1-schema/postgre/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/postgre/dolphinscheduler_ddl.sql similarity index 99% rename from dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1-schema/postgre/dolphinscheduler_ddl.sql rename to dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/postgre/dolphinscheduler_ddl.sql index 97acb028d..af062e2dc 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1-schema/postgre/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.1_schema/postgre/dolphinscheduler_ddl.sql @@ -37,4 +37,4 @@ BEGIN return SQLERRM; END; $BODY$; -d// \ No newline at end of file +d// diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh index 8591277ad..fed6dd0bb 100755 --- a/script/create-dolphinscheduler.sh +++ b/script/create-dolphinscheduler.sh @@ -20,6 +20,11 @@ BIN_DIR=`dirname $0` BIN_DIR=`cd "$BIN_DIR"; pwd` DOLPHINSCHEDULER_HOME=$BIN_DIR/.. +set -a +source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh" +source "${DOLPHINSCHEDULER_HOME}/conf/config/install_config.conf" +set +a + export JAVA_HOME=$JAVA_HOME export DATABASE_TYPE=${DATABASE_TYPE:-"h2"} diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh index a8ef4e730..9fe659d8a 100755 --- a/script/upgrade-dolphinscheduler.sh +++ b/script/upgrade-dolphinscheduler.sh @@ -20,6 +20,11 @@ BIN_DIR=`dirname $0` BIN_DIR=`cd "$BIN_DIR"; pwd` DOLPHINSCHEDULER_HOME=$BIN_DIR/.. +set -a +source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh" +source "${DOLPHINSCHEDULER_HOME}/conf/config/install_config.conf" +set +a + export JAVA_HOME=$JAVA_HOME export DATABASE_TYPE=${DATABASE_TYPE:-"h2"} -- GitLab