Revert "[Improvement][DAO] CreateDolphinScheduler (#5357) (#5358)"

This reverts commit b108ac43.

dolphinscheduler-dao/pom.xml

@@ -151,22 +151,5 @@
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-module-junit4</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-api-mockito2</artifactId>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.mockito</groupId>
-                    <artifactId>mockito-core</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
     </dependencies>
 </project>

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java

@@ -14,19 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.dolphinscheduler.dao.upgrade;
 
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.utils.SchemaUtils;
-
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.List;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.List;
 
 /**
  * upgrade manager
@@ -65,7 +60,7 @@ public class DolphinSchedulerManager {
     /**
      * init DolphinScheduler
      */
-    public void initDolphinScheduler() throws SQLException, IOException {
+    public void initDolphinScheduler() {
         // Determines whether the dolphinscheduler table structure has been init
         if (upgradeDao.isExistsTable("t_escheduler_version") ||
                 upgradeDao.isExistsTable("t_ds_version") ||
@@ -79,7 +74,7 @@ public class DolphinSchedulerManager {
     /**
      * init DolphinScheduler Schema
      */
-    public void initDolphinSchedulerSchema() throws SQLException, IOException {
+    public void initDolphinSchedulerSchema() {
 
         logger.info("Start initializing the DolphinScheduler manager table structure");
         upgradeDao.initSchema();

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@@ -14,25 +14,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.dolphinscheduler.dao.upgrade;
 
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.dolphinscheduler.common.enums.DbType;
 import org.apache.dolphinscheduler.common.process.ResourceInfo;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.SchemaUtils;
-import org.apache.dolphinscheduler.common.utils.ScriptRunner;
-import org.apache.dolphinscheduler.common.utils.StringUtils;
+import org.apache.dolphinscheduler.common.utils.*;
 import org.apache.dolphinscheduler.dao.AbstractBaseDao;
 import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.Reader;
+import javax.sql.DataSource;
+import java.io.*;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
@@ -43,14 +38,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import javax.sql.DataSource;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
 public abstract class UpgradeDao extends AbstractBaseDao {
 
     public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class);
@@ -60,9 +47,6 @@ public abstract class UpgradeDao extends AbstractBaseDao {
     protected static final DataSource dataSource = getDataSource();
     private static final DbType dbType = getCurrentDbType();
 
-    private static final String MYSQL_CREATE_SCRIPT = rootDir + "/sql/dolphinscheduler_mysql.sql";
-    private static final String POSTGRE_CREATE_SCRIPT = rootDir + "/sql/dolphinscheduler_postgre.sql";
-
 
     @Override
     protected void init() {
@@ -106,40 +90,119 @@ public abstract class UpgradeDao extends AbstractBaseDao {
     /**
      * init schema
      */
-    public void initSchema() throws SQLException, IOException {
+    public void initSchema() {
         DbType dbType = getDbType();
         String initSqlPath = "";
         if (dbType != null) {
             switch (dbType) {
                 case MYSQL:
-                    initSqlPath = MYSQL_CREATE_SCRIPT;
+                    initSqlPath = "/sql/create/release-1.0.0_schema/mysql/";
+                    initSchema(initSqlPath);
                     break;
                 case POSTGRESQL:
-                    initSqlPath = POSTGRE_CREATE_SCRIPT;
+                    initSqlPath = "/sql/create/release-1.2.0_schema/postgresql/";
+                    initSchema(initSqlPath);
                     break;
                 default:
                     logger.error("not support sql type: {},can't upgrade", dbType);
                     throw new IllegalArgumentException("not support sql type,can't upgrade");
             }
         }
+    }
+
+
+    /**
+     * init scheam
+     *
+     * @param initSqlPath initSqlPath
+     */
+    public void initSchema(String initSqlPath) {
+
+        // Execute the dolphinscheduler DDL, it cannot be rolled back
+        runInitDDL(initSqlPath);
+
+        // Execute the dolphinscheduler DML, it can be rolled back
+        runInitDML(initSqlPath);
+
+
+    }
+
+    /**
+     * run DML
+     *
+     * @param initSqlPath initSqlPath
+     */
+    private void runInitDML(String initSqlPath) {
+        Connection conn = null;
         if (StringUtils.isEmpty(rootDir)) {
             throw new RuntimeException("Environment variable user.dir not found");
         }
-        logger.info("Init sql filePath: {}", initSqlPath);
-        try (Connection conn = dataSource.getConnection()) {
+        String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_dml.sql";
+        try {
+            conn = dataSource.getConnection();
+            conn.setAutoCommit(false);
+
+            // Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler
+            ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true);
+            Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
+            initScriptRunner.runScript(initSqlReader);
+
+            conn.commit();
+        } catch (IOException e) {
             try {
-                conn.setAutoCommit(false);
-                ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true);
-                Reader initSqlReader = new FileReader(initSqlPath);
-                initScriptRunner.runScript(initSqlReader);
-                conn.commit();
-            } catch (IOException | SQLException e) {
                 conn.rollback();
-                logger.error("execute init script error.", e);
-                throw e;
+            } catch (SQLException e1) {
+                logger.error(e1.getMessage(), e1);
             }
+            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e.getMessage(), e);
+        } catch (Exception e) {
+            try {
+                if (null != conn) {
+                    conn.rollback();
+                }
+            } catch (SQLException e1) {
+                logger.error(e1.getMessage(), e1);
+            }
+            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e.getMessage(), e);
         } finally {
-            // ignore
+            ConnectionUtils.releaseResource(conn);
+
+        }
+
+    }
+
+    /**
+     * run DDL
+     *
+     * @param initSqlPath initSqlPath
+     */
+    private void runInitDDL(String initSqlPath) {
+        Connection conn = null;
+        if (StringUtils.isEmpty(rootDir)) {
+            throw new RuntimeException("Environment variable user.dir not found");
+        }
+        //String mysqlSQLFilePath = rootDir + "/sql/create/release-1.0.0_schema/mysql/dolphinscheduler_ddl.sql";
+        String mysqlSQLFilePath = rootDir + initSqlPath + "dolphinscheduler_ddl.sql";
+        try {
+            conn = dataSource.getConnection();
+            // Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler
+            ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
+            Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
+            initScriptRunner.runScript(initSqlReader);
+
+        } catch (IOException e) {
+
+            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e.getMessage(), e);
+        } catch (Exception e) {
+
+            logger.error(e.getMessage(), e);
+            throw new RuntimeException(e.getMessage(), e);
+        } finally {
+            ConnectionUtils.releaseResource(conn);
+
         }
 
     }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinScheduler.java

@@ -14,35 +14,35 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.dolphinscheduler.dao.upgrade.shell;
 
 import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * create DolphinScheduler
+ *
  */
 public class CreateDolphinScheduler {
 
-    private static final Logger logger = LoggerFactory.getLogger(CreateDolphinScheduler.class);
+	private static final Logger logger = LoggerFactory.getLogger(CreateDolphinScheduler.class);
 
-    /**
+	/**
 	 * create dolphin scheduler db
-     *
-     * @param args args
-    */
-    public static void main(String[] args) {
-        DolphinSchedulerManager dolphinSchedulerManager = new DolphinSchedulerManager();
-        try {
-            logger.info("create DolphinScheduler begin");
-            dolphinSchedulerManager.initDolphinScheduler();
-            logger.info("create DolphinScheduler success");
-        } catch (Exception e) {
-            logger.error("create DolphinScheduler failed", e);
-        }
+	 * @param args args
+	 */
+	public static void main(String[] args) {
+		DolphinSchedulerManager dolphinSchedulerManager = new DolphinSchedulerManager();
+		try {
+			dolphinSchedulerManager.initDolphinScheduler();
+			logger.info("init DolphinScheduler finished");
+			dolphinSchedulerManager.upgradeDolphinScheduler();
+			logger.info("upgrade DolphinScheduler finished");
+			logger.info("create DolphinScheduler success");
+		} catch (Exception e) {
+			logger.error("create DolphinScheduler failed",e);
+		}
 
-    }
+	}
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinScheduler.java

@@ -14,34 +14,29 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.dolphinscheduler.dao.upgrade.shell;
 
 import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * init DolphinScheduler
+ *
  */
 public class InitDolphinScheduler {
 
-    private static final Logger logger = LoggerFactory.getLogger(InitDolphinScheduler.class);
+	private static final Logger logger = LoggerFactory.getLogger(InitDolphinScheduler.class);
 
 	/**
 	 * init dolphin scheduler db
 	 * @param args args
 	 */
-    public static void main(String[] args) {
-        Thread.currentThread().setName("manager-InitDolphinScheduler");
-        DolphinSchedulerManager dolphinSchedulerManager = new DolphinSchedulerManager();
-        try {
-            dolphinSchedulerManager.initDolphinScheduler();
-            logger.info("init DolphinScheduler finished");
-        } catch (Exception ex) {
-            logger.error("init DolphinScheduler error", ex);
-        }
-
-    }
+	public static void main(String[] args) {
+		Thread.currentThread().setName("manager-InitDolphinScheduler");
+		DolphinSchedulerManager dolphinSchedulerManager = new DolphinSchedulerManager();
+		dolphinSchedulerManager.initDolphinScheduler();
+		logger.info("init DolphinScheduler finished");
+		
+	}
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDaoTest.java

@@ -14,76 +14,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.dolphinscheduler.dao.upgrade;
 
-import org.apache.dolphinscheduler.common.utils.ScriptRunner;
-import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
-
-import java.io.FileReader;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
+import org.junit.Test;
 
 import javax.sql.DataSource;
+import java.util.Map;
 
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
+import static org.apache.dolphinscheduler.dao.upgrade.UpgradeDao.getDataSource;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.junit.Assert.assertThat;
 
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ConnectionFactory.class, ScriptRunner.class, FileReader.class})
 public class UpgradeDaoTest {
+    PostgresqlUpgradeDao postgresqlUpgradeDao = PostgresqlUpgradeDao.getInstance();
 
     @Test
-    public void testGetCurrentVersion() throws SQLException {
-        PowerMockito.mockStatic(ConnectionFactory.class);
-        ConnectionFactory mockConnectionFactory = PowerMockito.mock(ConnectionFactory.class);
-        PowerMockito.when(ConnectionFactory.getInstance()).thenReturn(mockConnectionFactory);
-        DataSource mockDatasource = PowerMockito.mock(DataSource.class);
-        PowerMockito.when(mockConnectionFactory.getDataSource()).thenReturn(mockDatasource);
-        Connection mockConnection = PowerMockito.mock(Connection.class);
-        PowerMockito.when(mockDatasource.getConnection()).thenReturn(mockConnection);
-        PreparedStatement mockPrepareStatement = PowerMockito.mock(PreparedStatement.class);
-        PowerMockito.when(mockConnection.prepareStatement(Mockito.any())).thenReturn(mockPrepareStatement);
-        ResultSet mockResultSet = PowerMockito.mock(ResultSet.class);
-        PowerMockito.when(mockPrepareStatement.executeQuery()).thenReturn(mockResultSet);
-
-        DatabaseMetaData mockMetaData = PowerMockito.mock(DatabaseMetaData.class);
-        PowerMockito.when(mockConnection.getMetaData()).thenReturn(mockMetaData);
-        PowerMockito.when(mockMetaData.getDatabaseProductName()).thenReturn("mysql");
-
-        UpgradeDao upgradeDao = MysqlUpgradeDao.getInstance();
-        upgradeDao.getCurrentVersion("xx");
-        Assert.assertTrue(true);
+    public void testQueryQueryAllOldWorkerGroup() throws Exception{
+        postgresqlUpgradeDao.updateProcessDefinitionJsonWorkerGroup();
     }
 
-    @Test(expected = IOException.class)
-    public void testInitSchema() throws Exception {
-
-        PowerMockito.mockStatic(ConnectionFactory.class);
-        ConnectionFactory mockConnectionFactory = PowerMockito.mock(ConnectionFactory.class);
-        PowerMockito.when(ConnectionFactory.getInstance()).thenReturn(mockConnectionFactory);
-
-        DataSource mockDatasource = PowerMockito.mock(DataSource.class);
-        PowerMockito.when(mockConnectionFactory.getDataSource()).thenReturn(mockDatasource);
-        Connection mockConnection = PowerMockito.mock(Connection.class);
-        PowerMockito.when(mockDatasource.getConnection()).thenReturn(mockConnection);
-
-        DatabaseMetaData mockMetaData = PowerMockito.mock(DatabaseMetaData.class);
-        PowerMockito.when(mockConnection.getMetaData()).thenReturn(mockMetaData);
-        PowerMockito.when(mockMetaData.getDatabaseProductName()).thenReturn("mysql");
-
-        UpgradeDao upgradeDao = MysqlUpgradeDao.getInstance();
-        upgradeDao.initSchema();
-        Assert.assertTrue(true);
-
-    }
 }

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/CreateDolphinSchedulerTest.java

-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.dao.upgrade.shell;
-
-import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({DolphinSchedulerManager.class, CreateDolphinScheduler.class})
-public class CreateDolphinSchedulerTest {
-
-    @Test
-    public void mainTest() throws Exception {
-        DolphinSchedulerManager mockManager = PowerMockito.mock(DolphinSchedulerManager.class);
-        PowerMockito.whenNew(DolphinSchedulerManager.class).withNoArguments().thenReturn(mockManager);
-        CreateDolphinScheduler.main(null);
-        Assert.assertTrue(true);
-    }
-}
\ No newline at end of file

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/shell/InitDolphinSchedulerTest.java

-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.dao.upgrade.shell;
-
-import org.apache.dolphinscheduler.dao.upgrade.DolphinSchedulerManager;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({DolphinSchedulerManager.class, InitDolphinScheduler.class})
-public class InitDolphinSchedulerTest {
-
-    @Test
-    public void main() throws Exception {
-        DolphinSchedulerManager mockManager = PowerMockito.mock(DolphinSchedulerManager.class);
-        PowerMockito.whenNew(DolphinSchedulerManager.class).withNoArguments().thenReturn(mockManager);
-        InitDolphinScheduler.main(null);
-        Assert.assertTrue(true);
-    }
-}
\ No newline at end of file

pom.xml

@@ -910,8 +910,6 @@
                         <include>**/dao/datasource/MySQLDataSourceTest.java</include>
                         <include>**/dao/entity/TaskInstanceTest.java</include>
                         <include>**/dao/entity/UdfFuncTest.java</include>
-                        <include>**/dao/upgrade/shell/CreateDolphinSchedulerTest.java</include>
-                        <include>**/dao/upgrade/shell/InitDolphinSchedulerTest.java</include>
                         <include>**/remote/command/alert/AlertSendRequestCommandTest.java</include>
                         <include>**/remote/command/alert/AlertSendResponseCommandTest.java</include>
                         <include>**/remote/command/future/ResponseFutureTest.java</include>