[Feature-9474] [CI] Add cluster test script verify on shell script (#9997)

* cluster test

* fix init db failed

* fix init db failed

* fix init db failed

* fix init db failed

* fix init db failed

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add sudo

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* add github actions

* remove cluster-test workflows

* add github actions

* add github actions

* refactor test to docker mode

* refactor test to docker mode

* refactor test to docker mode

* refactor test to docker mode

* remove create schema logic

* remove create schema logic

* remove create schema logic

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* fix runnint cluster test error

* add github actions

* add github actions

* add github actions

* add cluster test timeout

* add cluster start test loop check

* add cluster start test loop check

* optimize cluster start test loop check

(cherry picked from commit baf654cb)

.github/workflows/backend.yml

@@ -78,11 +78,38 @@ jobs:
                  -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
       - name: Check dependency license
         run: tools/dependencies/check-LICENSE.sh
+      - uses: actions/upload-artifact@v2
+        name: Upload Binary Package
+        with:
+          name: binary-package
+          path: ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz
+          retention-days: 1
+  cluster-test:
+    name: Cluster-Test
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: actions/download-artifact@v2
+        name: Download Binary Package
+        with:
+          name: binary-package
+          path: ./
+      - name: Running cluster test(Mysql)
+        run: |
+          docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-base.yaml up -d
+          docker build -t jdk8:ds_cluster -f .github/workflows/cluster-test/mysql/Dockerfile .
+          docker-compose -f .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml up -d
+          /bin/bash .github/workflows/cluster-test/mysql/running_test.sh
+          docker rm -f $(docker ps -aq)
   result:
     name: Build
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    needs: [ build, paths-filter ]
+    needs: [ build, paths-filter, cluster-test ]
     if: always()
     steps:
       - name: Status
@@ -91,7 +118,7 @@ jobs:
             echo "Skip Build!"
             exit 0
           fi
-          if [[ ${{ needs.build.result }} != 'success' ]]; then
+          if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then
             echo "Build Failed!"
             exit -1
           fi

.github/workflows/cluster-test/mysql/Dockerfile

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jre-slim-buster
+
+RUN apt update ; \
+    apt install -y curl wget default-mysql-client sudo openssh-server netcat-traditional ;
+
+#COPY ./dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz
+COPY ./apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz
+RUN tar -zxvf /root/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz -C ~
+
+ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-dev-SNAPSHOT-bin
+
+#Setting install.sh
+COPY .github/workflows/cluster-test/mysql/install_env.sh $DOLPHINSCHEDULER_HOME/bin/env/install_env.sh
+
+#Setting dolphinscheduler_env.sh
+COPY .github/workflows/cluster-test/mysql/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
+
+#Download mysql jar
+ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
+RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
+cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
+cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
+cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
+cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
+
+#Deploy
+COPY .github/workflows/cluster-test/mysql/deploy.sh /root/deploy.sh
+
+CMD [ "/bin/bash", "/root/deploy.sh" ]

.github/workflows/cluster-test/mysql/deploy.sh

+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euox pipefail
+
+
+USER=root
+DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-dev-SNAPSHOT-bin
+
+#Create database
+mysql -hmysql -P3306 -uroot -p123456 -e "CREATE DATABASE IF NOT EXISTS dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;"
+
+#Sudo
+sed -i '$a'$USER'  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
+sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers
+
+#SSH
+ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+chmod 600 ~/.ssh/authorized_keys
+service ssh start
+
+#Init schema
+/bin/bash $DOLPHINSCHEDULER_HOME/tools/bin/upgrade-schema.sh
+
+#Start Cluster
+/bin/bash $DOLPHINSCHEDULER_HOME/bin/start-all.sh
+
+#Keep running
+tail -f /dev/null

.github/workflows/cluster-test/mysql/docker-compose-base.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+  mysql:
+    container_name: mysql
+    image: mysql:5.7.36
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: 123456
+    ports:
+      - "3306:3306"
+    healthcheck:
+      test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+  zoo1:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo1
+    ports:
+      - "2181:2181"
+    environment:
+      ZOO_MY_ID: 1
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo2:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo2
+    ports:
+      - "2182:2181"
+    environment:
+      ZOO_MY_ID: 2
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+
+  zoo3:
+    image: zookeeper:3.8.0
+    restart: always
+    hostname: zoo3
+    ports:
+      - "2183:2181"
+    environment:
+      ZOO_MY_ID: 3
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+

dolphinscheduler-tools/src/main/bin/create-schema.sh → .github/workflows/cluster-test/mysql/docker-compose-cluster.yaml

-#!/bin/bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -16,13 +15,15 @@
 # limitations under the License.
 #
 
-BIN_DIR=$(dirname $0)
-DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)}
+version: "3"
 
-source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh"
-
-JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"}
-
-java $JAVA_OPTS \
-  -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \
-  org.apache.dolphinscheduler.tools.datasource.CreateDolphinScheduler
+services:
+  ds:
+    container_name: ds
+    image: jdk8:ds_cluster
+    restart: always
+    ports:
+      - "12345:12345"
+      - "5679:5679"
+      - "1235:1235"
+      - "50053:50053"

.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# JAVA_HOME, will use it to start DolphinScheduler server
+export JAVA_HOME=${JAVA_HOME:-/usr/local/openjdk-8}
+
+# Database related configuration, set database type, username and password
+export DATABASE=${DATABASE:-mysql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver
+export SPRING_DATASOURCE_URL="jdbc:mysql://mysql:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
+export SPRING_DATASOURCE_USERNAME=root
+export SPRING_DATASOURCE_PASSWORD=123456
+
+# DolphinScheduler server related configuration
+export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
+export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
+
+# Registry center configuration, determines the type and link of the registry center
+export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
+export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zoo1:2181,zoo2:2182,zoo3:2183}
+
+# Tasks related configurations, need to change the configuration if you use the related tasks.
+export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
+export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
+export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
+export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
+export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
+export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH

.github/workflows/cluster-test/mysql/install_env.sh

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# ---------------------------------------------------------
+# INSTALL MACHINE
+# ---------------------------------------------------------
+# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
+# including master, worker, api, alert. If you want to deploy in pseudo-distributed
+# mode, just write a pseudo-distributed hostname
+# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
+ips=${ips:-"localhost"}
+
+# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
+# modify it if you use different ssh port
+sshPort=${sshPort:-"22"}
+
+# A comma separated list of machine hostname or IP would be installed Master server, it
+# must be a subset of configuration `ips`.
+# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
+masters=${masters:-"localhost"}
+
+# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
+# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
+# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
+workers=${workers:-"localhost:default"}
+
+# A comma separated list of machine hostname or IP would be installed Alert server, it
+# must be a subset of configuration `ips`.
+# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
+alertServer=${alertServer:-"localhost"}
+
+# A comma separated list of machine hostname or IP would be installed API server, it
+# must be a subset of configuration `ips`.
+# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
+apiServers=${apiServers:-"localhost"}
+
+# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
+# Do not set this configuration same as the current path (pwd)
+installPath=${installPath:-"/root/apache-dolphinscheduler-dev-SNAPSHOT-bin"}
+
+# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
+# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
+# to be created by this user
+deployUser=${deployUser:-"dolphinscheduler"}
+
+# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
+zkRoot=${zkRoot:-"/dolphinscheduler"}

.github/workflows/cluster-test/mysql/running_test.sh

+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -x
+
+
+API_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:12345/dolphinscheduler/actuator/health"
+MASTER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:5679/actuator/health"
+WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:1235/actuator/health"
+
+#Cluster start health check
+TIMEOUT=120
+START_HEALTHCHECK_EXITCODE=0
+
+for ((i=1; i<=TIMEOUT; i++))
+do
+  MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
+  WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
+  API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
+  if [[ $MASTER_HTTP_STATUS -eq 200 && $WORKER_HTTP_STATUS -eq 200 && $API_HTTP_STATUS -eq 200 ]];then
+    START_HEALTHCHECK_EXITCODE=0
+  else
+    START_HEALTHCHECK_EXITCODE=2
+  fi
+
+  if [[ $START_HEALTHCHECK_EXITCODE -eq 0 ]];then
+    echo "cluster start health check success"
+    break
+  fi
+
+  if [[ $i -eq $TIMEOUT ]];then
+    docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-dev-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log"
+    echo "cluster start health check failed"
+    exit $START_HEALTHCHECK_EXITCODE
+  fi
+
+  sleep 1
+done
+
+#Stop Cluster
+docker exec -u root ds bash -c "/root/apache-dolphinscheduler-dev-SNAPSHOT-bin/bin/stop-all.sh"
+
+#Cluster stop health check
+sleep 5
+MASTER_HTTP_STATUS=$(eval "$MASTER_HEALTHCHECK_COMMAND")
+if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
+  echo "master stop health check success"
+else
+  echo "master stop health check failed"
+  exit 3
+fi
+
+WORKER_HTTP_STATUS=$(eval "$WORKER_HEALTHCHECK_COMMAND")
+if [[ $WORKER_HTTP_STATUS -ne 200 ]];then
+  echo "worker stop health check success"
+else
+  echo "worker stop health check failed"
+  exit 3
+fi
+
+API_HTTP_STATUS=$(eval "$API_HEALTHCHECK_COMMAND")
+if [[ $API_HTTP_STATUS -ne 200 ]];then
+  echo "api stop health check success"
+else
+  echo "api stop health check failed"
+  exit 3
+fi

docs/docs/en/guide/installation/pseudo-cluster.md

@@ -115,7 +115,7 @@ export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
 export DATABASE=${DATABASE:-postgresql}
 export SPRING_PROFILES_ACTIVE=${DATABASE}
 export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-export SPRING_DATASOURCE_URL=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler"
 export SPRING_DATASOURCE_USERNAME={user}
 export SPRING_DATASOURCE_PASSWORD={password}
 
@@ -180,7 +180,7 @@ Then, modify `./bin/env/dolphinscheduler_env.sh` to use mysql, change {user} and
 export DATABASE=${DATABASE:-mysql}
 export SPRING_PROFILES_ACTIVE=${DATABASE}
 export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver
-export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
 export SPRING_DATASOURCE_USERNAME={user}
 export SPRING_DATASOURCE_PASSWORD={password}
 ```
@@ -188,7 +188,7 @@ export SPRING_DATASOURCE_PASSWORD={password}
 After the above steps done you would create a new database for DolphinScheduler, then run the Shell script to init database:
 
 ```shell
-sh tools/bin/create-schema.sh
+sh tools/bin/upgrade-schema.sh
 ```
 
 ## Start DolphinScheduler

docs/docs/zh/guide/installation/pseudo-cluster.md

@@ -112,7 +112,7 @@ export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
 export DATABASE=${DATABASE:-postgresql}
 export SPRING_PROFILES_ACTIVE=${DATABASE}
 export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
-export SPRING_DATASOURCE_URL=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler"
 export SPRING_DATASOURCE_USERNAME={user}
 export SPRING_DATASOURCE_PASSWORD={password}
 
@@ -177,7 +177,7 @@ mysql> FLUSH PRIVILEGES;
 export DATABASE=${DATABASE:-mysql}
 export SPRING_PROFILES_ACTIVE=${DATABASE}
 export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver
-export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&useSSL=false"
 export SPRING_DATASOURCE_USERNAME={user}
 export SPRING_DATASOURCE_PASSWORD={password}
 ```  
@@ -185,7 +185,7 @@ export SPRING_DATASOURCE_PASSWORD={password}
 完成上述步骤后，您已经为 DolphinScheduler 创建一个新数据库，现在你可以通过快速的 Shell 脚本来初始化数据库
 
 ```shell
-sh tools/bin/create-schema.sh
+sh tools/bin/upgrade-schema.sh
 ```
 
 ## 启动 DolphinScheduler

dolphinscheduler-tools/src/main/bin/upgrade-schema.sh

@@ -27,4 +27,5 @@ JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms
 
 java $JAVA_OPTS \
   -cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \
+  -Dspring.profiles.active=upgrade \
   org.apache.dolphinscheduler.tools.datasource.UpgradeDolphinScheduler

dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/CreateDolphinScheduler.java

-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.tools.datasource;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.boot.CommandLineRunner;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.stereotype.Component;
-
-@SpringBootApplication
-public class CreateDolphinScheduler {
-    public static void main(String[] args) {
-        SpringApplication.run(CreateDolphinScheduler.class, args);
-    }
-
-    @Component
-    static class CreateRunner implements CommandLineRunner {
-        private static final Logger logger = LoggerFactory.getLogger(CreateRunner.class);
-
-        private final DolphinSchedulerManager dolphinSchedulerManager;
-
-        CreateRunner(DolphinSchedulerManager dolphinSchedulerManager) {
-            this.dolphinSchedulerManager = dolphinSchedulerManager;
-        }
-
-        @Override
-        public void run(String... args) throws Exception {
-            if (dolphinSchedulerManager.schemaIsInitialized()) {
-                dolphinSchedulerManager.upgradeDolphinScheduler();
-                logger.info("upgrade DolphinScheduler finished");
-            } else {
-                dolphinSchedulerManager.initDolphinScheduler();
-                logger.info("init DolphinScheduler finished");
-            }
-            logger.info("create DolphinScheduler success");
-        }
-    }
-}

dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java

@@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Component;
 
 @SpringBootApplication
@@ -31,6 +32,7 @@ public class InitDolphinScheduler {
     }
 
     @Component
+    @Profile("init")
     static class InitRunner implements CommandLineRunner {
         private static final Logger logger = LoggerFactory.getLogger(InitRunner.class);
 

dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java

@@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Component;
 
 @SpringBootApplication
@@ -31,6 +32,7 @@ public class UpgradeDolphinScheduler {
     }
 
     @Component
+    @Profile("upgrade")
     static class UpgradeRunner implements CommandLineRunner {
         private static final Logger logger = LoggerFactory.getLogger(UpgradeRunner.class);
 
@@ -42,8 +44,13 @@ public class UpgradeDolphinScheduler {
 
         @Override
         public void run(String... args) throws Exception {
-            dolphinSchedulerManager.upgradeDolphinScheduler();
-            logger.info("upgrade DolphinScheduler success");
+            if (dolphinSchedulerManager.schemaIsInitialized()) {
+                dolphinSchedulerManager.upgradeDolphinScheduler();
+                logger.info("upgrade DolphinScheduler finished");
+            } else {
+                dolphinSchedulerManager.initDolphinScheduler();
+                logger.info("init DolphinScheduler finished");
+            }
         }
     }
 }

script/start-all.sh

@@ -33,7 +33,7 @@ mastersHost=(${masters//,/ })
 for master in ${mastersHost[@]}
 do
   echo "$master master server is starting"
-	ssh -p $sshPort $master  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start master-server;"
+	ssh -o StrictHostKeyChecking=no -p $sshPort $master  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start master-server;"
 
 done
 
@@ -41,16 +41,16 @@ for worker in ${workerNames[@]}
 do
   echo "$worker worker server is starting"
 
-  ssh -p $sshPort $worker  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start worker-server;"
+  ssh -o StrictHostKeyChecking=no -p $sshPort $worker  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start worker-server;"
 done
 
-ssh -p $sshPort $alertServer  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start alert-server;"
+ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start alert-server;"
 
 apiServersHost=(${apiServers//,/ })
 for apiServer in ${apiServersHost[@]}
 do
   echo "$apiServer api server is starting"
-  ssh -p $sshPort $apiServer  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start api-server;"
+  ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh start api-server;"
 done
 
 # query server status

script/stop-all.sh

@@ -33,21 +33,21 @@ mastersHost=(${masters//,/ })
 for master in ${mastersHost[@]}
 do
   echo "$master master server is stopping"
-	ssh -p $sshPort $master  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop master-server;"
+	ssh -o StrictHostKeyChecking=no -p $sshPort $master  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop master-server;"
 
 done
 
 for worker in ${workerNames[@]}
 do
   echo "$worker worker server is stopping"
-  ssh -p $sshPort $worker  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop worker-server;"
+  ssh -o StrictHostKeyChecking=no -p $sshPort $worker  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop worker-server;"
 done
 
-ssh -p $sshPort $alertServer  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop alert-server;"
+ssh -o StrictHostKeyChecking=no -p $sshPort $alertServer  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop alert-server;"
 
 apiServersHost=(${apiServers//,/ })
 for apiServer in ${apiServersHost[@]}
 do
   echo "$apiServer api server is stopping"
-  ssh -p $sshPort $apiServer  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop api-server;"
+  ssh -o StrictHostKeyChecking=no -p $sshPort $apiServer  "cd $installPath/; bash bin/dolphinscheduler-daemon.sh stop api-server;"
 done