Merge pull request #9 from apache/dev

update code 

.github/workflows/ci_e2e.yml

@@ -44,15 +44,14 @@ jobs:
             ${{ runner.os }}-maven-
       - name: Build Image
         run: |
-          export VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
-          sh ./dockerfile/hooks/build
+          sh ./docker/build/hooks/build
       - name: Docker Run
         run: |
-          VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
-          mkdir -p /tmp/logs
-          docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all
+          export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
+          sed -i "s/apache\/dolphinscheduler:latest/apache\/dolphinscheduler:${VERSION}/g" $(pwd)/docker/docker-swarm/docker-compose.yml
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d
       - name: Check Server Status
-        run: sh ./dockerfile/hooks/check
+        run: sh $(pwd)/docker/docker-swarm/check
       - name: Prepare e2e env
         run: |
           sudo apt-get install -y libxss1 libappindicator1 libindicator7 xvfb unzip libgbm1
@@ -70,6 +69,6 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: dslogs
-          path: /tmp/logs
+          path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data
 
 

.github/workflows/ci_ut.yml

@@ -21,7 +21,6 @@ on:
     branches:
       - dev
 env:
-  DOCKER_DIR: ./docker
   LOG_DIR: /tmp/dolphinscheduler
 
 name: Unit Test
@@ -47,7 +46,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Bootstrap database
-        run: cd ${DOCKER_DIR} && docker-compose up -d
+        run: |
+          sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml create --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql
+          sudo cp $(pwd)/sql/dolphinscheduler-postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}')
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
@@ -82,6 +85,5 @@ jobs:
       - name: Collect logs
         run: |
           mkdir -p ${LOG_DIR}
-          cd ${DOCKER_DIR}
-          docker-compose logs db > ${LOG_DIR}/db.txt
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt
         continue-on-error: true

dockerfile/Dockerfile → docker/build/Dockerfile

@@ -37,32 +37,19 @@ RUN apk add openjdk8
 ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk
 ENV PATH $JAVA_HOME/bin:$PATH
 
-#3. install zk
-RUN cd /opt && \
-    wget https://downloads.apache.org/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz && \
-    tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz && \
-    mv apache-zookeeper-3.5.7-bin zookeeper && \
-    mkdir -p /tmp/zookeeper && \
-    rm -rf ./zookeeper-*tar.gz && \
-    rm -rf /opt/zookeeper/conf/zoo_sample.cfg
-ADD ./conf/zookeeper/zoo.cfg /opt/zookeeper/conf
-ENV ZK_HOME /opt/zookeeper
-ENV PATH $ZK_HOME/bin:$PATH
+#3. add dolphinscheduler
+ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
+RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
+ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
 
 #4. install pg
 RUN apk add postgresql postgresql-contrib
-
-#5. add dolphinscheduler
-ADD ./apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz /opt/
-RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin/ /opt/dolphinscheduler/
-ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
-
-#6. modify nginx
+#5. modify nginx
 RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
     rm -rf /etc/nginx/conf.d/*
 ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
 
-#7. add configuration and modify permissions and set soft links
+#6. add configuration and modify permissions and set soft links
 ADD ./checkpoint.sh /root/checkpoint.sh
 ADD ./startup-init-conf.sh /root/startup-init-conf.sh
 ADD ./startup.sh /root/startup.sh
@@ -75,22 +62,21 @@ RUN chmod +x /root/checkpoint.sh && \
     chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
     chmod +x /opt/dolphinscheduler/script/*.sh && \
     chmod +x /opt/dolphinscheduler/bin/*.sh && \
-    chmod +x /opt/zookeeper/bin/*.sh && \
     dos2unix /root/checkpoint.sh && \
     dos2unix /root/startup-init-conf.sh && \
     dos2unix /root/startup.sh && \
     dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
     dos2unix /opt/dolphinscheduler/script/*.sh && \
     dos2unix /opt/dolphinscheduler/bin/*.sh && \
-    dos2unix /opt/zookeeper/bin/*.sh && \
     rm -rf /bin/sh && \
     ln -s /bin/bash /bin/sh && \
     mkdir -p /tmp/xls
 
-#8. remove apk index cache
-RUN rm -rf /var/cache/apk/*
+#7. remove apk index cache and disable coredup for sudo
+RUN rm -rf /var/cache/apk/* && \
+    echo "Set disable_coredump false" >> /etc/sudo.conf
 
-#9. expose port
+#8. expose port
 EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888
 
-ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]
\ No newline at end of file

dockerfile/README.md → docker/build/README.md

@@ -109,16 +109,16 @@ In Unix-Like, Example:
 
 ```bash
 $ cd path/incubator-dolphinscheduler
-$ sh ./dockerfile/hooks/build
+$ sh ./docker/build/hooks/build
 ```
 
 In Windows, Example:
 
 ```bat
-c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat
+c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
 ```
 
-Please read `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat` script files if you don't understand
+Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand
 
 ## Environment Variables
 

dockerfile/README_zh_CN.md → docker/build/README_zh_CN.md

@@ -109,16 +109,16 @@ dolphinscheduler frontend
 
 ```bash
 $ cd path/incubator-dolphinscheduler
-$ sh ./dockerfile/hooks/build
+$ sh ./docker/build/hooks/build
 ```
 
 Windows系统, 如下:
 
 ```bat
-c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat
+c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
 ```
 
-如果你不理解这些脚本 `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat`，请阅读里面的内容。
+如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。
 
 ## 环境变量
 

dockerfile/hooks/build → docker/build/hooks/build

@@ -24,13 +24,13 @@ printenv
 if [ -z "${VERSION}" ]
 then
     echo "set default environment variable [VERSION]"
-    VERSION=$(cat $(pwd)/sql/soft_version)
+    export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
 fi
 
 if [ "${DOCKER_REPO}x" = "x" ]
 then
     echo "set default environment variable [DOCKER_REPO]"
-    DOCKER_REPO='dolphinscheduler'
+    export DOCKER_REPO='apache/dolphinscheduler'
 fi
 
 echo "Version: $VERSION"
@@ -42,12 +42,12 @@ echo -e "Current Directory is $(pwd)\n"
 echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true"
 mvn -B clean compile package -Prelease -Dmaven.test.skip=true
 
-# mv dolphinscheduler-bin.tar.gz file to dockerfile directory
-echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n"
-mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/
+# mv dolphinscheduler-bin.tar.gz file to docker/build directory
+echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n"
+mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
 
 # docker build
-echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/\n"
-docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/
+echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n"
+sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/
 
 echo "------ dolphinscheduler end   - build -------"

dockerfile/hooks/build.bat → docker/build/hooks/build.bat

@@ -38,13 +38,13 @@ echo "call mvn clean compile package -Prelease"
 call mvn clean compile package -Prelease -DskipTests=true
 if "%errorlevel%"=="1" goto :mvnFailed
 
-:: move dolphinscheduler-bin.tar.gz file to dockerfile directory
-echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\dockerfile\"
-move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\dockerfile\
+:: move dolphinscheduler-bin.tar.gz file to docker/build directory
+echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\"
+move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\
 
 :: docker build
-echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\dockerfile\"
-docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\dockerfile\
+echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\"
+docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\
 if "%errorlevel%"=="1" goto :dockerBuildFailed
 
 echo "------ dolphinscheduler end - build -------"
@@ -53,4 +53,4 @@ echo "------ dolphinscheduler end - build -------"
 echo "MAVEN PACKAGE FAILED!"
 
 :dockerBuildFailed
-echo "DOCKER BUILD FAILED!"
+echo "DOCKER BUILD FAILED!"
\ No newline at end of file

dockerfile/startup.sh → docker/build/startup.sh

@@ -24,31 +24,6 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
 
 # start postgresql
 initPostgreSQL() {
-    echo "checking postgresql"
-    if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" = "localhost" ]]; then
-        export PGPORT=${POSTGRESQL_PORT}
-
-        echo "start postgresql service"
-        rc-service postgresql restart
-
-        # role if not exists, create
-        flag=$(sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${POSTGRESQL_USERNAME}'")
-        if [ -z "${flag}" ]; then
-            echo "create user"
-            sudo -u postgres psql -tAc "create user ${POSTGRESQL_USERNAME} with password '${POSTGRESQL_PASSWORD}'"
-        fi
-
-        # database if not exists, create
-        flag=$(sudo -u postgres psql -tAc "select 1 from pg_database where datname='dolphinscheduler'")
-        if [ -z "${flag}" ]; then
-            echo "init db"
-            sudo -u postgres psql -tAc "create database dolphinscheduler owner ${POSTGRESQL_USERNAME}"
-        fi
-
-        # grant
-        sudo -u postgres psql -tAc "grant all privileges on database dolphinscheduler to ${POSTGRESQL_USERNAME}"
-    fi
-
     echo "test postgresql service"
     while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
         counter=$((counter+1))
@@ -73,24 +48,18 @@ initPostgreSQL() {
 
 # start zk
 initZK() {
-    echo -e "checking zookeeper"
-    if [[ "${ZOOKEEPER_QUORUM}" = "127.0.0.1:2181" || "${ZOOKEEPER_QUORUM}" = "localhost:2181" ]]; then
-        echo "start local zookeeper"
-        /opt/zookeeper/bin/zkServer.sh restart
-    else
-        echo "connect remote zookeeper"
-        echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
-            while ! nc -z ${line%:*} ${line#*:}; do
-                counter=$((counter+1))
-                if [ $counter == 30 ]; then
-                    echo "Error: Couldn't connect to zookeeper."
-                    exit 1
-                fi
-                echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
-                sleep 5
-            done
+    echo "connect remote zookeeper"
+    echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
+        while ! nc -z ${line%:*} ${line#*:}; do
+            counter=$((counter+1))
+            if [ $counter == 30 ]; then
+                echo "Error: Couldn't connect to zookeeper."
+                exit 1
+            fi
+            echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
+            sleep 5
         done
-    fi
+    done
 }
 
 # start nginx

docker/docker-compose.yml

-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-version: '2'
-services:
-  zookeeper:
-    image: zookeeper
-    restart: always
-    container_name: zookeeper
-    ports:
-      - "2181:2181"
-    environment:
-      ZOO_MY_ID: 1
-      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
-  db:
-    image: postgres
-    container_name: postgres
-    environment:
-      - POSTGRES_USER=test
-      - POSTGRES_PASSWORD=test
-      - POSTGRES_DB=dolphinscheduler
-    ports:
-      - "5432:5432"
-    volumes:
-      - pgdata:/var/lib/postgresql/data
-      - ./postgres/docker-entrypoint-initdb:/docker-entrypoint-initdb.d
-volumes:
-  pgdata:

dockerfile/hooks/check → docker/docker-swarm/check

@@ -17,7 +17,7 @@
 #
 echo "------ dolphinscheduler check - server - status -------"
 sleep 60
-server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
+server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
 if [ $server_num -eq 5 ]
 then
     echo "Server all start successfully"

docker/docker-swarm/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       POSTGRESQL_DATABASE: dolphinscheduler
     volumes:
     - dolphinscheduler-postgresql:/bitnami/postgresql
+    - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
     networks:
     - dolphinscheduler
 
@@ -41,13 +42,14 @@ services:
     environment:
       TZ: Asia/Shanghai
       ALLOW_ANONYMOUS_LOGIN: "yes"
+      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
     volumes:
     - dolphinscheduler-zookeeper:/bitnami/zookeeper
     networks:
     - dolphinscheduler
 
   dolphinscheduler-api:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     container_name: dolphinscheduler-api
     command: ["api-server"]
     ports:
@@ -70,12 +72,12 @@ services:
     - dolphinscheduler-postgresql
     - dolphinscheduler-zookeeper
     volumes:
-    - dolphinscheduler-api:/opt/dolphinscheduler/logs
-    networks: 
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+    networks:
     - dolphinscheduler
 
   dolphinscheduler-frontend:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     container_name: dolphinscheduler-frontend
     command: ["frontend"]
     ports:
@@ -93,12 +95,12 @@ services:
     depends_on:
     - dolphinscheduler-api
     volumes:
-    - dolphinscheduler-frontend:/var/log/nginx
+    - dolphinscheduler-logs:/var/log/nginx
     networks:
     - dolphinscheduler
 
   dolphinscheduler-alert:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     container_name: dolphinscheduler-alert
     command: ["alert-server"]
     environment:
@@ -130,18 +132,18 @@ services:
       start_period: 30s
     depends_on:
     - dolphinscheduler-postgresql
-    volumes: 
-    - dolphinscheduler-alert:/opt/dolphinscheduler/logs
-    networks: 
+    volumes:
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+    networks:
     - dolphinscheduler
 
   dolphinscheduler-master:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     container_name: dolphinscheduler-master
     command: ["master-server"]
-    ports: 
+    ports:
     - 5678:5678
-    environment: 
+    environment:
       TZ: Asia/Shanghai
       MASTER_EXEC_THREADS: "100"
       MASTER_EXEC_TASK_NUM: "20"
@@ -162,22 +164,22 @@ services:
       timeout: 5s
       retries: 3
       start_period: 30s
-    depends_on: 
+    depends_on:
     - dolphinscheduler-postgresql
     - dolphinscheduler-zookeeper
-    volumes: 
-    - dolphinscheduler-master:/opt/dolphinscheduler/logs
+    volumes:
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
     networks:
     - dolphinscheduler
 
   dolphinscheduler-worker:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     container_name: dolphinscheduler-worker
     command: ["worker-server"]
-    ports: 
+    ports:
     - 1234:1234
     - 50051:50051
-    environment: 
+    environment:
       TZ: Asia/Shanghai
       WORKER_EXEC_THREADS: "100"
       WORKER_HEARTBEAT_INTERVAL: "10"
@@ -209,7 +211,7 @@ services:
       source: dolphinscheduler-worker-data
       target: /tmp/dolphinscheduler
     - type: volume
-      source: dolphinscheduler-worker-logs
+      source: dolphinscheduler-logs
       target: /opt/dolphinscheduler/logs
     networks:
     - dolphinscheduler
@@ -220,13 +222,10 @@ networks:
 
 volumes:
   dolphinscheduler-postgresql:
+  dolphinscheduler-postgresql-initdb:
   dolphinscheduler-zookeeper:
-  dolphinscheduler-api:
-  dolphinscheduler-frontend:
-  dolphinscheduler-alert:
-  dolphinscheduler-master:
   dolphinscheduler-worker-data:
-  dolphinscheduler-worker-logs:
+  dolphinscheduler-logs:
 
 configs:
   dolphinscheduler-worker-task-env:

docker/docker-swarm/docker-stack.yml

@@ -42,6 +42,7 @@ services:
     environment:
       TZ: Asia/Shanghai
       ALLOW_ANONYMOUS_LOGIN: "yes"
+      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
     volumes:
     - dolphinscheduler-zookeeper:/bitnami/zookeeper
     networks:
@@ -51,7 +52,7 @@ services:
       replicas: 1
 
   dolphinscheduler-api:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     command: ["api-server"]
     ports:
     - 12345:12345
@@ -70,7 +71,7 @@ services:
       retries: 3
       start_period: 30s
     volumes:
-    - dolphinscheduler-api:/opt/dolphinscheduler/logs
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
     networks: 
     - dolphinscheduler
     deploy:
@@ -78,7 +79,7 @@ services:
       replicas: 1
 
   dolphinscheduler-frontend:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     command: ["frontend"]
     ports:
     - 8888:8888
@@ -93,7 +94,7 @@ services:
       retries: 3
       start_period: 30s
     volumes:
-    - dolphinscheduler-frontend:/var/log/nginx
+    - dolphinscheduler-logs:/var/log/nginx
     networks:
     - dolphinscheduler
     deploy:
@@ -101,7 +102,7 @@ services:
       replicas: 1
 
   dolphinscheduler-alert:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     command: ["alert-server"]
     environment:
       TZ: Asia/Shanghai
@@ -131,7 +132,7 @@ services:
       retries: 3
       start_period: 30s
     volumes:
-    - dolphinscheduler-alert:/opt/dolphinscheduler/logs
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
     networks: 
     - dolphinscheduler
     deploy:
@@ -139,7 +140,7 @@ services:
       replicas: 1
 
   dolphinscheduler-master:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     command: ["master-server"]
     ports: 
     - 5678:5678
@@ -165,7 +166,7 @@ services:
       retries: 3
       start_period: 30s
     volumes:
-    - dolphinscheduler-master:/opt/dolphinscheduler/logs
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
     networks:
     - dolphinscheduler
     deploy:
@@ -173,7 +174,7 @@ services:
       replicas: 1
 
   dolphinscheduler-worker:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
     command: ["worker-server"]
     ports: 
     - 1234:1234
@@ -201,7 +202,7 @@ services:
       start_period: 30s
     volumes:
     - dolphinscheduler-worker-data:/tmp/dolphinscheduler
-    - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
     configs:
     - source: dolphinscheduler-worker-task-env
       target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
@@ -218,12 +219,8 @@ networks:
 volumes:
   dolphinscheduler-postgresql:
   dolphinscheduler-zookeeper:
-  dolphinscheduler-api:
-  dolphinscheduler-frontend:
-  dolphinscheduler-alert:
-  dolphinscheduler-master:
   dolphinscheduler-worker-data:
-  dolphinscheduler-worker-logs:
+  dolphinscheduler-logs:
 
 configs:
   dolphinscheduler-worker-task-env:

docker/kubernetes/dolphinscheduler/values.yaml

@@ -25,9 +25,9 @@ fullnameOverride: ""
 timezone: "Asia/Shanghai"
 
 image:
-  registry: "docker.io"
+  registry: "apache"
   repository: "dolphinscheduler"
-  tag: "1.3.0"
+  tag: "latest"
   pullPolicy: "IfNotPresent"
 
 imagePullSecrets: []
@@ -56,6 +56,8 @@ externalDatabase:
 zookeeper:
   enabled: true
   taskQueue: "zookeeper"
+  config:
+    ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
   service:
     port: "2181"
   persistence:

dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java

@@ -56,7 +56,7 @@ public class EmailAlertPlugin implements AlertPlugin {
 
     @Override
     public String getId() {
-        return Constants.PLUGIN_DEFAULT_EMAIL;
+        return Constants.PLUGIN_DEFAULT_EMAIL_ID;
     }
 
     @Override

dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java

@@ -80,7 +80,7 @@ public class AlertSender {
 
             alertInfo.addProp("receivers", receviersList);
 
-            AlertPlugin emailPlugin = pluginManager.findOne(Constants.PLUGIN_DEFAULT_EMAIL);
+            AlertPlugin emailPlugin = pluginManager.findOne(Constants.PLUGIN_DEFAULT_EMAIL_ID);
             retMaps = emailPlugin.process(alertInfo);
 
             if (retMaps == null) {

dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java

@@ -179,7 +179,7 @@ public class Constants {
      */
     public static final String PLUGIN_DIR = "plugin.dir";
 
-    public static final String PLUGIN_DEFAULT_EMAIL = "email";
+    public static final String PLUGIN_DEFAULT_EMAIL_ID = "email";
 
     public static final String PLUGIN_DEFAULT_EMAIL_CH = "邮件";
 

dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java

@@ -47,7 +47,7 @@ public class EmailAlertPluginTest {
     @Test
     public void getId() {
         String id = plugin.getId();
-        assertEquals(Constants.PLUGIN_DEFAULT_EMAIL, id);
+        assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_ID, id);
     }
 
     @Test

dolphinscheduler-common/pom.xml

@@ -594,5 +594,19 @@
 			<artifactId>janino</artifactId>
 			<version>${codehaus.janino.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>com.github.rholder</groupId>
+			<artifactId>guava-retrying</artifactId>
+			<exclusions>
+				<exclusion>
+					<groupId>com.google.guava</groupId>
+					<artifactId>guava</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>com.google.code.findbugs</groupId>
+					<artifactId>jsr305</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
 	</dependencies>
 </project>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java

@@ -90,8 +90,8 @@ public class FilePluginManager implements PluginManager {
     }
 
     @Override
-    public AlertPlugin findOne(String name) {
-        return pluginMap.get(name);
+    public AlertPlugin findOne(String pluginId) {
+        return pluginMap.get(pluginId);
     }
 
     @Override

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.utils;
+
+import com.github.rholder.retry.*;
+import org.apache.dolphinscheduler.common.Constants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * The Retryer util.
+ */
+public class RetryerUtils {
+    private static final Logger logger = LoggerFactory.getLogger(RetryerUtils.class);
+    private static Retryer<Boolean> defaultRetryerResultCheck;
+    private static Retryer<Boolean> defaultRetryerResultNoCheck;
+
+    private RetryerUtils() {
+
+    }
+
+    private static Retryer<Boolean> getDefaultRetryerResultNoCheck() {
+        if (defaultRetryerResultNoCheck == null) {
+            defaultRetryerResultNoCheck = RetryerBuilder
+                    .<Boolean>newBuilder()
+                    .retryIfException()
+                    .withWaitStrategy(WaitStrategies.fixedWait(Constants.SLEEP_TIME_MILLIS, TimeUnit.MILLISECONDS))
+                    .withStopStrategy(StopStrategies.stopAfterAttempt(3))
+                    .build();
+        }
+        return defaultRetryerResultNoCheck;
+    }
+
+    /**
+     * Gets default retryer.
+     * the retryer will retry 3 times if exceptions throw
+     * and wait 1 second between each retry
+     *
+     * @param checkResult true means the callable must return true before retrying
+     *                    false means that retry callable only throw exceptions
+     * @return the default retryer
+     */
+    public static Retryer<Boolean> getDefaultRetryer(boolean checkResult) {
+        return checkResult ? getDefaultRetryer() : getDefaultRetryerResultNoCheck();
+    }
+
+    /**
+     * Gets default retryer.
+     * the retryer will retry 3 times if exceptions throw
+     * and wait 1 second between each retry
+     *
+     * @return the default retryer
+     */
+    public static Retryer<Boolean> getDefaultRetryer() {
+        if (defaultRetryerResultCheck == null) {
+            defaultRetryerResultCheck = RetryerBuilder
+                    .<Boolean>newBuilder()
+                    .retryIfResult(Boolean.FALSE::equals)
+                    .retryIfException()
+                    .withWaitStrategy(WaitStrategies.fixedWait(Constants.SLEEP_TIME_MILLIS, TimeUnit.MILLISECONDS))
+                    .withStopStrategy(StopStrategies.stopAfterAttempt(3))
+                    .build();
+        }
+        return defaultRetryerResultCheck;
+    }
+
+    /**
+     * Use RETRYER to invoke the Callable
+     *
+     * @param callable    the callable
+     * @param checkResult true means that retry callable before returning true
+     *                    false means that retry callable only throw exceptions
+     * @return the final result of callable
+     * @throws ExecutionException the execution exception
+     * @throws RetryException     the retry exception
+     */
+    public static Boolean retryCall(final Callable<Boolean> callable, boolean checkResult) throws ExecutionException, RetryException {
+        return getDefaultRetryer(checkResult).call(callable);
+    }
+
+    /**
+     * Use RETRYER to invoke the Callable before returning true
+     *
+     * @param callable the callable
+     * @return the boolean
+     * @throws ExecutionException the execution exception
+     * @throws RetryException     the retry exception
+     */
+    public static Boolean retryCall(final Callable<Boolean> callable) throws ExecutionException, RetryException {
+        return retryCall(callable, true);
+    }
+
+    /**
+     * Retry call silent without exceptions thrown
+     *
+     * @param callable    the callable
+     * @param checkResult whether check result
+     * @return if no exceptions ,it's result returned by callable ,else always false
+     */
+    public static boolean retryCallSilent(final Callable<Boolean> callable, boolean checkResult) {
+        boolean result = false;
+        try {
+            result = getDefaultRetryer(checkResult).call(callable);
+        } catch (ExecutionException | RetryException e) {
+            logger.warn("Retry call {} failed {}", callable, e.getMessage(), e);
+        }
+        return result;
+    }
+
+    /**
+     * Retry call silent without exceptions thrown
+     *
+     * @param callable the callable
+     * @return if no exceptions ,it's result returned by callable ,else always false
+     */
+    public static boolean retryCallSilent(final Callable<Boolean> callable) {
+        return retryCallSilent(callable, true);
+    }
+}

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common.utils;
+
+import com.github.rholder.retry.RetryException;
+import com.github.rholder.retry.Retryer;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.ExecutionException;
+
+public class RetryerUtilsTest {
+
+    @Test
+    public void testDefaultRetryer() {
+        Retryer<Boolean> retryer = RetryerUtils.getDefaultRetryer();
+        Assert.assertNotNull(retryer);
+        try {
+            boolean result = retryer.call(() -> true);
+            Assert.assertTrue(result);
+        } catch (ExecutionException | RetryException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+        Retryer<Boolean> retryer1 = RetryerUtils.getDefaultRetryer(true);
+        Assert.assertEquals(retryer, retryer1);
+    }
+
+    @Test
+    public void testDefaultRetryerResultCheck() {
+        Retryer<Boolean> retryer = RetryerUtils.getDefaultRetryer();
+        Assert.assertNotNull(retryer);
+        try {
+            for (int execTarget = 1; execTarget <= 3; execTarget++) {
+                int finalExecTarget = execTarget;
+                int[] execTime = {0};
+                boolean result = retryer.call(() -> {
+                    execTime[0]++;
+                    return execTime[0] == finalExecTarget;
+                });
+                Assert.assertEquals(finalExecTarget, execTime[0]);
+                Assert.assertTrue(result);
+            }
+        } catch (ExecutionException | RetryException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+        int[] execTime = {0};
+        try {
+            retryer.call(() -> {
+                execTime[0]++;
+                return execTime[0] == 4;
+            });
+            Assert.fail("Retry times not reached");
+        } catch (RetryException e) {
+            Assert.assertEquals(3, e.getNumberOfFailedAttempts());
+            Assert.assertEquals(3, execTime[0]);
+        } catch (ExecutionException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+    }
+
+    @Test
+    public void testDefaultRetryerResultNoCheck() {
+        Retryer<Boolean> retryer = RetryerUtils.getDefaultRetryer(false);
+        Assert.assertNotNull(retryer);
+        try {
+            for (int execTarget = 1; execTarget <= 5; execTarget++) {
+                int[] execTime = {0};
+                boolean result = retryer.call(() -> {
+                    execTime[0]++;
+                    return execTime[0] > 1;
+                });
+                Assert.assertEquals(1, execTime[0]);
+                Assert.assertFalse(result);
+            }
+        } catch (ExecutionException | RetryException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+    }
+
+    @Test
+    public void testRecallResultCheck() {
+        try {
+            for (int execTarget = 1; execTarget <= 3; execTarget++) {
+                int finalExecTarget = execTarget;
+                int[] execTime = {0};
+                boolean result = RetryerUtils.retryCall(() -> {
+                    execTime[0]++;
+                    return execTime[0] == finalExecTarget;
+                });
+                Assert.assertEquals(finalExecTarget, execTime[0]);
+                Assert.assertTrue(result);
+            }
+        } catch (ExecutionException | RetryException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+        int[] execTime = {0};
+        try {
+            RetryerUtils.retryCall(() -> {
+                execTime[0]++;
+                return execTime[0] == 4;
+            });
+            Assert.fail("Recall times not reached");
+        } catch (RetryException e) {
+            Assert.assertEquals(3, e.getNumberOfFailedAttempts());
+            Assert.assertEquals(3, execTime[0]);
+        } catch (ExecutionException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+    }
+
+    @Test
+    public void testRecallResultCheckWithPara() {
+        try {
+            for (int execTarget = 1; execTarget <= 3; execTarget++) {
+                int finalExecTarget = execTarget;
+                int[] execTime = {0};
+                boolean result = RetryerUtils.retryCall(() -> {
+                    execTime[0]++;
+                    return execTime[0] == finalExecTarget;
+                }, true);
+                Assert.assertEquals(finalExecTarget, execTime[0]);
+                Assert.assertTrue(result);
+            }
+        } catch (ExecutionException | RetryException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+        int[] execTime = {0};
+        try {
+            RetryerUtils.retryCall(() -> {
+                execTime[0]++;
+                return execTime[0] == 4;
+            }, true);
+            Assert.fail("Recall times not reached");
+        } catch (RetryException e) {
+            Assert.assertEquals(3, e.getNumberOfFailedAttempts());
+            Assert.assertEquals(3, execTime[0]);
+        } catch (ExecutionException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+    }
+
+    @Test
+    public void testRecallResultNoCheck() {
+        try {
+            for (int execTarget = 1; execTarget <= 5; execTarget++) {
+                int[] execTime = {0};
+                boolean result = RetryerUtils.retryCall(() -> {
+                    execTime[0]++;
+                    return execTime[0] > 1;
+                }, false);
+                Assert.assertEquals(1, execTime[0]);
+                Assert.assertFalse(result);
+            }
+        } catch (ExecutionException | RetryException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+    }
+
+    private void testRetryExceptionWithPara(boolean checkResult) {
+        try {
+            for (int execTarget = 1; execTarget <= 3; execTarget++) {
+                int finalExecTarget = execTarget;
+                int[] execTime = {0};
+                boolean result = RetryerUtils.retryCall(() -> {
+                    execTime[0]++;
+                    if (execTime[0] != finalExecTarget) {
+                        throw new IllegalArgumentException(String.valueOf(execTime[0]));
+                    }
+                    return true;
+                }, checkResult);
+                Assert.assertEquals(finalExecTarget, execTime[0]);
+                Assert.assertTrue(result);
+            }
+        } catch (ExecutionException | RetryException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+        int[] execTime = {0};
+        try {
+            RetryerUtils.retryCall(() -> {
+                execTime[0]++;
+                if (execTime[0] != 4) {
+                    throw new IllegalArgumentException(String.valueOf(execTime[0]));
+                }
+                return true;
+            }, checkResult);
+            Assert.fail("Recall times not reached");
+        } catch (RetryException e) {
+            Assert.assertEquals(3, e.getNumberOfFailedAttempts());
+            Assert.assertEquals(3, execTime[0]);
+            Assert.assertNotNull(e.getCause());
+            Assert.assertEquals(3, Integer.parseInt(e.getCause().getMessage()));
+        } catch (ExecutionException e) {
+            Assert.fail("Retry call failed " + e.getMessage());
+        }
+    }
+
+    @Test
+    public void testRetryException() {
+        testRetryExceptionWithPara(true);
+        testRetryExceptionWithPara(false);
+    }
+
+    @Test
+    public void testRetrySilent() {
+        try {
+            for (int execTarget = 1; execTarget <= 3; execTarget++) {
+                int finalExecTarget = execTarget;
+                int[] execTime = {0};
+                boolean result = RetryerUtils.retryCallSilent(() -> {
+                    execTime[0]++;
+                    return execTime[0] == finalExecTarget;
+                });
+                Assert.assertEquals(finalExecTarget, execTime[0]);
+                Assert.assertTrue(result);
+            }
+        } catch (Exception e) {
+            Assert.fail("Unexpected exception " + e.getMessage());
+        }
+        int[] execTime = {0};
+        try {
+            boolean result = RetryerUtils.retryCallSilent(() -> {
+                execTime[0]++;
+                return execTime[0] == 4;
+            });
+            Assert.assertFalse(result);
+        } catch (Exception e) {
+            Assert.fail("Unexpected exception  " + e.getMessage());
+        }
+    }
+
+    @Test
+    public void testRetrySilentWithPara() {
+        try {
+            for (int execTarget = 1; execTarget <= 3; execTarget++) {
+                int finalExecTarget = execTarget;
+                int[] execTime = {0};
+                boolean result = RetryerUtils.retryCallSilent(() -> {
+                    execTime[0]++;
+                    return execTime[0] == finalExecTarget;
+                }, true);
+                Assert.assertEquals(finalExecTarget, execTime[0]);
+                Assert.assertTrue(result);
+            }
+        } catch (Exception e) {
+            Assert.fail("Unexpected exception " + e.getMessage());
+        }
+        int[] execTime = {0};
+        try {
+            boolean result = RetryerUtils.retryCallSilent(() -> {
+                execTime[0]++;
+                return execTime[0] == 4;
+            }, true);
+            Assert.assertFalse(result);
+        } catch (Exception e) {
+            Assert.fail("Unexpected exception  " + e.getMessage());
+        }
+    }
+    @Test
+    public void testRetrySilentNoCheckResult(){
+        try {
+            for (int execTarget = 1; execTarget <= 5; execTarget++) {
+                int[] execTime = {0};
+                boolean result = RetryerUtils.retryCallSilent(() -> {
+                    execTime[0]++;
+                    return execTime[0] > 1;
+                }, false);
+                Assert.assertEquals(1, execTime[0]);
+                Assert.assertFalse(result);
+            }
+        } catch (Exception e) {
+            Assert.fail("Unexpected exception " + e.getMessage());
+        }
+    }
+    private void testRetrySilentExceptionWithPara(boolean checkResult) {
+        try {
+            for (int execTarget = 1; execTarget <= 3; execTarget++) {
+                int finalExecTarget = execTarget;
+                int[] execTime = {0};
+                boolean result = RetryerUtils.retryCallSilent(() -> {
+                    execTime[0]++;
+                    if (execTime[0] != finalExecTarget) {
+                        throw new IllegalArgumentException(String.valueOf(execTime[0]));
+                    }
+                    return true;
+                }, checkResult);
+                Assert.assertEquals(finalExecTarget, execTime[0]);
+                Assert.assertTrue(result);
+            }
+        } catch (Exception e) {
+            Assert.fail("Unexpected exception " + e.getMessage());
+        }
+        int[] execTime = {0};
+        try {
+            boolean result = RetryerUtils.retryCallSilent(() -> {
+                execTime[0]++;
+                if (execTime[0] != 4) {
+                    throw new IllegalArgumentException(String.valueOf(execTime[0]));
+                }
+                return true;
+            }, checkResult);
+            Assert.assertFalse(result);
+        } catch (Exception e) {
+            Assert.fail("Unexpected exception " + e.getMessage());
+        }
+    }
+    @Test
+    public void testRetrySilentException() {
+        testRetrySilentExceptionWithPara(true);
+        testRetrySilentExceptionWithPara(false);
+    }
+}

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java

@@ -17,7 +17,6 @@
 
 package org.apache.dolphinscheduler.server.master.dispatch.executor;
 
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.dolphinscheduler.remote.NettyRemotingClient;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
@@ -36,10 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.PostConstruct;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.*;
 
 /**
  *  netty executor manager
@@ -87,17 +83,11 @@ public class NettyExecutorManager extends AbstractExecutorManager<Boolean>{
      */
     @Override
     public Boolean execute(ExecutionContext context) throws ExecuteException {
-
-        /**
-         *  all nodes
-         */
-        Set<String> allNodes = getAllNodes(context);
-
-        /**
-         * fail nodes
-         */
-        Set<String> failNodeSet = new HashSet<>();
-
+        LinkedList<String> allNodes = new LinkedList<>();
+        Set<String> nodes = getAllNodes(context);
+        if (nodes != null) {
+            allNodes.addAll(nodes);
+        }
         /**
          *  build command accord executeContext
          */
@@ -106,31 +96,27 @@ public class NettyExecutorManager extends AbstractExecutorManager<Boolean>{
         /**
          * execute task host
          */
-        Host host = context.getHost();
+        String startHostAddress = context.getHost().getAddress();
+        // remove start host address and add it to head
+        allNodes.remove(startHostAddress);
+        allNodes.addFirst(startHostAddress);
+ 
         boolean success = false;
-        while (!success) {
+        for (String address : allNodes) {
             try {
-                doExecute(host,command);
+                Host host = Host.of(address);
+                doExecute(host, command);
                 success = true;
                 context.setHost(host);
+                break;
             } catch (ExecuteException ex) {
-                logger.error(String.format("execute command : %s error", command), ex);
-                try {
-                    failNodeSet.add(host.getAddress());
-                    Set<String> tmpAllIps = new HashSet<>(allNodes);
-                    Collection<String> remained = CollectionUtils.subtract(tmpAllIps, failNodeSet);
-                    if (remained != null && remained.size() > 0) {
-                        host = Host.of(remained.iterator().next());
-                        logger.error("retry execute command : {} host : {}", command, host);
-                    } else {
-                        throw new ExecuteException("fail after try all nodes");
-                    }
-                } catch (Throwable t) {
-                    throw new ExecuteException("fail after try all nodes");
-                }
+                logger.error("retry execute command : {} host : {}", command, address);
             }
         }
-
+        if (!success) {
+            throw new ExecuteException("fail after try all nodes");
+        }
+        
         return success;
     }
 

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java

@@ -20,11 +20,13 @@ package org.apache.dolphinscheduler.server.worker.processor;
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.classic.sift.SiftingAppender;
 import com.alibaba.fastjson.JSONObject;
+import com.github.rholder.retry.RetryException;
 import io.netty.channel.Channel;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.common.enums.TaskType;
 import org.apache.dolphinscheduler.common.utils.OSUtils;
+import org.apache.dolphinscheduler.common.utils.RetryerUtils;
 import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
@@ -43,6 +45,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.Date;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 
 /**
@@ -101,21 +104,19 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
         taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
                 new NettyRemoteChannel(channel, command.getOpaque()));
 
+        // tell master that task is in executing
+        final Command ackCommand = buildAckCommand(taskExecutionContext).convert2Command();
+        
         try {
-            this.doAck(taskExecutionContext);
-        }catch (Exception e){
-            ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
-            this.doAck(taskExecutionContext);
+            RetryerUtils.retryCall(() -> {
+                taskCallbackService.sendAck(taskExecutionContext.getTaskInstanceId(),ackCommand);
+                return Boolean.TRUE;
+            });
+            // submit task
+            workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService));
+        } catch (ExecutionException | RetryException e) {
+            logger.error(e.getMessage(), e);
         }
-
-        // submit task
-        workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService));
-    }
-
-    private void doAck(TaskExecutionContext taskExecutionContext){
-        // tell master that task is in executing
-        TaskExecuteAckCommand ackCommand = buildAckCommand(taskExecutionContext);
-        taskCallbackService.sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand.convert2Command());
     }
 
     /**

dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java

@@ -20,6 +20,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.framework.api.ACLProvider;
+import org.apache.curator.framework.api.transaction.CuratorOp;
 import org.apache.curator.framework.state.ConnectionState;
 import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.curator.utils.CloseableUtils;
@@ -171,7 +172,11 @@ public class ZookeeperOperator implements InitializingBean {
 
     public void update(final String key, final String value) {
         try {
-            zkClient.inTransaction().check().forPath(key).and().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8)).and().commit();
+
+            CuratorOp check = zkClient.transactionOp().check().forPath(key);
+            CuratorOp setData = zkClient.transactionOp().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8));
+            zkClient.transaction().forOperations(check, setData);
+
         } catch (Exception ex) {
             logger.error("update key : {} , value : {}", key, value, ex);
         }

pom.xml

@@ -118,6 +118,7 @@
         <servlet-api.version>2.5</servlet-api.version>
         <swagger.version>1.9.3</swagger.version>
         <springfox.version>2.9.2</springfox.version>
+        <guava-retry.version>2.0.0</guava-retry.version>
     </properties>
 
     <dependencyManagement>
@@ -544,6 +545,12 @@
                 <artifactId>swagger-bootstrap-ui</artifactId>
                 <version>${swagger.version}</version>
             </dependency>
+
+            <dependency>
+                <groupId>com.github.rholder</groupId>
+                <artifactId>guava-retrying</artifactId>
+                <version>${guava-retry.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -771,6 +778,7 @@
                         <include>**/common/utils/HttpUtilsTest.java</include>
                         <include>**/common/ConstantsTest.java</include>
                         <include>**/common/utils/HadoopUtils.java</include>
+                        <include>**/common/utils/RetryerUtilsTest.java</include>
                         <include>**/common/plugin/FilePluginManagerTest</include>
                         <include>**/common/plugin/PluginClassLoaderTest</include>
                         <include>**/dao/mapper/AccessTokenMapperTest.java</include>

script/scp-hosts.sh

@@ -20,6 +20,12 @@ workDir=`dirname $0`
 workDir=`cd ${workDir};pwd`
 source $workDir/../conf/config/install_config.conf
 
+txt=""
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # Mac OSX
+    txt="''"
+fi
+
 hostsArr=(${ips//,/ })
 for host in ${hostsArr[@]}
 do
@@ -33,6 +39,11 @@ do
 
   for dsDir in bin conf lib script sql ui install.sh
   do
+    # if worker in workersGroup
+    if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then
+      sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties
+    fi
+
     echo "start to scp $dsDir to $host/$installPath"
     scp -P $sshPort -r $workDir/../$dsDir  $host:$installPath
   done

script/start-all.sh

@@ -28,8 +28,7 @@ do
 
 done
 
-workersHost=(${workers//,/ })
-for worker in ${workersHost[@]}
+for worker in ${!workersGroup[*]}
 do
   echo "$worker worker server is starting"
 

script/stop-all.sh

@@ -29,8 +29,7 @@ do
 
 done
 
-workersHost=(${workers//,/ })
-for worker in ${workersHost[@]}
+for worker in ${!workersGroup[*]}
 do
   echo "$worker worker server is stopping"
   ssh -p $sshPort $worker  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"

sql/dolphinscheduler-postgre.sql

@@ -623,6 +623,7 @@ CREATE TABLE t_ds_user (
   create_time timestamp DEFAULT NULL ,
   update_time timestamp DEFAULT NULL ,
   queue varchar(64) DEFAULT NULL ,
+  state int DEFAULT 1 ,
   PRIMARY KEY (id)
 );
 
@@ -759,4 +760,4 @@ INSERT INTO t_ds_relation_user_alertgroup(alertgroup_id,user_id,create_time,upda
 INSERT INTO t_ds_queue(queue_name,queue,create_time,update_time) VALUES ('default', 'default','2018-11-29 10:22:33', '2018-11-29 10:22:33');
 
 -- Records of t_ds_queue,default queue name : default
-INSERT INTO t_ds_version(version) VALUES ('2.0.0');
+INSERT INTO t_ds_version(version) VALUES ('2.0.0');
\ No newline at end of file

sql/dolphinscheduler_mysql.sql

@@ -775,6 +775,7 @@ CREATE TABLE `t_ds_user` (
   `create_time` datetime DEFAULT NULL COMMENT 'create time',
   `update_time` datetime DEFAULT NULL COMMENT 'update time',
   `queue` varchar(64) DEFAULT NULL COMMENT 'queue',
+  `state` int(1) DEFAULT 1 COMMENT 'state 0:disable 1:enable',
   PRIMARY KEY (`id`),
   UNIQUE KEY `user_name_unique` (`user_name`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

tools/dependencies/known-dependencies.txt

@@ -209,3 +209,4 @@ xml-apis-1.4.01.jar
 xmlenc-0.52.jar
 xz-1.0.jar
 zookeeper-3.4.14.jar
+guava-retrying-2.0.0.jar