Skip to content

DolphinScheduler
DolphinScheduler

qq_38905368 / DolphinScheduler
与 Fork 源项目一致

Fork自 apache / DolphinScheduler

通知 3
Star 0
Fork 0
DolphinScheduler
DolphinScheduler

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 72408b26 编写于 作者: B BoYiZhang 提交者： GitHub
浏览文件
操作

Merge pull request #9 from apache/dev 

update code
上级 b9f7da80 1a78c115
展开全部 隐藏空白更改
内联 并排
Showing with 641 addition and 1001 deletion
.github/workflows/ci_e2e.yml
浏览文件 @ 72408b26
......@@ -44,15 +44,14 @@ jobs:
${{ runner.os }}-maven-
- name: Build Image
run: |
export VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
sh ./dockerfile/hooks/build
sh ./docker/build/hooks/build
- name: Docker Run
run: |
VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
mkdir -p /tmp/logs
docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all
export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
sed -i "s/apache\/dolphinscheduler:latest/apache\/dolphinscheduler:${VERSION}/g" $(pwd)/docker/docker-swarm/docker-compose.yml
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d
- name: Check Server Status
run: sh ./dockerfile/hooks/check
run: sh $(pwd)/docker/docker-swarm/check
- name: Prepare e2e env
run: |
sudo apt-get install -y libxss1 libappindicator1 libindicator7 xvfb unzip libgbm1
......@@ -70,6 +69,6 @@ jobs:
uses: actions/upload-artifact@v1
with:
name: dslogs
path: /tmp/logs
path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data
.github/workflows/ci_ut.yml
浏览文件 @ 72408b26
......@@ -21,7 +21,6 @@ on:
branches:
- dev
env:
DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler
name: Unit Test
......@@ -47,7 +46,11 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
- name: Bootstrap database
run: cd ${DOCKER_DIR} && docker-compose up -d
run: |
sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml create --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql
sudo cp $(pwd)/sql/dolphinscheduler-postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}')
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
......@@ -82,6 +85,5 @@ jobs:
- name: Collect logs
run: |
mkdir -p ${LOG_DIR}
cd ${DOCKER_DIR}
docker-compose logs db > ${LOG_DIR}/db.txt
docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt
continue-on-error: true
dockerfile/Dockerfile → docker/build/Dockerfile
浏览文件 @ 72408b26
......@@ -37,32 +37,19 @@ RUN apk add openjdk8
ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk
ENV PATH $JAVA_HOME/bin:$PATH
#3. install zk
RUN cd /opt && \
wget https://downloads.apache.org/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz && \
tar -zxvf apache-zookeeper-3.5.7-bin.tar.gz && \
mv apache-zookeeper-3.5.7-bin zookeeper && \
mkdir -p /tmp/zookeeper && \
rm -rf ./zookeeper-*tar.gz && \
rm -rf /opt/zookeeper/conf/zoo_sample.cfg
ADD ./conf/zookeeper/zoo.cfg /opt/zookeeper/conf
ENV ZK_HOME /opt/zookeeper
ENV PATH $ZK_HOME/bin:$PATH
#3. add dolphinscheduler
ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
#4. install pg
RUN apk add postgresql postgresql-contrib
#5. add dolphinscheduler
ADD ./apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz /opt/
RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin/ /opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
#6. modify nginx
#5. modify nginx
RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
rm -rf /etc/nginx/conf.d/*
ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
#7. add configuration and modify permissions and set soft links
#6. add configuration and modify permissions and set soft links
ADD ./checkpoint.sh /root/checkpoint.sh
ADD ./startup-init-conf.sh /root/startup-init-conf.sh
ADD ./startup.sh /root/startup.sh
......@@ -75,22 +62,21 @@ RUN chmod +x /root/checkpoint.sh && \
chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
chmod +x /opt/dolphinscheduler/script/*.sh && \
chmod +x /opt/dolphinscheduler/bin/*.sh && \
chmod +x /opt/zookeeper/bin/*.sh && \
dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \
dos2unix /root/startup.sh && \
dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
dos2unix /opt/dolphinscheduler/script/*.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \
dos2unix /opt/zookeeper/bin/*.sh && \
rm -rf /bin/sh && \
ln -s /bin/bash /bin/sh && \
mkdir -p /tmp/xls
#8. remove apk index cache
RUN rm -rf /var/cache/apk/*
#7. remove apk index cache and disable coredup for sudo
RUN rm -rf /var/cache/apk/* && \
echo "Set disable_coredump false" >> /etc/sudo.conf
#9. expose port
#8. expose port
EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888
ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]
\ No newline at end of file
ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]
dockerfile/README.md → docker/build/README.md
浏览文件 @ 72408b26
......@@ -109,16 +109,16 @@ In Unix-Like, Example:
```bash
$ cd path/incubator-dolphinscheduler
$ sh ./dockerfile/hooks/build
$ sh ./docker/build/hooks/build
```
In Windows, Example:
```bat
c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat
c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
```
Please read `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat` script files if you don't understand
Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand
## Environment Variables
......
dockerfile/README_zh_CN.md → docker/build/README_zh_CN.md
浏览文件 @ 72408b26
......@@ -109,16 +109,16 @@ dolphinscheduler frontend
```bash
$ cd path/incubator-dolphinscheduler
$ sh ./dockerfile/hooks/build
$ sh ./docker/build/hooks/build
```
Windows系统, 如下:
```bat
c:\incubator-dolphinscheduler>.\dockerfile\hooks\build.bat
c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
```
如果你不理解这些脚本 `./dockerfile/hooks/build` `./dockerfile/hooks/build.bat`，请阅读里面的内容。
如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。
## 环境变量
......
dockerfile/hooks/build → docker/build/hooks/build
浏览文件 @ 72408b26
......@@ -24,13 +24,13 @@ printenv
if [ -z "${VERSION}" ]
then
echo "set default environment variable [VERSION]"
VERSION=$(cat $(pwd)/sql/soft_version)
export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
fi
if [ "${DOCKER_REPO}x" = "x" ]
then
echo "set default environment variable [DOCKER_REPO]"
DOCKER_REPO='dolphinscheduler'
export DOCKER_REPO='apache/dolphinscheduler'
fi
echo "Version: $VERSION"
......@@ -42,12 +42,12 @@ echo -e "Current Directory is $(pwd)\n"
echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true"
mvn -B clean compile package -Prelease -Dmaven.test.skip=true
# mv dolphinscheduler-bin.tar.gz file to dockerfile directory
echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n"
mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/
# mv dolphinscheduler-bin.tar.gz file to docker/build directory
echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n"
mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
# docker build
echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/\n"
docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/
echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n"
sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/
echo "------ dolphinscheduler end - build -------"
dockerfile/hooks/build.bat → docker/build/hooks/build.bat
浏览文件 @ 72408b26
......@@ -38,13 +38,13 @@ echo "call mvn clean compile package -Prelease"
call mvn clean compile package -Prelease -DskipTests=true
if "%errorlevel%"=="1" goto :mvnFailed
:: move dolphinscheduler-bin.tar.gz file to dockerfile directory
echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\dockerfile\"
move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\dockerfile\
:: move dolphinscheduler-bin.tar.gz file to docker/build directory
echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\"
move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\
:: docker build
echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\dockerfile\"
docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\dockerfile\
echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\"
docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\
if "%errorlevel%"=="1" goto :dockerBuildFailed
echo "------ dolphinscheduler end - build -------"
......@@ -53,4 +53,4 @@ echo "------ dolphinscheduler end - build -------"
echo "MAVEN PACKAGE FAILED!"
:dockerBuildFailed
echo "DOCKER BUILD FAILED!"
\ No newline at end of file
echo "DOCKER BUILD FAILED!"
dockerfile/startup.sh → docker/build/startup.sh
浏览文件 @ 72408b26
......@@ -24,31 +24,6 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
# start postgresql
initPostgreSQL() {
echo "checking postgresql"
if [[ "${POSTGRESQL_HOST}" = "127.0.0.1" || "${POSTGRESQL_HOST}" = "localhost" ]]; then
export PGPORT=${POSTGRESQL_PORT}
echo "start postgresql service"
rc-service postgresql restart
# role if not exists, create
flag=$(sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='${POSTGRESQL_USERNAME}'")
if [ -z "${flag}" ]; then
echo "create user"
sudo -u postgres psql -tAc "create user ${POSTGRESQL_USERNAME} with password '${POSTGRESQL_PASSWORD}'"
fi
# database if not exists, create
flag=$(sudo -u postgres psql -tAc "select 1 from pg_database where datname='dolphinscheduler'")
if [ -z "${flag}" ]; then
echo "init db"
sudo -u postgres psql -tAc "create database dolphinscheduler owner ${POSTGRESQL_USERNAME}"
fi
# grant
sudo -u postgres psql -tAc "grant all privileges on database dolphinscheduler to ${POSTGRESQL_USERNAME}"
fi
echo "test postgresql service"
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
counter=$((counter+1))
......@@ -73,24 +48,18 @@ initPostgreSQL() {
# start zk
initZK() {
echo -e "checking zookeeper"
if [[ "${ZOOKEEPER_QUORUM}" = "127.0.0.1:2181" || "${ZOOKEEPER_QUORUM}" = "localhost:2181" ]]; then
echo "start local zookeeper"
/opt/zookeeper/bin/zkServer.sh restart
else
echo "connect remote zookeeper"
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
while ! nc -z ${line%:*} ${line#*:}; do
counter=$((counter+1))
if [ $counter == 30 ]; then
echo "Error: Couldn't connect to zookeeper."
exit 1
fi
echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
sleep 5
done
echo "connect remote zookeeper"
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
while ! nc -z ${line%:*} ${line#*:}; do
counter=$((counter+1))
if [ $counter == 30 ]; then
echo "Error: Couldn't connect to zookeeper."
exit 1
fi
echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
sleep 5
done
fi
done
}
# start nginx
......
docker/docker-compose.yml 已删除 100644 → 0
浏览文件 @ b9f7da80
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: '2'
services:
zookeeper:
image: zookeeper
restart: always
container_name: zookeeper
ports:
- "2181:2181"
environment:
ZOO_MY_ID: 1
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
db:
image: postgres
container_name: postgres
environment:
- POSTGRES_USER=test
- POSTGRES_PASSWORD=test
- POSTGRES_DB=dolphinscheduler
ports:
- "5432:5432"
volumes:
- pgdata:/var/lib/postgresql/data
- ./postgres/docker-entrypoint-initdb:/docker-entrypoint-initdb.d
volumes:
pgdata:
dockerfile/hooks/check → docker/docker-swarm/check
浏览文件 @ 72408b26
......@@ -17,7 +17,7 @@
#
echo "------ dolphinscheduler check - server - status -------"
sleep 60
server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
if [ $server_num -eq 5 ]
then
echo "Server all start successfully"
......
docker/docker-swarm/docker-compose.yml
浏览文件 @ 72408b26
......@@ -30,6 +30,7 @@ services:
POSTGRESQL_DATABASE: dolphinscheduler
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
- dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
networks:
- dolphinscheduler
......@@ -41,13 +42,14 @@ services:
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
- dolphinscheduler
dolphinscheduler-api:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-api
command: ["api-server"]
ports:
......@@ -70,12 +72,12 @@ services:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- dolphinscheduler-api:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
dolphinscheduler-frontend:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-frontend
command: ["frontend"]
ports:
......@@ -93,12 +95,12 @@ services:
depends_on:
- dolphinscheduler-api
volumes:
- dolphinscheduler-frontend:/var/log/nginx
- dolphinscheduler-logs:/var/log/nginx
networks:
- dolphinscheduler
dolphinscheduler-alert:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-alert
command: ["alert-server"]
environment:
......@@ -130,18 +132,18 @@ services:
start_period: 30s
depends_on:
- dolphinscheduler-postgresql
volumes:
- dolphinscheduler-alert:/opt/dolphinscheduler/logs
networks:
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
dolphinscheduler-master:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-master
command: ["master-server"]
ports:
ports:
- 5678:5678
environment:
environment:
TZ: Asia/Shanghai
MASTER_EXEC_THREADS: "100"
MASTER_EXEC_TASK_NUM: "20"
......@@ -162,22 +164,22 @@ services:
timeout: 5s
retries: 3
start_period: 30s
depends_on:
depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- dolphinscheduler-master:/opt/dolphinscheduler/logs
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
dolphinscheduler-worker:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-worker
command: ["worker-server"]
ports:
ports:
- 1234:1234
- 50051:50051
environment:
environment:
TZ: Asia/Shanghai
WORKER_EXEC_THREADS: "100"
WORKER_HEARTBEAT_INTERVAL: "10"
......@@ -209,7 +211,7 @@ services:
source: dolphinscheduler-worker-data
target: /tmp/dolphinscheduler
- type: volume
source: dolphinscheduler-worker-logs
source: dolphinscheduler-logs
target: /opt/dolphinscheduler/logs
networks:
- dolphinscheduler
......@@ -220,13 +222,10 @@ networks:
volumes:
dolphinscheduler-postgresql:
dolphinscheduler-postgresql-initdb:
dolphinscheduler-zookeeper:
dolphinscheduler-api:
dolphinscheduler-frontend:
dolphinscheduler-alert:
dolphinscheduler-master:
dolphinscheduler-worker-data:
dolphinscheduler-worker-logs:
dolphinscheduler-logs:
configs:
dolphinscheduler-worker-task-env:
......
docker/docker-swarm/docker-stack.yml
浏览文件 @ 72408b26
......@@ -42,6 +42,7 @@ services:
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
......@@ -51,7 +52,7 @@ services:
replicas: 1
dolphinscheduler-api:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
command: ["api-server"]
ports:
- 12345:12345
......@@ -70,7 +71,7 @@ services:
retries: 3
start_period: 30s
volumes:
- dolphinscheduler-api:/opt/dolphinscheduler/logs
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
deploy:
......@@ -78,7 +79,7 @@ services:
replicas: 1
dolphinscheduler-frontend:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
command: ["frontend"]
ports:
- 8888:8888
......@@ -93,7 +94,7 @@ services:
retries: 3
start_period: 30s
volumes:
- dolphinscheduler-frontend:/var/log/nginx
- dolphinscheduler-logs:/var/log/nginx
networks:
- dolphinscheduler
deploy:
......@@ -101,7 +102,7 @@ services:
replicas: 1
dolphinscheduler-alert:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
command: ["alert-server"]
environment:
TZ: Asia/Shanghai
......@@ -131,7 +132,7 @@ services:
retries: 3
start_period: 30s
volumes:
- dolphinscheduler-alert:/opt/dolphinscheduler/logs
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
deploy:
......@@ -139,7 +140,7 @@ services:
replicas: 1
dolphinscheduler-master:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
command: ["master-server"]
ports:
- 5678:5678
......@@ -165,7 +166,7 @@ services:
retries: 3
start_period: 30s
volumes:
- dolphinscheduler-master:/opt/dolphinscheduler/logs
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
deploy:
......@@ -173,7 +174,7 @@ services:
replicas: 1
dolphinscheduler-worker:
image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
image: apache/dolphinscheduler:latest
command: ["worker-server"]
ports:
- 1234:1234
......@@ -201,7 +202,7 @@ services:
start_period: 30s
volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
configs:
- source: dolphinscheduler-worker-task-env
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
......@@ -218,12 +219,8 @@ networks:
volumes:
dolphinscheduler-postgresql:
dolphinscheduler-zookeeper:
dolphinscheduler-api:
dolphinscheduler-frontend:
dolphinscheduler-alert:
dolphinscheduler-master:
dolphinscheduler-worker-data:
dolphinscheduler-worker-logs:
dolphinscheduler-logs:
configs:
dolphinscheduler-worker-task-env:
......
docker/kubernetes/dolphinscheduler/values.yaml
浏览文件 @ 72408b26
......@@ -25,9 +25,9 @@ fullnameOverride: ""
timezone: "Asia/Shanghai"
image:
registry: "docker.io"
registry: "apache"
repository: "dolphinscheduler"
tag: "1.3.0"
tag: "latest"
pullPolicy: "IfNotPresent"
imagePullSecrets: []
......@@ -56,6 +56,8 @@ externalDatabase:
zookeeper:
enabled: true
taskQueue: "zookeeper"
config:
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
service:
port: "2181"
persistence:
......
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java
浏览文件 @ 72408b26
......@@ -56,7 +56,7 @@ public class EmailAlertPlugin implements AlertPlugin {
@Override
public String getId() {
return Constants.PLUGIN_DEFAULT_EMAIL;
return Constants.PLUGIN_DEFAULT_EMAIL_ID;
}
@Override
......
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java
浏览文件 @ 72408b26
......@@ -80,7 +80,7 @@ public class AlertSender {
alertInfo.addProp("receivers", receviersList);
AlertPlugin emailPlugin = pluginManager.findOne(Constants.PLUGIN_DEFAULT_EMAIL);
AlertPlugin emailPlugin = pluginManager.findOne(Constants.PLUGIN_DEFAULT_EMAIL_ID);
retMaps = emailPlugin.process(alertInfo);
if (retMaps == null) {
......
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java
浏览文件 @ 72408b26
......@@ -179,7 +179,7 @@ public class Constants {
*/
public static final String PLUGIN_DIR = "plugin.dir";
public static final String PLUGIN_DEFAULT_EMAIL = "email";
public static final String PLUGIN_DEFAULT_EMAIL_ID = "email";
public static final String PLUGIN_DEFAULT_EMAIL_CH = "邮件";
......
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java
浏览文件 @ 72408b26
......@@ -47,7 +47,7 @@ public class EmailAlertPluginTest {
@Test
public void getId() {
String id = plugin.getId();
assertEquals(Constants.PLUGIN_DEFAULT_EMAIL, id);
assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_ID, id);
}
@Test
......
dolphinscheduler-common/pom.xml
浏览文件 @ 72408b26
......@@ -594,5 +594,19 @@
<artifactId>janino</artifactId>
<version>${codehaus.janino.version}</version>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java
浏览文件 @ 72408b26
......@@ -90,8 +90,8 @@ public class FilePluginManager implements PluginManager {
}
@Override
public AlertPlugin findOne(String name) {
return pluginMap.get(name);
public AlertPlugin findOne(String pluginId) {
return pluginMap.get(pluginId);
}
@Override
......
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java 0 → 100644
浏览文件 @ 72408b26
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import com.github.rholder.retry.*;
import org.apache.dolphinscheduler.common.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
/**
* The Retryer util.
*/
public class RetryerUtils {
private static final Logger logger = LoggerFactory.getLogger(RetryerUtils.class);
private static Retryer<Boolean> defaultRetryerResultCheck;
private static Retryer<Boolean> defaultRetryerResultNoCheck;
private RetryerUtils() {
}
private static Retryer<Boolean> getDefaultRetryerResultNoCheck() {
if (defaultRetryerResultNoCheck == null) {
defaultRetryerResultNoCheck = RetryerBuilder
.<Boolean>newBuilder()
.retryIfException()
.withWaitStrategy(WaitStrategies.fixedWait(Constants.SLEEP_TIME_MILLIS, TimeUnit.MILLISECONDS))
.withStopStrategy(StopStrategies.stopAfterAttempt(3))
.build();
}
return defaultRetryerResultNoCheck;
}
/**
* Gets default retryer.
* the retryer will retry 3 times if exceptions throw
* and wait 1 second between each retry
*
* @param checkResult true means the callable must return true before retrying
* false means that retry callable only throw exceptions
* @return the default retryer
*/
public static Retryer<Boolean> getDefaultRetryer(boolean checkResult) {
return checkResult ? getDefaultRetryer() : getDefaultRetryerResultNoCheck();
}
/**
* Gets default retryer.
* the retryer will retry 3 times if exceptions throw
* and wait 1 second between each retry
*
* @return the default retryer
*/
public static Retryer<Boolean> getDefaultRetryer() {
if (defaultRetryerResultCheck == null) {
defaultRetryerResultCheck = RetryerBuilder
.<Boolean>newBuilder()
.retryIfResult(Boolean.FALSE::equals)
.retryIfException()
.withWaitStrategy(WaitStrategies.fixedWait(Constants.SLEEP_TIME_MILLIS, TimeUnit.MILLISECONDS))
.withStopStrategy(StopStrategies.stopAfterAttempt(3))
.build();
}
return defaultRetryerResultCheck;
}
/**
* Use RETRYER to invoke the Callable
*
* @param callable the callable
* @param checkResult true means that retry callable before returning true
* false means that retry callable only throw exceptions
* @return the final result of callable
* @throws ExecutionException the execution exception
* @throws RetryException the retry exception
*/
public static Boolean retryCall(final Callable<Boolean> callable, boolean checkResult) throws ExecutionException, RetryException {
return getDefaultRetryer(checkResult).call(callable);
}
/**
* Use RETRYER to invoke the Callable before returning true
*
* @param callable the callable
* @return the boolean
* @throws ExecutionException the execution exception
* @throws RetryException the retry exception
*/
public static Boolean retryCall(final Callable<Boolean> callable) throws ExecutionException, RetryException {
return retryCall(callable, true);
}
/**
* Retry call silent without exceptions thrown
*
* @param callable the callable
* @param checkResult whether check result
* @return if no exceptions ,it's result returned by callable ,else always false
*/
public static boolean retryCallSilent(final Callable<Boolean> callable, boolean checkResult) {
boolean result = false;
try {
result = getDefaultRetryer(checkResult).call(callable);
} catch (ExecutionException | RetryException e) {
logger.warn("Retry call {} failed {}", callable, e.getMessage(), e);
}
return result;
}
/**
* Retry call silent without exceptions thrown
*
* @param callable the callable
* @return if no exceptions ,it's result returned by callable ,else always false
*/
public static boolean retryCallSilent(final Callable<Boolean> callable) {
return retryCallSilent(callable, true);
}
}
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java 0 → 100644
浏览文件 @ 72408b26
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.Retryer;
import org.junit.Assert;
import org.junit.Test;
import java.util.concurrent.ExecutionException;
public class RetryerUtilsTest {
@Test
public void testDefaultRetryer() {
Retryer<Boolean> retryer = RetryerUtils.getDefaultRetryer();
Assert.assertNotNull(retryer);
try {
boolean result = retryer.call(() -> true);
Assert.assertTrue(result);
} catch (ExecutionException | RetryException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
Retryer<Boolean> retryer1 = RetryerUtils.getDefaultRetryer(true);
Assert.assertEquals(retryer, retryer1);
}
@Test
public void testDefaultRetryerResultCheck() {
Retryer<Boolean> retryer = RetryerUtils.getDefaultRetryer();
Assert.assertNotNull(retryer);
try {
for (int execTarget = 1; execTarget <= 3; execTarget++) {
int finalExecTarget = execTarget;
int[] execTime = {0};
boolean result = retryer.call(() -> {
execTime[0]++;
return execTime[0] == finalExecTarget;
});
Assert.assertEquals(finalExecTarget, execTime[0]);
Assert.assertTrue(result);
}
} catch (ExecutionException | RetryException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
int[] execTime = {0};
try {
retryer.call(() -> {
execTime[0]++;
return execTime[0] == 4;
});
Assert.fail("Retry times not reached");
} catch (RetryException e) {
Assert.assertEquals(3, e.getNumberOfFailedAttempts());
Assert.assertEquals(3, execTime[0]);
} catch (ExecutionException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
}
@Test
public void testDefaultRetryerResultNoCheck() {
Retryer<Boolean> retryer = RetryerUtils.getDefaultRetryer(false);
Assert.assertNotNull(retryer);
try {
for (int execTarget = 1; execTarget <= 5; execTarget++) {
int[] execTime = {0};
boolean result = retryer.call(() -> {
execTime[0]++;
return execTime[0] > 1;
});
Assert.assertEquals(1, execTime[0]);
Assert.assertFalse(result);
}
} catch (ExecutionException | RetryException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
}
@Test
public void testRecallResultCheck() {
try {
for (int execTarget = 1; execTarget <= 3; execTarget++) {
int finalExecTarget = execTarget;
int[] execTime = {0};
boolean result = RetryerUtils.retryCall(() -> {
execTime[0]++;
return execTime[0] == finalExecTarget;
});
Assert.assertEquals(finalExecTarget, execTime[0]);
Assert.assertTrue(result);
}
} catch (ExecutionException | RetryException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
int[] execTime = {0};
try {
RetryerUtils.retryCall(() -> {
execTime[0]++;
return execTime[0] == 4;
});
Assert.fail("Recall times not reached");
} catch (RetryException e) {
Assert.assertEquals(3, e.getNumberOfFailedAttempts());
Assert.assertEquals(3, execTime[0]);
} catch (ExecutionException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
}
@Test
public void testRecallResultCheckWithPara() {
try {
for (int execTarget = 1; execTarget <= 3; execTarget++) {
int finalExecTarget = execTarget;
int[] execTime = {0};
boolean result = RetryerUtils.retryCall(() -> {
execTime[0]++;
return execTime[0] == finalExecTarget;
}, true);
Assert.assertEquals(finalExecTarget, execTime[0]);
Assert.assertTrue(result);
}
} catch (ExecutionException | RetryException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
int[] execTime = {0};
try {
RetryerUtils.retryCall(() -> {
execTime[0]++;
return execTime[0] == 4;
}, true);
Assert.fail("Recall times not reached");
} catch (RetryException e) {
Assert.assertEquals(3, e.getNumberOfFailedAttempts());
Assert.assertEquals(3, execTime[0]);
} catch (ExecutionException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
}
@Test
public void testRecallResultNoCheck() {
try {
for (int execTarget = 1; execTarget <= 5; execTarget++) {
int[] execTime = {0};
boolean result = RetryerUtils.retryCall(() -> {
execTime[0]++;
return execTime[0] > 1;
}, false);
Assert.assertEquals(1, execTime[0]);
Assert.assertFalse(result);
}
} catch (ExecutionException | RetryException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
}
private void testRetryExceptionWithPara(boolean checkResult) {
try {
for (int execTarget = 1; execTarget <= 3; execTarget++) {
int finalExecTarget = execTarget;
int[] execTime = {0};
boolean result = RetryerUtils.retryCall(() -> {
execTime[0]++;
if (execTime[0] != finalExecTarget) {
throw new IllegalArgumentException(String.valueOf(execTime[0]));
}
return true;
}, checkResult);
Assert.assertEquals(finalExecTarget, execTime[0]);
Assert.assertTrue(result);
}
} catch (ExecutionException | RetryException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
int[] execTime = {0};
try {
RetryerUtils.retryCall(() -> {
execTime[0]++;
if (execTime[0] != 4) {
throw new IllegalArgumentException(String.valueOf(execTime[0]));
}
return true;
}, checkResult);
Assert.fail("Recall times not reached");
} catch (RetryException e) {
Assert.assertEquals(3, e.getNumberOfFailedAttempts());
Assert.assertEquals(3, execTime[0]);
Assert.assertNotNull(e.getCause());
Assert.assertEquals(3, Integer.parseInt(e.getCause().getMessage()));
} catch (ExecutionException e) {
Assert.fail("Retry call failed " + e.getMessage());
}
}
@Test
public void testRetryException() {
testRetryExceptionWithPara(true);
testRetryExceptionWithPara(false);
}
@Test
public void testRetrySilent() {
try {
for (int execTarget = 1; execTarget <= 3; execTarget++) {
int finalExecTarget = execTarget;
int[] execTime = {0};
boolean result = RetryerUtils.retryCallSilent(() -> {
execTime[0]++;
return execTime[0] == finalExecTarget;
});
Assert.assertEquals(finalExecTarget, execTime[0]);
Assert.assertTrue(result);
}
} catch (Exception e) {
Assert.fail("Unexpected exception " + e.getMessage());
}
int[] execTime = {0};
try {
boolean result = RetryerUtils.retryCallSilent(() -> {
execTime[0]++;
return execTime[0] == 4;
});
Assert.assertFalse(result);
} catch (Exception e) {
Assert.fail("Unexpected exception " + e.getMessage());
}
}
@Test
public void testRetrySilentWithPara() {
try {
for (int execTarget = 1; execTarget <= 3; execTarget++) {
int finalExecTarget = execTarget;
int[] execTime = {0};
boolean result = RetryerUtils.retryCallSilent(() -> {
execTime[0]++;
return execTime[0] == finalExecTarget;
}, true);
Assert.assertEquals(finalExecTarget, execTime[0]);
Assert.assertTrue(result);
}
} catch (Exception e) {
Assert.fail("Unexpected exception " + e.getMessage());
}
int[] execTime = {0};
try {
boolean result = RetryerUtils.retryCallSilent(() -> {
execTime[0]++;
return execTime[0] == 4;
}, true);
Assert.assertFalse(result);
} catch (Exception e) {
Assert.fail("Unexpected exception " + e.getMessage());
}
}
@Test
public void testRetrySilentNoCheckResult(){
try {
for (int execTarget = 1; execTarget <= 5; execTarget++) {
int[] execTime = {0};
boolean result = RetryerUtils.retryCallSilent(() -> {
execTime[0]++;
return execTime[0] > 1;
}, false);
Assert.assertEquals(1, execTime[0]);
Assert.assertFalse(result);
}
} catch (Exception e) {
Assert.fail("Unexpected exception " + e.getMessage());
}
}
private void testRetrySilentExceptionWithPara(boolean checkResult) {
try {
for (int execTarget = 1; execTarget <= 3; execTarget++) {
int finalExecTarget = execTarget;
int[] execTime = {0};
boolean result = RetryerUtils.retryCallSilent(() -> {
execTime[0]++;
if (execTime[0] != finalExecTarget) {
throw new IllegalArgumentException(String.valueOf(execTime[0]));
}
return true;
}, checkResult);
Assert.assertEquals(finalExecTarget, execTime[0]);
Assert.assertTrue(result);
}
} catch (Exception e) {
Assert.fail("Unexpected exception " + e.getMessage());
}
int[] execTime = {0};
try {
boolean result = RetryerUtils.retryCallSilent(() -> {
execTime[0]++;
if (execTime[0] != 4) {
throw new IllegalArgumentException(String.valueOf(execTime[0]));
}
return true;
}, checkResult);
Assert.assertFalse(result);
} catch (Exception e) {
Assert.fail("Unexpected exception " + e.getMessage());
}
}
@Test
public void testRetrySilentException() {
testRetrySilentExceptionWithPara(true);
testRetrySilentExceptionWithPara(false);
}
}
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
浏览文件 @ 72408b26
......@@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.server.master.dispatch.executor;
import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
......@@ -36,10 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.*;
/**
* netty executor manager
......@@ -87,17 +83,11 @@ public class NettyExecutorManager extends AbstractExecutorManager<Boolean>{
*/
@Override
public Boolean execute(ExecutionContext context) throws ExecuteException {
/**
* all nodes
*/
Set<String> allNodes = getAllNodes(context);
/**
* fail nodes
*/
Set<String> failNodeSet = new HashSet<>();
LinkedList<String> allNodes = new LinkedList<>();
Set<String> nodes = getAllNodes(context);
if (nodes != null) {
allNodes.addAll(nodes);
}
/**
* build command accord executeContext
*/
......@@ -106,31 +96,27 @@ public class NettyExecutorManager extends AbstractExecutorManager<Boolean>{
/**
* execute task host
*/
Host host = context.getHost();
String startHostAddress = context.getHost().getAddress();
// remove start host address and add it to head
allNodes.remove(startHostAddress);
allNodes.addFirst(startHostAddress);
boolean success = false;
while (!success) {
for (String address : allNodes) {
try {
doExecute(host,command);
Host host = Host.of(address);
doExecute(host, command);
success = true;
context.setHost(host);
break;
} catch (ExecuteException ex) {
logger.error(String.format("execute command : %s error", command), ex);
try {
failNodeSet.add(host.getAddress());
Set<String> tmpAllIps = new HashSet<>(allNodes);
Collection<String> remained = CollectionUtils.subtract(tmpAllIps, failNodeSet);
if (remained != null && remained.size() > 0) {
host = Host.of(remained.iterator().next());
logger.error("retry execute command : {} host : {}", command, host);
} else {
throw new ExecuteException("fail after try all nodes");
}
} catch (Throwable t) {
throw new ExecuteException("fail after try all nodes");
}
logger.error("retry execute command : {} host : {}", command, address);
}
}
if (!success) {
throw new ExecuteException("fail after try all nodes");
}
return success;
}
......
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
浏览文件 @ 72408b26
......@@ -20,11 +20,13 @@ package org.apache.dolphinscheduler.server.worker.processor;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.sift.SiftingAppender;
import com.alibaba.fastjson.JSONObject;
import com.github.rholder.retry.RetryException;
import io.netty.channel.Channel;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.RetryerUtils;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
......@@ -43,6 +45,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Date;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
/**
......@@ -101,21 +104,19 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
new NettyRemoteChannel(channel, command.getOpaque()));
// tell master that task is in executing
final Command ackCommand = buildAckCommand(taskExecutionContext).convert2Command();
try {
this.doAck(taskExecutionContext);
}catch (Exception e){
ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
this.doAck(taskExecutionContext);
RetryerUtils.retryCall(() -> {
taskCallbackService.sendAck(taskExecutionContext.getTaskInstanceId(),ackCommand);
return Boolean.TRUE;
});
// submit task
workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService));
} catch (ExecutionException | RetryException e) {
logger.error(e.getMessage(), e);
}
// submit task
workerExecService.submit(new TaskExecuteThread(taskExecutionContext, taskCallbackService));
}
private void doAck(TaskExecutionContext taskExecutionContext){
// tell master that task is in executing
TaskExecuteAckCommand ackCommand = buildAckCommand(taskExecutionContext);
taskCallbackService.sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand.convert2Command());
}
/**
......
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperOperator.java
浏览文件 @ 72408b26
......@@ -20,6 +20,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.api.ACLProvider;
import org.apache.curator.framework.api.transaction.CuratorOp;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.utils.CloseableUtils;
......@@ -171,7 +172,11 @@ public class ZookeeperOperator implements InitializingBean {
public void update(final String key, final String value) {
try {
zkClient.inTransaction().check().forPath(key).and().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8)).and().commit();
CuratorOp check = zkClient.transactionOp().check().forPath(key);
CuratorOp setData = zkClient.transactionOp().setData().forPath(key, value.getBytes(StandardCharsets.UTF_8));
zkClient.transaction().forOperations(check, setData);
} catch (Exception ex) {
logger.error("update key : {} , value : {}", key, value, ex);
}
......
pom.xml
浏览文件 @ 72408b26
......@@ -118,6 +118,7 @@
<servlet-api.version>2.5</servlet-api.version>
<swagger.version>1.9.3</swagger.version>
<springfox.version>2.9.2</springfox.version>
<guava-retry.version>2.0.0</guava-retry.version>
</properties>
<dependencyManagement>
......@@ -544,6 +545,12 @@
<artifactId>swagger-bootstrap-ui</artifactId>
<version>${swagger.version}</version>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<version>${guava-retry.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
......@@ -771,6 +778,7 @@
<include>**/common/utils/HttpUtilsTest.java</include>
<include>**/common/ConstantsTest.java</include>
<include>**/common/utils/HadoopUtils.java</include>
<include>**/common/utils/RetryerUtilsTest.java</include>
<include>**/common/plugin/FilePluginManagerTest</include>
<include>**/common/plugin/PluginClassLoaderTest</include>
<include>**/dao/mapper/AccessTokenMapperTest.java</include>
......
script/scp-hosts.sh
浏览文件 @ 72408b26
......@@ -20,6 +20,12 @@ workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
source $workDir/../conf/config/install_config.conf
txt=""
if [[ "$OSTYPE" == "darwin"* ]]; then
# Mac OSX
txt="''"
fi
hostsArr=(${ips//,/ })
for host in ${hostsArr[@]}
do
......@@ -33,6 +39,11 @@ do
for dsDir in bin conf lib script sql ui install.sh
do
# if worker in workersGroup
if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then
sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties
fi
echo "start to scp $dsDir to $host/$installPath"
scp -P $sshPort -r $workDir/../$dsDir $host:$installPath
done
......
script/start-all.sh
浏览文件 @ 72408b26
......@@ -28,8 +28,7 @@ do
done
workersHost=(${workers//,/ })
for worker in ${workersHost[@]}
for worker in ${!workersGroup[*]}
do
echo "$worker worker server is starting"
......
script/stop-all.sh
浏览文件 @ 72408b26
......@@ -29,8 +29,7 @@ do
done
workersHost=(${workers//,/ })
for worker in ${workersHost[@]}
for worker in ${!workersGroup[*]}
do
echo "$worker worker server is stopping"
ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"
......
sql/dolphinscheduler-postgre.sql
浏览文件 @ 72408b26
......@@ -623,6 +623,7 @@ CREATE TABLE t_ds_user (
create_time timestamp DEFAULT NULL ,
update_time timestamp DEFAULT NULL ,
queue varchar(64) DEFAULT NULL ,
state int DEFAULT 1 ,
PRIMARY KEY (id)
);
......@@ -759,4 +760,4 @@ INSERT INTO t_ds_relation_user_alertgroup(alertgroup_id,user_id,create_time,upda
INSERT INTO t_ds_queue(queue_name,queue,create_time,update_time) VALUES ('default', 'default','2018-11-29 10:22:33', '2018-11-29 10:22:33');
-- Records of t_ds_queue,default queue name : default
INSERT INTO t_ds_version(version) VALUES ('2.0.0');
\ No newline at end of file
INSERT INTO t_ds_version(version) VALUES ('2.0.0');
sql/dolphinscheduler_mysql.sql
浏览文件 @ 72408b26
......@@ -775,6 +775,7 @@ CREATE TABLE `t_ds_user` (
`create_time` datetime DEFAULT NULL COMMENT 'create time',
`update_time` datetime DEFAULT NULL COMMENT 'update time',
`queue` varchar(64) DEFAULT NULL COMMENT 'queue',
`state` int(1) DEFAULT 1 COMMENT 'state 0:disable 1:enable',
PRIMARY KEY (`id`),
UNIQUE KEY `user_name_unique` (`user_name`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
......
tools/dependencies/known-dependencies.txt
浏览文件 @ 72408b26
......@@ -209,3 +209,4 @@ xml-apis-1.4.01.jar
xmlenc-0.52.jar
xz-1.0.jar
zookeeper-3.4.14.jar
guava-retrying-2.0.0.jar
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或