From b537f4eb2617f376094e918bbc598c75e37a8013 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Mon, 24 May 2021 00:03:35 +0800 Subject: [PATCH] [Improvement][Docker/K8s] Support more configurations and service accesses, reduce the duplication configurations, improve image for python and the compatibility of python home (#5393) * [Improvement][Docker] Support more configurations and improve image for python * [Improvement][K8s] Support more configurations, more service accesses, and reduce the duplication configurations * [Improvement][Python] Improve the compatibility of python home * [Improvement][Install] Fix install config and install.sh * [Improvement][Install] Fix workflow ut * [Improvement][Docker] Optimize docker-swarm/check script * [Improvement][DB] Update default username and password of database * Update comments in master/worker.properties * Specify version for all images * [Improvement][Docker] Optimize PS1 and WORKDIR * [Improvement][SQL] Reduce heap size to 64m in create-dolphinscheduler.sh and upgrade-dolphinscheduler.sh * [Fix-5431][K8s] Fix master and worker cannot get the right address with custom DNS --- .github/workflows/ci_ut.yml | 7 +- docker/README.md | 12 +- docker/build/Dockerfile | 41 +- docker/build/README.md | 488 ------------------ docker/build/README_zh_CN.md | 488 ------------------ .../application-api.properties.tpl | 11 +- .../dolphinscheduler/common.properties.tpl | 66 ++- .../datasource.properties.tpl | 8 +- .../dolphinscheduler/master.properties.tpl | 25 +- .../dolphinscheduler/worker.properties.tpl | 18 +- docker/build/startup-init-conf.sh | 83 ++- docker/build/startup.sh | 51 +- docker/docker-swarm/check | 43 +- docker/docker-swarm/config.env.sh | 122 +++++ docker/docker-swarm/docker-compose.yml | 110 +--- docker/docker-swarm/docker-stack.yml | 107 +--- docker/kubernetes/dolphinscheduler/Chart.yaml | 8 +- docker/kubernetes/dolphinscheduler/README.md | 364 ------------- .../dolphinscheduler/templates/NOTES.txt | 28 +- .../dolphinscheduler/templates/_helpers.tpl | 178 ++++++- .../configmap-dolphinscheduler-alert.yaml | 8 +- .../configmap-dolphinscheduler-api.yaml | 7 +- .../configmap-dolphinscheduler-common.yaml | 14 +- .../configmap-dolphinscheduler-master.yaml | 15 +- .../configmap-dolphinscheduler-worker.yaml | 14 +- .../deployment-dolphinscheduler-alert.yaml | 109 +--- .../deployment-dolphinscheduler-api.yaml | 165 +----- .../dolphinscheduler/templates/ingress.yaml | 3 +- .../templates/pvc-dolphinscheduler-alert.yaml | 3 +- .../templates/pvc-dolphinscheduler-api.yaml | 3 +- .../pvc-dolphinscheduler-fs-file.yaml | 3 +- .../pvc-dolphinscheduler-shared.yaml | 36 ++ .../templates/secret-external-database.yaml | 3 +- .../templates/secret-external-fs-s3a.yaml | 5 +- .../statefulset-dolphinscheduler-master.yaml | 178 +------ .../statefulset-dolphinscheduler-worker.yaml | 229 ++------ .../templates/svc-dolphinscheduler-alert.yaml | 8 +- .../templates/svc-dolphinscheduler-api.yaml | 31 +- .../svc-dolphinscheduler-master-headless.yaml | 10 +- .../svc-dolphinscheduler-worker-headless.yaml | 10 +- .../kubernetes/dolphinscheduler/values.yaml | 99 ++-- .../alert/utils/Constants.java | 2 +- .../src/main/resources/alert.properties | 2 +- .../main/resources/application-api.properties | 2 +- .../dolphinscheduler/common/Constants.java | 34 +- .../common/utils/NetUtils.java | 10 +- .../src/main/resources/common.properties | 56 +- .../common/utils/NetUtilsTest.java | 15 +- .../src/main/resources/datasource.properties | 10 +- .../server/master/config/MasterConfig.java | 22 +- .../server/worker/config/WorkerConfig.java | 6 +- .../worker/task/PythonCommandExecutor.java | 64 ++- .../main/resources/config/install_config.conf | 11 +- .../src/main/resources/master.properties | 23 +- .../src/main/resources/worker.properties | 20 +- .../server/worker/EnvFileTest.java | 82 --- .../task/PythonCommandExecutorTest.java | 58 +++ install.sh | 8 +- pom.xml | 3 +- script/create-dolphinscheduler.sh | 2 +- script/dolphinscheduler-daemon.sh | 40 +- script/upgrade-dolphinscheduler.sh | 2 +- 62 files changed, 1059 insertions(+), 2624 deletions(-) delete mode 100644 docker/build/README.md delete mode 100644 docker/build/README_zh_CN.md create mode 100755 docker/docker-swarm/config.env.sh delete mode 100644 docker/kubernetes/dolphinscheduler/README.md create mode 100644 docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml delete mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 5512d2f8b..54ce79c6e 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -47,10 +47,11 @@ jobs: ${{ runner.os }}-maven- - name: Bootstrap database run: | - sed -i "s/: root/: test/g" $(pwd)/docker/docker-swarm/docker-compose.yml - docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up --no-start --force-recreate dolphinscheduler-zookeeper dolphinscheduler-postgresql - sudo cp $(pwd)/sql/dolphinscheduler_postgre.sql $(docker volume inspect docker-swarm_dolphinscheduler-postgresql-initdb | grep "Mountpoint" | awk -F "\"" '{print $4}') + sed -i "/image: bitnami\/postgresql/a\ ports:\n - 5432:5432" $(pwd)/docker/docker-swarm/docker-compose.yml + sed -i "/image: bitnami\/zookeeper/a\ ports:\n - 2181:2181" $(pwd)/docker/docker-swarm/docker-compose.yml docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d dolphinscheduler-zookeeper dolphinscheduler-postgresql + until docker logs docker-swarm_dolphinscheduler-postgresql_1 2>&1 | grep 'listening on IPv4 address'; do echo "waiting for postgresql ready ..."; sleep 1; done + docker run --rm --network docker-swarm_dolphinscheduler -v $(pwd)/sql/dolphinscheduler_postgre.sql:/docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql bitnami/postgresql:latest bash -c "PGPASSWORD=root psql -h docker-swarm_dolphinscheduler-postgresql_1 -U root -d dolphinscheduler -v ON_ERROR_STOP=1 -f /docker-entrypoint-initdb.d/dolphinscheduler_postgre.sql" - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/docker/README.md b/docker/README.md index 05c74f3a9..dfa6198c0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1 +1,11 @@ -# DolphinScheduler for Docker +# DolphinScheduler for Docker and Kubernetes + +### QuickStart in Docker + +[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/docker-deployment.html) +[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/docker-deployment.html) + +### QuickStart in Kubernetes + +[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/kubernetes-deployment.html) +[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/kubernetes-deployment.html) diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index d3c040355..65ab79cbe 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -15,28 +15,35 @@ # limitations under the License. # -FROM openjdk:8-jdk-alpine +FROM openjdk:8-jre-slim-buster ARG VERSION +ARG DEBIAN_FRONTEND=noninteractive ENV TZ Asia/Shanghai ENV LANG C.UTF-8 ENV DOCKER true +ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler # 1. install command/library/software -# If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: -# RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories -# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories -RUN apk update && \ - apk add --no-cache tzdata dos2unix bash python2 python3 supervisor procps sudo shadow tini postgresql-client && \ - cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ - apk del tzdata && \ - rm -rf /var/cache/apk/* +# If install slowly, you can replcae debian's mirror with new mirror, Example: +# RUN { \ +# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster main contrib non-free"; \ +# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-updates main contrib non-free"; \ +# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ buster-backports main contrib non-free"; \ +# echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security buster/updates main contrib non-free"; \ +# } > /etc/apt/sources.list +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata dos2unix python supervisor procps psmisc netcat sudo tini && \ + echo "Asia/Shanghai" > /etc/timezone && \ + rm -f /etc/localtime && \ + dpkg-reconfigure tzdata && \ + rm -rf /var/lib/apt/lists/* /tmp/* # 2. add dolphinscheduler ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/ -RUN ln -s /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler +RUN ln -s -r /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler +WORKDIR /opt/apache-dolphinscheduler-${VERSION}-bin # 3. add configuration and modify permissions and set soft links COPY ./checkpoint.sh /root/checkpoint.sh @@ -44,19 +51,21 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh COPY ./startup.sh /root/startup.sh COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ -COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor.d/ +COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor/conf.d/ COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/ -RUN dos2unix /root/checkpoint.sh && \ +RUN sed -i 's/*.conf$/*.ini/' /etc/supervisor/supervisord.conf && \ + dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ - rm -rf /bin/sh && \ + rm -f /bin/sh && \ ln -s /bin/bash /bin/sh && \ - mkdir -p /var/mail /tmp/xls && \ + mkdir -p /tmp/xls && \ + echo PS1=\'\\w \\$ \' >> ~/.bashrc && \ echo "Set disable_coredump false" >> /etc/sudo.conf # 4. expose port EXPOSE 5678 1234 12345 50051 50052 -ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] +ENTRYPOINT ["/usr/bin/tini", "--", "/root/startup.sh"] diff --git a/docker/build/README.md b/docker/build/README.md deleted file mode 100644 index d9eaf0e60..000000000 --- a/docker/build/README.md +++ /dev/null @@ -1,488 +0,0 @@ -## What is DolphinScheduler? - -DolphinScheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. - -GitHub URL: https://github.com/apache/dolphinscheduler - -Official Website: https://dolphinscheduler.apache.org - -![DolphinScheduler](https://dolphinscheduler.apache.org/img/hlogo_colorful.svg) - -[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) -[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) - -## Prerequisites - -- [Docker](https://docs.docker.com/engine/) 1.13.1+ -- [Docker Compose](https://docs.docker.com/compose/) 1.11.0+ - -## How to use this docker image - -#### You can start a dolphinscheduler by docker-compose (recommended) - -``` -$ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d -``` - -The default **postgres** user `root`, postgres password `root` and database `dolphinscheduler` are created in the `docker-compose.yml`. - -The default **zookeeper** is created in the `docker-compose.yml`. - -Access the Web UI: http://192.168.xx.xx:12345/dolphinscheduler - -The default username is `admin` and the default password is `dolphinscheduler123` - -> **Tip**: For quick start in docker, you can create a tenant named `ds` and associate the user `admin` with the tenant `ds` - -#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`** - -You can specify **existing postgres and zookeeper service**. Example: - -``` -$ docker run -d --name dolphinscheduler \ --e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --p 12345:12345 \ -apache/dolphinscheduler:latest all -``` - -Access the Web UI：http://192.168.xx.xx:12345/dolphinscheduler - -#### Or start a standalone dolphinscheduler server - -You can start a standalone dolphinscheduler server. - -* Create a **local volume** for resource storage, For example: - -``` -docker volume create dolphinscheduler-resource-local -``` - -* Start a **master server**, For example: - -``` -$ docker run -d --name dolphinscheduler-master \ --e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -apache/dolphinscheduler:latest master-server -``` - -* Start a **worker server** (including **logger server**), For example: - -``` -$ docker run -d --name dolphinscheduler-worker \ --e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --e ALERT_LISTEN_HOST="dolphinscheduler-alert" \ --v dolphinscheduler-resource-local:/dolphinscheduler \ -apache/dolphinscheduler:latest worker-server -``` - -* Start a **api server**, For example: - -``` -$ docker run -d --name dolphinscheduler-api \ --e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --v dolphinscheduler-resource-local:/dolphinscheduler \ --p 12345:12345 \ -apache/dolphinscheduler:latest api-server -``` - -* Start a **alert server**, For example: - -``` -$ docker run -d --name dolphinscheduler-alert \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -apache/dolphinscheduler:latest alert-server -``` - -**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. - -## How to build a docker image - -You can build a docker image in A Unix-like operating system, You can also build it in Windows operating system. - -In Unix-Like, Example: - -```bash -$ cd path/dolphinscheduler -$ sh ./docker/build/hooks/build -``` - -In Windows, Example: - -```bat -C:\dolphinscheduler>.\docker\build\hooks\build.bat -``` - -Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand - -## Environment Variables - -The DolphinScheduler Docker container is configured through environment variables, and the default value will be used if an environment variable is not set. - -**`DATABASE_TYPE`** - -This environment variable sets the type for database. The default value is `postgresql`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. - -**`DATABASE_DRIVER`** - -This environment variable sets the type for database. The default value is `org.postgresql.Driver`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. - -**`DATABASE_HOST`** - -This environment variable sets the host for database. The default value is `127.0.0.1`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. - -**`DATABASE_PORT`** - -This environment variable sets the port for database. The default value is `5432`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. - -**`DATABASE_USERNAME`** - -This environment variable sets the username for database. The default value is `root`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. - -**`DATABASE_PASSWORD`** - -This environment variable sets the password for database. The default value is `root`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. - -**`DATABASE_DATABASE`** - -This environment variable sets the database for database. The default value is `dolphinscheduler`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. - -**`DATABASE_PARAMS`** - -This environment variable sets the database for database. The default value is `characterEncoding=utf8`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. - -**`HADOOP_HOME`** - -This environment variable sets `HADOOP_HOME`. The default value is `/opt/soft/hadoop`. - -**`HADOOP_CONF_DIR`** - -This environment variable sets `HADOOP_CONF_DIR`. The default value is `/opt/soft/hadoop/etc/hadoop`. - -**`SPARK_HOME1`** - -This environment variable sets `SPARK_HOME1`. The default value is `/opt/soft/spark1`. - -**`SPARK_HOME2`** - -This environment variable sets `SPARK_HOME2`. The default value is `/opt/soft/spark2`. - -**`PYTHON_HOME`** - -This environment variable sets `PYTHON_HOME`. The default value is `/usr`. - -**`JAVA_HOME`** - -This environment variable sets `JAVA_HOME`. The default value is `/usr/lib/jvm/java-1.8-openjdk`. - -**`HIVE_HOME`** - -This environment variable sets `HIVE_HOME`. The default value is `/opt/soft/hive`. - -**`FLINK_HOME`** - -This environment variable sets `FLINK_HOME`. The default value is `/opt/soft/flink`. - -**`DATAX_HOME`** - -This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax`. - -**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** - -User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler` - -**`DOLPHINSCHEDULER_OPTS`** - -This environment variable sets java options. The default value is empty. - -**`RESOURCE_STORAGE_TYPE`** - -This environment variable sets resource storage type for dolphinscheduler like `HDFS`, `S3`, `NONE`. The default value is `HDFS`. - -**`RESOURCE_UPLOAD_PATH`** - -This environment variable sets resource store path on HDFS/S3 for resource storage. The default value is `/dolphinscheduler`. - -**`FS_DEFAULT_FS`** - -This environment variable sets fs.defaultFS for resource storage like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`. The default value is `file:///`. - -**`FS_S3A_ENDPOINT`** - -This environment variable sets s3 endpoint for resource storage. The default value is `s3.xxx.amazonaws.com`. - -**`FS_S3A_ACCESS_KEY`** - -This environment variable sets s3 access key for resource storage. The default value is `xxxxxxx`. - -**`FS_S3A_SECRET_KEY`** - -This environment variable sets s3 secret key for resource storage. The default value is `xxxxxxx`. - -**`ZOOKEEPER_QUORUM`** - -This environment variable sets zookeeper quorum for `master-server` and `worker-serverr`. The default value is `127.0.0.1:2181`. - -**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`. - -**`ZOOKEEPER_ROOT`** - -This environment variable sets zookeeper root directory for dolphinscheduler. The default value is `/dolphinscheduler`. - -**`MASTER_EXEC_THREADS`** - -This environment variable sets exec thread num for `master-server`. The default value is `100`. - -**`MASTER_EXEC_TASK_NUM`** - -This environment variable sets exec task num for `master-server`. The default value is `20`. - -**`MASTER_HEARTBEAT_INTERVAL`** - -This environment variable sets heartbeat interval for `master-server`. The default value is `10`. - -**`MASTER_TASK_COMMIT_RETRYTIMES`** - -This environment variable sets task commit retry times for `master-server`. The default value is `5`. - -**`MASTER_TASK_COMMIT_INTERVAL`** - -This environment variable sets task commit interval for `master-server`. The default value is `1000`. - -**`MASTER_MAX_CPULOAD_AVG`** - -This environment variable sets max cpu load avg for `master-server`. The default value is `100`. - -**`MASTER_RESERVED_MEMORY`** - -This environment variable sets reserved memory for `master-server`. The default value is `0.1`. - -**`MASTER_LISTEN_PORT`** - -This environment variable sets port for `master-server`. The default value is `5678`. - -**`WORKER_EXEC_THREADS`** - -This environment variable sets exec thread num for `worker-server`. The default value is `100`. - -**`WORKER_HEARTBEAT_INTERVAL`** - -This environment variable sets heartbeat interval for `worker-server`. The default value is `10`. - -**`WORKER_MAX_CPULOAD_AVG`** - -This environment variable sets max cpu load avg for `worker-server`. The default value is `100`. - -**`WORKER_RESERVED_MEMORY`** - -This environment variable sets reserved memory for `worker-server`. The default value is `0.1`. - -**`WORKER_LISTEN_PORT`** - -This environment variable sets port for `worker-server`. The default value is `1234`. - -**`WORKER_GROUPS`** - -This environment variable sets groups for `worker-server`. The default value is `default`. - -**`WORKER_HOST_WEIGHT`** - -This environment variable sets weight for `worker-server`. The default value is `100`. - -**`ALERT_LISTEN_HOST`** - -This environment variable sets the host of `alert-server` for `worker-server`. The default value is `127.0.0.1`. - -**`ALERT_PLUGIN_DIR`** - -This environment variable sets the alert plugin directory for `alert-server`. The default value is `lib/plugin/alert`. - -## Initialization scripts - -If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`. - -For example, to add an environment variable `API_SERVER_PORT` in `/root/start-init-conf.sh`: - -``` -export API_SERVER_PORT=5555 -``` - -and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port: -``` -server.port=${API_SERVER_PORT} -``` - -`/root/start-init-conf.sh` will dynamically generate config file: - -```sh -echo "generate dolphinscheduler config" -ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do -eval "cat << EOF -$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) -EOF -" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} -done -``` - -## FAQ - -### How to stop dolphinscheduler by docker-compose? - -Stop containers: - -``` -docker-compose stop -``` - -Stop containers and remove containers, networks and volumes: - -``` -docker-compose down -v -``` - -### How to deploy dolphinscheduler on Docker Swarm? - -Assuming that the Docker Swarm cluster has been created (If there is no Docker Swarm cluster, please refer to [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/)) - -Start a stack named dolphinscheduler - -``` -docker stack deploy -c docker-stack.yml dolphinscheduler -``` - -Stop and remove the stack named dolphinscheduler - -``` -docker stack rm dolphinscheduler -``` - -### How to use MySQL as the DolphinScheduler's database instead of PostgreSQL? - -> Because of the commercial license, we cannot directly use the driver and client of MySQL. -> -> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image as follows. - -1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`) - -2. Create a new `Dockerfile` to add MySQL driver and client: - -``` -FROM apache/dolphinscheduler:latest -COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib -RUN apk add --update --no-cache mysql-client -``` - -3. Build a new docker image including MySQL driver and client: - -``` -docker build -t apache/dolphinscheduler:mysql . -``` - -4. Modify all `image` fields to `apache/dolphinscheduler:mysql` in `docker-compose.yml` - -> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml` - -5. Comment the `dolphinscheduler-postgresql` block in `docker-compose.yml` - -6. Add `dolphinscheduler-mysql` service in `docker-compose.yml` (**Optional**, you can directly use a external MySQL database) - -7. Modify all DATABASE environments in `docker-compose.yml` - -``` -DATABASE_TYPE: mysql -DATABASE_DRIVER: com.mysql.jdbc.Driver -DATABASE_HOST: dolphinscheduler-mysql -DATABASE_PORT: 3306 -DATABASE_USERNAME: root -DATABASE_PASSWORD: root -DATABASE_DATABASE: dolphinscheduler -DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8 -``` - -> If you have added `dolphinscheduler-mysql` service in `docker-compose.yml`, just set `DATABASE_HOST` to `dolphinscheduler-mysql` - -8. Run a dolphinscheduler (See **How to use this docker image**) - -### How to support MySQL datasource in `Datasource manage`? - -> Because of the commercial license, we cannot directly use the driver of MySQL. -> -> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows. - -1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`) - -2. Create a new `Dockerfile` to add MySQL driver: - -``` -FROM apache/dolphinscheduler:latest -COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib -``` - -3. Build a new docker image including MySQL driver: - -``` -docker build -t apache/dolphinscheduler:mysql-driver . -``` - -4. Modify all `image` fields to `apache/dolphinscheduler:mysql-driver` in `docker-compose.yml` - -> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml` - -5. Run a dolphinscheduler (See **How to use this docker image**) - -6. Add a MySQL datasource in `Datasource manage` - -### How to support Oracle datasource in `Datasource manage`? - -> Because of the commercial license, we cannot directly use the driver of Oracle. -> -> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows. - -1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`) - -2. Create a new `Dockerfile` to add Oracle driver: - -``` -FROM apache/dolphinscheduler:latest -COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib -``` - -3. Build a new docker image including Oracle driver: - -``` -docker build -t apache/dolphinscheduler:oracle-driver . -``` - -4. Modify all `image` fields to `apache/dolphinscheduler:oracle-driver` in `docker-compose.yml` - -> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml` - -5. Run a dolphinscheduler (See **How to use this docker image**) - -6. Add a Oracle datasource in `Datasource manage` - -For more information please refer to the [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) documentation. diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md deleted file mode 100644 index af835d483..000000000 --- a/docker/build/README_zh_CN.md +++ /dev/null @@ -1,488 +0,0 @@ -## DolphinScheduler是什么? - -一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系，使调度系统在数据处理流程中`开箱即用`。 - -GitHub URL: https://github.com/apache/dolphinscheduler - -Official Website: https://dolphinscheduler.apache.org - -![DolphinScheduler](https://dolphinscheduler.apache.org/img/hlogo_colorful.svg) - -[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) -[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) - -## 先决条件 - -- [Docker](https://docs.docker.com/engine/) 1.13.1+ -- [Docker Compose](https://docs.docker.com/compose/) 1.11.0+ - -## 如何使用docker镜像 - -#### 以 docker-compose 的方式启动dolphinscheduler(推荐) - -``` -$ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d -``` - -在`docker-compose.yml`文件中，默认的创建`Postgres`的用户、密码和数据库，默认值分别为：`root`、`root`、`dolphinscheduler`。 - -同时，默认的`Zookeeper`也会在`docker-compose.yml`文件中被创建。 - -访问前端页面：http://192.168.xx.xx:12345/dolphinscheduler - -默认的用户是`admin`，默认的密码是`dolphinscheduler123` - -> **提示**: 为了在docker中快速开始，你可以创建一个名为`ds`的租户，并将这个租户`ds`关联到用户`admin` - -#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务 - -你可以指定已经存在的 **`Postgres`** 和 **`Zookeeper`** 服务. 如下: - -``` -$ docker run -d --name dolphinscheduler \ --e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --p 12345:12345 \ -apache/dolphinscheduler:latest all -``` - -访问前端页面：http://192.168.xx.xx:12345/dolphinscheduler - -#### 或者运行dolphinscheduler中的部分服务 - -你能够运行dolphinscheduler中的部分服务。 - -* 创建一个 **本地卷** 用于资源存储，如下: - -``` -docker volume create dolphinscheduler-resource-local -``` - -* 启动一个 **master server**, 如下: - -``` -$ docker run -d --name dolphinscheduler-master \ --e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -apache/dolphinscheduler:latest master-server -``` - -* 启动一个 **worker server** (包括 **logger server**), 如下: - -``` -$ docker run -d --name dolphinscheduler-worker \ --e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --e ALERT_LISTEN_HOST="dolphinscheduler-alert" \ --v dolphinscheduler-resource-local:/dolphinscheduler \ -apache/dolphinscheduler:latest worker-server -``` - -* 启动一个 **api server**, 如下: - -``` -$ docker run -d --name dolphinscheduler-api \ --e ZOOKEEPER_QUORUM="192.168.x.x:2181" \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ --v dolphinscheduler-resource-local:/dolphinscheduler \ --p 12345:12345 \ -apache/dolphinscheduler:latest api-server -``` - -* 启动一个 **alert server**, 如下: - -``` -$ docker run -d --name dolphinscheduler-alert \ --e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ --e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -apache/dolphinscheduler:latest alert-server -``` - -**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。 - -## 如何构建一个docker镜像 - -你能够在类Unix系统和Windows系统中构建一个docker镜像。 - -类Unix系统, 如下: - -```bash -$ cd path/dolphinscheduler -$ sh ./docker/build/hooks/build -``` - -Windows系统, 如下: - -```bat -C:\dolphinscheduler>.\docker\build\hooks\build.bat -``` - -如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。 - -## 环境变量 - -DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会使用默认值 - -**`DATABASE_TYPE`** - -配置`database`的`TYPE`， 默认值 `postgresql`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`DATABASE_DRIVER`** - -配置`database`的`DRIVER`， 默认值 `org.postgresql.Driver`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`DATABASE_HOST`** - -配置`database`的`HOST`， 默认值 `127.0.0.1`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`DATABASE_PORT`** - -配置`database`的`PORT`， 默认值 `5432`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`DATABASE_USERNAME`** - -配置`database`的`USERNAME`， 默认值 `root`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`DATABASE_PASSWORD`** - -配置`database`的`PASSWORD`， 默认值 `root`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`DATABASE_DATABASE`** - -配置`database`的`DATABASE`， 默认值 `dolphinscheduler`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`DATABASE_PARAMS`** - -配置`database`的`PARAMS`， 默认值 `characterEncoding=utf8`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`HADOOP_HOME`** - -配置`dolphinscheduler`的`HADOOP_HOME`，默认值 `/opt/soft/hadoop`。 - -**`HADOOP_CONF_DIR`** - -配置`dolphinscheduler`的`HADOOP_CONF_DIR`，默认值 `/opt/soft/hadoop/etc/hadoop`。 - -**`SPARK_HOME1`** - -配置`dolphinscheduler`的`SPARK_HOME1`，默认值 `/opt/soft/spark1`。 - -**`SPARK_HOME2`** - -配置`dolphinscheduler`的`SPARK_HOME2`，默认值 `/opt/soft/spark2`。 - -**`PYTHON_HOME`** - -配置`dolphinscheduler`的`PYTHON_HOME`，默认值 `/usr`。 - -**`JAVA_HOME`** - -配置`dolphinscheduler`的`JAVA_HOME`，默认值 `/usr/lib/jvm/java-1。8-openjdk`。 - -**`HIVE_HOME`** - -配置`dolphinscheduler`的`HIVE_HOME`，默认值 `/opt/soft/hive`。 - -**`FLINK_HOME`** - -配置`dolphinscheduler`的`FLINK_HOME`，默认值 `/opt/soft/flink`。 - -**`DATAX_HOME`** - -配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax`。 - -**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** - -用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler`。 - -**`DOLPHINSCHEDULER_OPTS`** - -配置`dolphinscheduler`的`java options`，默认值 `""`、 - -**`RESOURCE_STORAGE_TYPE`** - -配置`dolphinscheduler`的资源存储类型，可选项为 `HDFS`、`S3`、`NONE`，默认值 `HDFS`。 - -**`RESOURCE_UPLOAD_PATH`** - -配置`HDFS/S3`上的资源存储路径，默认值 `/dolphinscheduler`。 - -**`FS_DEFAULT_FS`** - -配置资源存储的文件系统协议，如 `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`，默认值 `file:///`。 - -**`FS_S3A_ENDPOINT`** - -当`RESOURCE_STORAGE_TYPE=S3`时，需要配置`S3`的访问路径，默认值 `s3.xxx.amazonaws.com`。 - -**`FS_S3A_ACCESS_KEY`** - -当`RESOURCE_STORAGE_TYPE=S3`时，需要配置`S3`的`s3 access key`，默认值 `xxxxxxx`。 - -**`FS_S3A_SECRET_KEY`** - -当`RESOURCE_STORAGE_TYPE=S3`时，需要配置`S3`的`s3 secret key`，默认值 `xxxxxxx`。 - -**`ZOOKEEPER_QUORUM`** - -配置`master-server`和`worker-serverr`的`Zookeeper`地址, 默认值 `127.0.0.1:2181`。 - -**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 - -**`ZOOKEEPER_ROOT`** - -配置`dolphinscheduler`在`zookeeper`中数据存储的根目录，默认值 `/dolphinscheduler`。 - -**`MASTER_EXEC_THREADS`** - -配置`master-server`中的执行线程数量，默认值 `100`。 - -**`MASTER_EXEC_TASK_NUM`** - -配置`master-server`中的执行任务数量，默认值 `20`。 - -**`MASTER_HEARTBEAT_INTERVAL`** - -配置`master-server`中的心跳交互时间，默认值 `10`。 - -**`MASTER_TASK_COMMIT_RETRYTIMES`** - -配置`master-server`中的任务提交重试次数，默认值 `5`。 - -**`MASTER_TASK_COMMIT_INTERVAL`** - -配置`master-server`中的任务提交交互时间，默认值 `1000`。 - -**`MASTER_MAX_CPULOAD_AVG`** - -配置`master-server`中的CPU中的`load average`值，默认值 `100`。 - -**`MASTER_RESERVED_MEMORY`** - -配置`master-server`的保留内存，默认值 `0.1`。 - -**`MASTER_LISTEN_PORT`** - -配置`master-server`的端口，默认值 `5678`。 - -**`WORKER_EXEC_THREADS`** - -配置`worker-server`中的执行线程数量，默认值 `100`。 - -**`WORKER_HEARTBEAT_INTERVAL`** - -配置`worker-server`中的心跳交互时间，默认值 `10`。 - -**`WORKER_MAX_CPULOAD_AVG`** - -配置`worker-server`中的CPU中的最大`load average`值，默认值 `100`。 - -**`WORKER_RESERVED_MEMORY`** - -配置`worker-server`的保留内存，默认值 `0.1`。 - -**`WORKER_LISTEN_PORT`** - -配置`worker-server`的端口，默认值 `1234`。 - -**`WORKER_GROUPS`** - -配置`worker-server`的分组，默认值 `default`。 - -**`WORKER_HOST_WEIGHT`** - -配置`worker-server`的权重，默认之`100`。 - -**`ALERT_LISTEN_HOST`** - -配置`worker-server`的告警主机，即`alert-server`的主机名，默认值 `127.0.0.1`。 - -**`ALERT_PLUGIN_DIR`** - -配置`alert-server`的告警插件目录，默认值 `lib/plugin/alert`。 - -## 初始化脚本 - -如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量，你可以在`/root/start-init-conf.sh`文件中进行修改，同时如果涉及到配置文件的修改，请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件 - -例如，在`/root/start-init-conf.sh`添加一个环境变量`API_SERVER_PORT`： - -``` -export API_SERVER_PORT=5555 -``` - -当添加以上环境变量后，你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置: -``` -server.port=${API_SERVER_PORT} -``` - -`/root/start-init-conf.sh`将根据模板文件动态的生成配置文件： - -```sh -echo "generate dolphinscheduler config" -ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do -eval "cat << EOF -$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) -EOF -" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} -done -``` - -## FAQ - -### 如何通过 docker-compose 停止 dolphinscheduler？ - -停止所有容器: - -``` -docker-compose stop -``` - -停止所有容器并移除所有容器，网络和存储卷: - -``` -docker-compose down -v -``` - -### 如何在 Docker Swarm 上部署 dolphinscheduler？ - -假设 Docker Swarm 集群已经部署（如果还没有创建 Docker Swarm 集群，请参考 [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/)） - -启动名为 dolphinscheduler 的 stack - -``` -docker stack deploy -c docker-stack.yml dolphinscheduler -``` - -启动并移除名为 dolphinscheduler 的 stack - -``` -docker stack rm dolphinscheduler -``` - -### 如何用 MySQL 替代 PostgreSQL 作为 DolphinScheduler 的数据库？ - -> 由于商业许可证的原因，我们不能直接使用 MySQL 的驱动包和客户端. -> -> 如果你要使用 MySQL, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建. - -1. 下载 MySQL 驱动包 [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (要求 `>=5.1.47`) - -2. 创建一个新的 `Dockerfile`，用于添加 MySQL 的驱动包和客户端: - -``` -FROM apache/dolphinscheduler:latest -COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib -RUN apk add --update --no-cache mysql-client -``` - -3. 构建一个包含 MySQL 的驱动包和客户端的新镜像: - -``` -docker build -t apache/dolphinscheduler:mysql . -``` - -4. 修改 `docker-compose.yml` 文件中的所有 image 字段为 `apache/dolphinscheduler:mysql` - -> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml` - -5. 注释 `docker-compose.yml` 文件中的 `dolphinscheduler-postgresql` 块 - -6. 在 `docker-compose.yml` 文件中添加 `dolphinscheduler-mysql` 服务（**可选**，你可以直接使用一个外部的 MySQL 数据库） - -7. 修改 `docker-compose.yml` 文件中的所有 DATABASE 环境变量 - -``` -DATABASE_TYPE: mysql -DATABASE_DRIVER: com.mysql.jdbc.Driver -DATABASE_HOST: dolphinscheduler-mysql -DATABASE_PORT: 3306 -DATABASE_USERNAME: root -DATABASE_PASSWORD: root -DATABASE_DATABASE: dolphinscheduler -DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8 -``` - -> 如果你已经添加了 `dolphinscheduler-mysql` 服务，设置 `DATABASE_HOST` 为 `dolphinscheduler-mysql` 即可 - -8. 运行 dolphinscheduler (详见**如何使用docker镜像**) - -### 如何在数据源中心支持 MySQL 数据源？ - -> 由于商业许可证的原因，我们不能直接使用 MySQL 的驱动包. -> -> 如果你要添加 MySQL 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建. - -1. 下载 MySQL 驱动包 [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (要求 `>=5.1.47`) - -2. 创建一个新的 `Dockerfile`，用于添加 MySQL 驱动包: - -``` -FROM apache/dolphinscheduler:latest -COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib -``` - -3. 构建一个包含 MySQL 驱动包的新镜像: - -``` -docker build -t apache/dolphinscheduler:mysql-driver . -``` - -4. 将 `docker-compose.yml` 文件中的所有 image 字段 修改为 `apache/dolphinscheduler:mysql-driver` - -> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml` - -5. 运行 dolphinscheduler (详见**如何使用docker镜像**) - -6. 在数据源中心添加一个 MySQL 数据源 - -### 如何在数据源中心支持 Oracle 数据源？ - -> 由于商业许可证的原因，我们不能直接使用 Oracle 的驱动包. -> -> 如果你要添加 Oracle 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建. - -1. 下载 Oracle 驱动包 [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`) - -2. 创建一个新的 `Dockerfile`，用于添加 Oracle 驱动包: - -``` -FROM apache/dolphinscheduler:latest -COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib -``` - -3. 构建一个包含 Oracle 驱动包的新镜像: - -``` -docker build -t apache/dolphinscheduler:oracle-driver . -``` - -4. 将 `docker-compose.yml` 文件中的所有 image 字段 修改为 `apache/dolphinscheduler:oracle-driver` - -> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml` - -5. 运行 dolphinscheduler (详见**如何使用docker镜像**) - -6. 在数据源中心添加一个 Oracle 数据源 - -更多信息请查看 [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) 文档. diff --git a/docker/build/conf/dolphinscheduler/application-api.properties.tpl b/docker/build/conf/dolphinscheduler/application-api.properties.tpl index caa8a9e35..d78db2d63 100644 --- a/docker/build/conf/dolphinscheduler/application-api.properties.tpl +++ b/docker/build/conf/dolphinscheduler/application-api.properties.tpl @@ -41,12 +41,21 @@ server.jetty.max-http-form-post-size=5000000 # messages encoding spring.messages.encoding=UTF-8 -# i18n classpath folder , file prefix messages， if have many files, use "," seperator +# i18n classpath folder, file prefix messages. if have many files, use "," seperator spring.messages.basename=i18n/messages # Authentication types (supported types: PASSWORD) security.authentication.type=PASSWORD +# Traffic control, if you turn on this config, the maximum number of request/s will be limited. +# global max request number per second +# default tenant-level max request number +#traffic.control.global.switch=true +#traffic.control.max.global.qps.rate=500 +#traffic.control.tenant.switch=true +#traffic.control.default.tenant.qps.rate=10 +#traffic.control.customize.tenant.qps.rate={'tenant1':11,'tenant2':20} + #============================================================================ # LDAP Config # mock ldap server from https://www.forumsys.com/tutorials/integration-how-to/ldap/online-ldap-test-server/ diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index 36bc5b8de..ccf25d85b 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -15,34 +15,37 @@ # limitations under the License. # -# resource storage type : HDFS, S3, NONE +# user data local directory path, please make sure the directory exists and have read write permissions +data.basedir.path=${DATA_BASEDIR_PATH} + +# resource storage type: HDFS, S3, NONE resource.storage.type=${RESOURCE_STORAGE_TYPE} -# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.upload.path=${RESOURCE_UPLOAD_PATH} -# user data local directory path, please make sure the directory exists and have read write permissions -data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} - -# whether kerberos starts -hadoop.security.authentication.startup.state=false +# whether to startup kerberos +hadoop.security.authentication.startup.state=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE} # java.security.krb5.conf path -java.security.krb5.conf.path=/opt/krb5.conf +java.security.krb5.conf.path=${JAVA_SECURITY_KRB5_CONF_PATH} # login user from keytab username -login.user.keytab.username=hdfs-mycluster@ESZ.COM +login.user.keytab.username=${LOGIN_USER_KEYTAB_USERNAME} # login user from keytab path -login.user.keytab.path=/opt/hdfs.headless.keytab +login.user.keytab.path=${LOGIN_USER_KEYTAB_PATH} + +# kerberos expire time, the unit is hour +kerberos.expire.time=${KERBEROS_EXPIRE_TIME} -#resource.view.suffixs +# resource view suffixs #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js -# if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path -hdfs.root.user=hdfs +# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path +hdfs.root.user=${HDFS_ROOT_USER} -# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir +# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir fs.defaultFS=${FS_DEFAULT_FS} # if resource.storage.type=S3, s3 endpoint @@ -54,25 +57,32 @@ fs.s3a.access.key=${FS_S3A_ACCESS_KEY} # if resource.storage.type=S3, s3 secret key fs.s3a.secret.key=${FS_S3A_SECRET_KEY} -# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty -yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=${RESOURCE_MANAGER_HTTPADDRESS_PORT} -# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. -yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s +# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty +yarn.resourcemanager.ha.rm.ids=${YARN_RESOURCEMANAGER_HA_RM_IDS} -# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000) -yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s +# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname +yarn.application.status.address=${YARN_APPLICATION_STATUS_ADDRESS} -# system env path, If you want to set your own path, you need to set this env file to an absolute path -#dolphinscheduler.env.path=env/dolphinscheduler_env.sh -development.state=false +# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) +yarn.job.history.status.address=${YARN_JOB_HISTORY_STATUS_ADDRESS} -# kerberos tgt expire time, unit is hours -kerberos.expire.time=2 +# datasource encryption enable +datasource.encryption.enable=${DATASOURCE_ENCRYPTION_ENABLE} # datasource encryption salt -datasource.encryption.enable=false -datasource.encryption.salt=!@#$%^&* +datasource.encryption.salt=${DATASOURCE_ENCRYPTION_SALT} + +# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions +sudo.enable=${SUDO_ENABLE} -# Network IP gets priority, default inner outer +# network IP gets priority, default: inner outer #dolphin.scheduler.network.priority.strategy=default + +# system env path +#dolphinscheduler.env.path=env/dolphinscheduler_env.sh + +# development state +development.state=false diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl index b414d2143..4c447df2f 100644 --- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl +++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl @@ -24,14 +24,14 @@ spring.datasource.password=${DATABASE_PASSWORD} # postgresql #spring.datasource.driver-class-name=org.postgresql.Driver #spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler -#spring.datasource.username=test -#spring.datasource.password=test +#spring.datasource.username=root +#spring.datasource.password=root # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 -#spring.datasource.username=xxxx -#spring.datasource.password=xxxx +#spring.datasource.username=ds_user +#spring.datasource.password=dolphinscheduler # connection configuration #spring.datasource.initialSize=5 diff --git a/docker/build/conf/dolphinscheduler/master.properties.tpl b/docker/build/conf/dolphinscheduler/master.properties.tpl index 3d7910fe6..046d5c15f 100644 --- a/docker/build/conf/dolphinscheduler/master.properties.tpl +++ b/docker/build/conf/dolphinscheduler/master.properties.tpl @@ -15,29 +15,32 @@ # limitations under the License. # -# master execute thread number +# master listen port +#master.listen.port=5678 + +# master execute thread number to limit process instances in parallel master.exec.threads=${MASTER_EXEC_THREADS} -# master execute task number in parallel +# master execute task number in parallel per process instance master.exec.task.num=${MASTER_EXEC_TASK_NUM} -# master dispatch task number -#master.dispatch.task.num=3 +# master dispatch task number per batch +master.dispatch.task.num=${MASTER_DISPATCH_TASK_NUM} + +# master host selector to select a suitable worker, default value: LowerWeight. Optional values include Random, RoundRobin, LowerWeight +master.host.selector=${MASTER_HOST_SELECTOR} -# master heartbeat interval +# master heartbeat interval, the unit is second master.heartbeat.interval=${MASTER_HEARTBEAT_INTERVAL} # master commit task retry times master.task.commit.retryTimes=${MASTER_TASK_COMMIT_RETRYTIMES} -# master commit task interval +# master commit task interval, the unit is millisecond master.task.commit.interval=${MASTER_TASK_COMMIT_INTERVAL} -# only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 +# master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2 master.max.cpuload.avg=${MASTER_MAX_CPULOAD_AVG} -# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. +# master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G master.reserved.memory=${MASTER_RESERVED_MEMORY} - -# master listen port -master.listen.port=${MASTER_LISTEN_PORT} diff --git a/docker/build/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl index 7dd69355d..94a335261 100644 --- a/docker/build/conf/dolphinscheduler/worker.properties.tpl +++ b/docker/build/conf/dolphinscheduler/worker.properties.tpl @@ -15,10 +15,13 @@ # limitations under the License. # -# worker execute thread number +# worker listener port +#worker.listen.port=1234 + +# worker execute thread number to limit task instances in parallel worker.exec.threads=${WORKER_EXEC_THREADS} -# worker heartbeat interval +# worker heartbeat interval, the unit is second worker.heartbeat.interval=${WORKER_HEARTBEAT_INTERVAL} # worker host weight to dispatch tasks, default value 100 @@ -27,17 +30,14 @@ worker.host.weight=${WORKER_HOST_WEIGHT} # worker tenant auto create worker.tenant.auto.create=true -# only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2 +# worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2 worker.max.cpuload.avg=${WORKER_MAX_CPULOAD_AVG} -# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. +# worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G worker.reserved.memory=${WORKER_RESERVED_MEMORY} -# worker listener port -worker.listen.port=${WORKER_LISTEN_PORT} - -# default worker groups, if this worker belongs different groups, you can config the following like that 'worker.groups=default,test' +# default worker groups separated by comma, like 'worker.groups=default,test' worker.groups=${WORKER_GROUPS} -# alert server listener host +# alert server listen host alert.listen.host=${ALERT_LISTEN_HOST} diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 80baa92d2..4c03c844d 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -23,7 +23,7 @@ echo "init env variables" # Define parameters default value #============================================================================ -# Database Source +# Database #============================================================================ export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"} export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"} @@ -34,64 +34,91 @@ export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"} export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"} export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} +#============================================================================ +# ZooKeeper +#============================================================================ +export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"} +export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"} + #============================================================================ # Common #============================================================================ +# common opts +export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} +# common env +export DATA_BASEDIR_PATH=${DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} +export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"} +export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"} +export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"} +export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} +export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} +export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} +export HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=${HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE:-"false"} +export JAVA_SECURITY_KRB5_CONF_PATH=${JAVA_SECURITY_KRB5_CONF_PATH:-"/opt/krb5.conf"} +export LOGIN_USER_KEYTAB_USERNAME=${LOGIN_USER_KEYTAB_USERNAME:-"hdfs@HADOOP.COM"} +export LOGIN_USER_KEYTAB_PATH=${LOGIN_USER_KEYTAB_PATH:-"/opt/hdfs.keytab"} +export KERBEROS_EXPIRE_TIME=${KERBEROS_EXPIRE_TIME:-"2"} +export HDFS_ROOT_USER=${HDFS_ROOT_USER:-"hdfs"} +export RESOURCE_MANAGER_HTTPADDRESS_PORT=${RESOURCE_MANAGER_HTTPADDRESS_PORT:-"8088"} +export YARN_RESOURCEMANAGER_HA_RM_IDS=${YARN_RESOURCEMANAGER_HA_RM_IDS:-""} +export YARN_APPLICATION_STATUS_ADDRESS=${YARN_APPLICATION_STATUS_ADDRESS:-"http://ds1:%s/ws/v1/cluster/apps/%s"} +export YARN_JOB_HISTORY_STATUS_ADDRESS=${YARN_JOB_HISTORY_STATUS_ADDRESS:-"http://ds1:19888/ws/v1/history/mapreduce/jobs/%s"} +export DATASOURCE_ENCRYPTION_ENABLE=${DATASOURCE_ENCRYPTION_ENABLE:-"false"} +export DATASOURCE_ENCRYPTION_SALT=${DATASOURCE_ENCRYPTION_SALT:-"!@#$%^&*"} +export SUDO_ENABLE=${SUDO_ENABLE:-"true"} # dolphinscheduler env export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"} export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"} export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"} export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"} -export PYTHON_HOME=${PYTHON_HOME:-"/usr"} -export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"} +export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"} +export JAVA_HOME=${JAVA_HOME:-"/usr/local/openjdk-8"} export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"} export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"} export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"} -# common env -export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} -export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} -export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"} -export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"} -export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"} -export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} -export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} -export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} - -#============================================================================ -# Zookeeper -#============================================================================ -export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"} -export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"} #============================================================================ # Master Server #============================================================================ +export MASTER_SERVER_OPTS=${MASTER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"} export MASTER_EXEC_THREADS=${MASTER_EXEC_THREADS:-"100"} export MASTER_EXEC_TASK_NUM=${MASTER_EXEC_TASK_NUM:-"20"} +export MASTER_DISPATCH_TASK_NUM=${MASTER_DISPATCH_TASK_NUM:-"3"} +export MASTER_HOST_SELECTOR=${MASTER_HOST_SELECTOR:-"LowerWeight"} export MASTER_HEARTBEAT_INTERVAL=${MASTER_HEARTBEAT_INTERVAL:-"10"} export MASTER_TASK_COMMIT_RETRYTIMES=${MASTER_TASK_COMMIT_RETRYTIMES:-"5"} export MASTER_TASK_COMMIT_INTERVAL=${MASTER_TASK_COMMIT_INTERVAL:-"1000"} -export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"100"} -export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.1"} -export MASTER_LISTEN_PORT=${MASTER_LISTEN_PORT:-"5678"} +export MASTER_MAX_CPULOAD_AVG=${MASTER_MAX_CPULOAD_AVG:-"-1"} +export MASTER_RESERVED_MEMORY=${MASTER_RESERVED_MEMORY:-"0.3"} #============================================================================ # Worker Server #============================================================================ +export WORKER_SERVER_OPTS=${WORKER_SERVER_OPTS:-"-Xms1g -Xmx1g -Xmn512m"} export WORKER_EXEC_THREADS=${WORKER_EXEC_THREADS:-"100"} export WORKER_HEARTBEAT_INTERVAL=${WORKER_HEARTBEAT_INTERVAL:-"10"} -export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"100"} -export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"} -export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"} -export WORKER_GROUPS=${WORKER_GROUPS:-"default"} export WORKER_HOST_WEIGHT=${WORKER_HOST_WEIGHT:-"100"} -export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"} +export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"-1"} +export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.3"} +export WORKER_GROUPS=${WORKER_GROUPS:-"default"} +export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"localhost"} #============================================================================ # Alert Server #============================================================================ +export ALERT_SERVER_OPTS=${ALERT_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"} export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"} +#============================================================================ +# Api Server +#============================================================================ +export API_SERVER_OPTS=${API_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"} + +#============================================================================ +# Logger Server +#============================================================================ +export LOGGER_SERVER_OPTS=${LOGGER_SERVER_OPTS:-"-Xms512m -Xmx512m -Xmn256m"} + echo "generate dolphinscheduler config" ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do eval "cat << EOF @@ -100,9 +127,9 @@ EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done -# generate dolphinscheduler env only in docker +# generate dolphinscheduler env DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh -if [ -z "${KUBERNETES_SERVICE_HOST}" ] && [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then +if [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then eval "cat << EOF $(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl) EOF diff --git a/docker/build/startup.sh b/docker/build/startup.sh index 934fac3ae..ae1ed3677 100755 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -27,7 +27,7 @@ export LOGGER_START_ENABLED=false # wait database waitDatabase() { - echo "test ${DATABASE_TYPE} service" + echo "try to connect ${DATABASE_TYPE} ..." while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do local counter=$((counter+1)) if [ $counter == 30 ]; then @@ -37,21 +37,7 @@ waitDatabase() { echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 5 done - - echo "connect ${DATABASE_TYPE} service" - if [ ${DATABASE_TYPE} = "mysql" ]; then - v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1) - if [ "$(echo ${v} | grep 'ERROR' | wc -l)" -eq 1 ]; then - echo "Error: Can't connect to database...${v}" - exit 1 - fi - else - v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") - if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then - echo "Error: Can't connect to database...${v}" - exit 1 - fi - fi + echo "${DATABASE_TYPE} connection is ok" } # init database @@ -60,38 +46,9 @@ initDatabase() { ${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh } -# check ds version -checkDSVersion() { - if [ ${DATABASE_TYPE} = "mysql" ]; then - v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "SELECT * FROM public.t_ds_version" 2>/dev/null) - else - v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "SELECT * FROM public.t_ds_version" 2>/dev/null) - fi - if [ -n "$v" ]; then - echo "ds version: $v" - return 0 - else - return 1 - fi -} - -# check init database -checkInitDatabase() { - echo "check init database" - while ! checkDSVersion; do - local counter=$((counter+1)) - if [ $counter == 30 ]; then - echo "Error: Couldn't check init database." - exit 1 - fi - echo "Trying to check init database. Attempt $counter." - sleep 5 - done -} - # wait zk waitZK() { - echo "connect remote zookeeper" + echo "try to connect zookeeper ..." echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do while ! nc -z ${line%:*} ${line#*:}; do local counter=$((counter+1)) @@ -103,6 +60,7 @@ waitZK() { sleep 5 done done + echo "zookeeper connection is ok" } # print usage @@ -150,7 +108,6 @@ case "$1" in ;; (alert-server) waitDatabase - checkInitDatabase export ALERT_START_ENABLED=true ;; (help) diff --git a/docker/docker-swarm/check b/docker/docker-swarm/check index f4093664d..66484ae0c 100755 --- a/docker/docker-swarm/check +++ b/docker/docker-swarm/check @@ -15,21 +15,28 @@ # See the License for the specific language governing permissions and # limitations under the License. # -echo "------ dolphinscheduler check - server - status -------" -sleep 60 -server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -if [ $server_num -eq 5 ] -then - echo "Server all start successfully" -else - echo "Server start failed "$server_num - exit 1 -fi -ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` -if [ $ready -eq 1 ] -then - echo "Servers is ready" -else - echo "Servers is not ready" - exit 1 -fi + +echo "------- check server status -------" +cd $(dirname $0) +until [ $(docker-compose top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) -eq 5 ]; do + counter=$((counter+1)) + if [ $counter -eq 30 ]; then + echo "error: all servers start failed" + exit 1 + fi + echo "waiting for all servers started ..." + sleep 5 +done +echo "all servers start successfully" + +counter=0 +until docker logs docker-swarm_dolphinscheduler-api_1 2>&1 | grep '0.0.0.0:12345' &>/dev/null; do + counter=$((counter+1)) + if [ $counter -eq 30 ]; then + echo "error: api server is not ready" + exit 1 + fi + echo "waiting for api server ready ..." + sleep 5 +done +echo "api server is ready" diff --git a/docker/docker-swarm/config.env.sh b/docker/docker-swarm/config.env.sh new file mode 100755 index 000000000..7ef4c98ab --- /dev/null +++ b/docker/docker-swarm/config.env.sh @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#============================================================================ +# Database +#============================================================================ +# postgresql +DATABASE_TYPE=postgresql +DATABASE_DRIVER=org.postgresql.Driver +DATABASE_HOST=dolphinscheduler-postgresql +DATABASE_PORT=5432 +DATABASE_USERNAME=root +DATABASE_PASSWORD=root +DATABASE_DATABASE=dolphinscheduler +DATABASE_PARAMS=characterEncoding=utf8 +# mysql +# DATABASE_TYPE=mysql +# DATABASE_DRIVER=com.mysql.jdbc.Driver +# DATABASE_HOST=dolphinscheduler-mysql +# DATABASE_PORT=3306 +# DATABASE_USERNAME=root +# DATABASE_PASSWORD=root +# DATABASE_DATABASE=dolphinscheduler +# DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8 + +#============================================================================ +# ZooKeeper +#============================================================================ +ZOOKEEPER_QUORUM=dolphinscheduler-zookeeper:2181 +ZOOKEEPER_ROOT=/dolphinscheduler + +#============================================================================ +# Common +#============================================================================ +# common opts +DOLPHINSCHEDULER_OPTS= +# common env +DATA_BASEDIR_PATH=/tmp/dolphinscheduler +RESOURCE_STORAGE_TYPE=HDFS +RESOURCE_UPLOAD_PATH=/dolphinscheduler +FS_DEFAULT_FS=file:/// +FS_S3A_ENDPOINT=s3.xxx.amazonaws.com +FS_S3A_ACCESS_KEY=xxxxxxx +FS_S3A_SECRET_KEY=xxxxxxx +HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE=false +JAVA_SECURITY_KRB5_CONF_PATH=/opt/krb5.conf +LOGIN_USER_KEYTAB_USERNAME=hdfs@HADOOP.COM +LOGIN_USER_KEYTAB_PATH=/opt/hdfs.keytab +KERBEROS_EXPIRE_TIME=2 +HDFS_ROOT_USER=hdfs +RESOURCE_MANAGER_HTTPADDRESS_PORT=8088 +YARN_RESOURCEMANAGER_HA_RM_IDS= +YARN_APPLICATION_STATUS_ADDRESS=http://ds1:%s/ws/v1/cluster/apps/%s +YARN_JOB_HISTORY_STATUS_ADDRESS=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s +DATASOURCE_ENCRYPTION_ENABLE=false +DATASOURCE_ENCRYPTION_SALT=!@#$%^&* +SUDO_ENABLE=true +# dolphinscheduler env +HADOOP_HOME=/opt/soft/hadoop +HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop +SPARK_HOME1=/opt/soft/spark1 +SPARK_HOME2=/opt/soft/spark2 +PYTHON_HOME=/usr/bin/python +JAVA_HOME=/usr/local/openjdk-8 +HIVE_HOME=/opt/soft/hive +FLINK_HOME=/opt/soft/flink +DATAX_HOME=/opt/soft/datax + +#============================================================================ +# Master Server +#============================================================================ +MASTER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m +MASTER_EXEC_THREADS=100 +MASTER_EXEC_TASK_NUM=20 +MASTER_DISPATCH_TASK_NUM=3 +MASTER_HOST_SELECTOR=LowerWeight +MASTER_HEARTBEAT_INTERVAL=10 +MASTER_TASK_COMMIT_RETRYTIMES=5 +MASTER_TASK_COMMIT_INTERVAL=1000 +MASTER_MAX_CPULOAD_AVG=-1 +MASTER_RESERVED_MEMORY=0.3 + +#============================================================================ +# Worker Server +#============================================================================ +WORKER_SERVER_OPTS=-Xms1g -Xmx1g -Xmn512m +WORKER_EXEC_THREADS=100 +WORKER_HEARTBEAT_INTERVAL=10 +WORKER_HOST_WEIGHT=100 +WORKER_MAX_CPULOAD_AVG=-1 +WORKER_RESERVED_MEMORY=0.3 +WORKER_GROUPS=default +ALERT_LISTEN_HOST=dolphinscheduler-alert + +#============================================================================ +# Alert Server +#============================================================================ +ALERT_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m +ALERT_PLUGIN_DIR=lib/plugin/alert + +#============================================================================ +# Api Server +#============================================================================ +API_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m + +#============================================================================ +# Logger Server +#============================================================================ +LOGGER_SERVER_OPTS=-Xms512m -Xmx512m -Xmn256m diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 8f6a0b85c..faa7abb65 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -19,10 +19,7 @@ version: "3.1" services: dolphinscheduler-postgresql: - image: bitnami/postgresql:latest - container_name: dolphinscheduler-postgresql - ports: - - 5432:5432 + image: bitnami/postgresql:11.11.0 environment: TZ: Asia/Shanghai POSTGRESQL_USERNAME: root @@ -30,16 +27,12 @@ services: POSTGRESQL_DATABASE: dolphinscheduler volumes: - dolphinscheduler-postgresql:/bitnami/postgresql - - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-zookeeper: - image: bitnami/zookeeper:latest - container_name: dolphinscheduler-zookeeper - ports: - - 2181:2181 + image: bitnami/zookeeper:3.6.2 environment: TZ: Asia/Shanghai ALLOW_ANONYMOUS_LOGIN: "yes" @@ -51,26 +44,13 @@ services: - dolphinscheduler dolphinscheduler-api: - image: apache/dolphinscheduler:latest - container_name: dolphinscheduler-api + image: apache/dolphinscheduler:1.3.6-SNAPSHOT command: api-server ports: - 12345:12345 environment: TZ: Asia/Shanghai - DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" - DATABASE_TYPE: postgresql - DATABASE_DRIVER: org.postgresql.Driver - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler - DATABASE_PARAMS: characterEncoding=utf8 - ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - RESOURCE_STORAGE_TYPE: HDFS - RESOURCE_UPLOAD_PATH: /dolphinscheduler - FS_DEFAULT_FS: file:/// + env_file: config.env.sh healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30s @@ -81,29 +61,18 @@ services: - dolphinscheduler-zookeeper volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-alert: - image: apache/dolphinscheduler:latest - container_name: dolphinscheduler-alert + image: apache/dolphinscheduler:1.3.6-SNAPSHOT command: alert-server - ports: - - 50052:50052 environment: TZ: Asia/Shanghai - ALERT_PLUGIN_DIR: lib/plugin/alert - DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" - DATABASE_TYPE: postgresql - DATABASE_DRIVER: org.postgresql.Driver - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler - DATABASE_PARAMS: characterEncoding=utf8 + env_file: config.env.sh healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30s @@ -118,31 +87,11 @@ services: - dolphinscheduler dolphinscheduler-master: - image: apache/dolphinscheduler:latest - container_name: dolphinscheduler-master + image: apache/dolphinscheduler:1.3.6-SNAPSHOT command: master-server - ports: - - 5678:5678 environment: TZ: Asia/Shanghai - MASTER_EXEC_THREADS: "100" - MASTER_EXEC_TASK_NUM: "20" - MASTER_HEARTBEAT_INTERVAL: "10" - MASTER_TASK_COMMIT_RETRYTIMES: "5" - MASTER_TASK_COMMIT_INTERVAL: "1000" - MASTER_MAX_CPULOAD_AVG: "100" - MASTER_RESERVED_MEMORY: "0.1" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler - DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" - DATABASE_TYPE: postgresql - DATABASE_DRIVER: org.postgresql.Driver - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler - DATABASE_PARAMS: characterEncoding=utf8 - ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + env_file: config.env.sh healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] interval: 30s @@ -153,49 +102,17 @@ services: - dolphinscheduler-zookeeper volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft restart: unless-stopped networks: - dolphinscheduler dolphinscheduler-worker: - image: apache/dolphinscheduler:latest - container_name: dolphinscheduler-worker + image: apache/dolphinscheduler:1.3.6-SNAPSHOT command: worker-server - ports: - - 1234:1234 - - 50051:50051 environment: TZ: Asia/Shanghai - WORKER_EXEC_THREADS: "100" - WORKER_HEARTBEAT_INTERVAL: "10" - WORKER_MAX_CPULOAD_AVG: "100" - WORKER_RESERVED_MEMORY: "0.1" - WORKER_GROUPS: "default" - WORKER_HOST_WEIGHT: "100" - ALERT_LISTEN_HOST: dolphinscheduler-alert - HADOOP_HOME: "/opt/soft/hadoop" - HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" - SPARK_HOME1: "/opt/soft/spark1" - SPARK_HOME2: "/opt/soft/spark2" - #PYTHON_HOME: "/opt/soft/python" - JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" - HIVE_HOME: "/opt/soft/hive" - FLINK_HOME: "/opt/soft/flink" - DATAX_HOME: "/opt/soft/datax" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler - DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" - DATABASE_TYPE: postgresql - DATABASE_DRIVER: org.postgresql.Driver - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler - DATABASE_PARAMS: characterEncoding=utf8 - ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - RESOURCE_STORAGE_TYPE: HDFS - RESOURCE_UPLOAD_PATH: /dolphinscheduler - FS_DEFAULT_FS: file:/// + env_file: config.env.sh healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] interval: 30s @@ -207,6 +124,7 @@ services: volumes: - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler restart: unless-stopped networks: @@ -218,8 +136,8 @@ networks: volumes: dolphinscheduler-postgresql: - dolphinscheduler-postgresql-initdb: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: dolphinscheduler-logs: + dolphinscheduler-shared-local: dolphinscheduler-resource-local: \ No newline at end of file diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index d78e7261e..52617ca65 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -19,9 +19,7 @@ version: "3.1" services: dolphinscheduler-postgresql: - image: bitnami/postgresql:latest - ports: - - 5432:5432 + image: bitnami/postgresql:11.11.0 environment: TZ: Asia/Shanghai POSTGRESQL_USERNAME: root @@ -36,9 +34,7 @@ services: replicas: 1 dolphinscheduler-zookeeper: - image: bitnami/zookeeper:latest - ports: - - 2181:2181 + image: bitnami/zookeeper:3.6.2 environment: TZ: Asia/Shanghai ALLOW_ANONYMOUS_LOGIN: "yes" @@ -52,25 +48,13 @@ services: replicas: 1 dolphinscheduler-api: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.6-SNAPSHOT command: api-server ports: - 12345:12345 environment: TZ: Asia/Shanghai - DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" - DATABASE_TYPE: postgresql - DATABASE_DRIVER: org.postgresql.Driver - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler - DATABASE_PARAMS: characterEncoding=utf8 - ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - RESOURCE_STORAGE_TYPE: HDFS - RESOURCE_UPLOAD_PATH: /dolphinscheduler - FS_DEFAULT_FS: file:/// + env_file: config.env.sh healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30s @@ -78,6 +62,8 @@ services: retries: 3 volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler networks: - dolphinscheduler deploy: @@ -85,22 +71,11 @@ services: replicas: 1 dolphinscheduler-alert: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.6-SNAPSHOT command: alert-server - ports: - - 50052:50052 environment: TZ: Asia/Shanghai - ALERT_PLUGIN_DIR: lib/plugin/alert - DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" - DATABASE_TYPE: postgresql - DATABASE_DRIVER: org.postgresql.Driver - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler - DATABASE_PARAMS: characterEncoding=utf8 + env_file: config.env.sh healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30s @@ -115,30 +90,11 @@ services: replicas: 1 dolphinscheduler-master: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.6-SNAPSHOT command: master-server - ports: - - 5678:5678 environment: TZ: Asia/Shanghai - MASTER_EXEC_THREADS: "100" - MASTER_EXEC_TASK_NUM: "20" - MASTER_HEARTBEAT_INTERVAL: "10" - MASTER_TASK_COMMIT_RETRYTIMES: "5" - MASTER_TASK_COMMIT_INTERVAL: "1000" - MASTER_MAX_CPULOAD_AVG: "100" - MASTER_RESERVED_MEMORY: "0.1" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler - DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" - DATABASE_TYPE: postgresql - DATABASE_DRIVER: org.postgresql.Driver - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler - DATABASE_PARAMS: characterEncoding=utf8 - ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + env_file: config.env.sh healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] interval: 30s @@ -146,6 +102,7 @@ services: retries: 3 volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft networks: - dolphinscheduler deploy: @@ -153,43 +110,11 @@ services: replicas: 1 dolphinscheduler-worker: - image: apache/dolphinscheduler:latest + image: apache/dolphinscheduler:1.3.6-SNAPSHOT command: worker-server - ports: - - 1234:1234 - - 50051:50051 environment: TZ: Asia/Shanghai - WORKER_EXEC_THREADS: "100" - WORKER_HEARTBEAT_INTERVAL: "10" - WORKER_MAX_CPULOAD_AVG: "100" - WORKER_RESERVED_MEMORY: "0.1" - WORKER_GROUPS: "default" - WORKER_HOST_WEIGHT: "100" - ALERT_LISTEN_HOST: dolphinscheduler-alert - HADOOP_HOME: "/opt/soft/hadoop" - HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" - SPARK_HOME1: "/opt/soft/spark1" - SPARK_HOME2: "/opt/soft/spark2" - #PYTHON_HOME: "/opt/soft/python" - JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" - HIVE_HOME: "/opt/soft/hive" - FLINK_HOME: "/opt/soft/flink" - DATAX_HOME: "/opt/soft/datax" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler - DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" - DATABASE_TYPE: postgresql - DATABASE_DRIVER: org.postgresql.Driver - DATABASE_HOST: dolphinscheduler-postgresql - DATABASE_PORT: 5432 - DATABASE_USERNAME: root - DATABASE_PASSWORD: root - DATABASE_DATABASE: dolphinscheduler - DATABASE_PARAMS: characterEncoding=utf8 - ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 - RESOURCE_STORAGE_TYPE: HDFS - RESOURCE_UPLOAD_PATH: /dolphinscheduler - FS_DEFAULT_FS: file:/// + env_file: config.env.sh healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] interval: 30s @@ -198,6 +123,8 @@ services: volumes: - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler networks: - dolphinscheduler deploy: @@ -212,4 +139,6 @@ volumes: dolphinscheduler-postgresql: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: - dolphinscheduler-logs: \ No newline at end of file + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml index 1e0ca755b..486f35012 100644 --- a/docker/kubernetes/dolphinscheduler/Chart.yaml +++ b/docker/kubernetes/dolphinscheduler/Chart.yaml @@ -35,18 +35,18 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 1.2.0 +version: 2.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 1.4.0 +appVersion: 1.3.6-SNAPSHOT dependencies: - name: postgresql - version: 10.x.x + version: 10.3.18 repository: https://charts.bitnami.com/bitnami condition: postgresql.enabled - name: zookeeper - version: 6.x.x + version: 6.5.3 repository: https://charts.bitnami.com/bitnami condition: zookeeper.enabled diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md deleted file mode 100644 index ed80e7b5f..000000000 --- a/docker/kubernetes/dolphinscheduler/README.md +++ /dev/null @@ -1,364 +0,0 @@ -# DolphinScheduler - -[DolphinScheduler](https://dolphinscheduler.apache.org) is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. - -## Introduction -This chart bootstraps a [DolphinScheduler](https://dolphinscheduler.apache.org) distributed deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. - -## Prerequisites - -- [Helm](https://helm.sh/) 3.1.0+ -- [Kubernetes](https://kubernetes.io/) 1.12+ -- PV provisioner support in the underlying infrastructure - -## Installing the Chart - -To install the chart with the release name `dolphinscheduler`: - -```bash -$ git clone https://github.com/apache/dolphinscheduler.git -$ cd dolphinscheduler/docker/kubernetes/dolphinscheduler -$ helm repo add bitnami https://charts.bitnami.com/bitnami -$ helm dependency update . -$ helm install dolphinscheduler . -``` - -To install the chart with a namespace named `test`: - -```bash -$ helm install dolphinscheduler . -n test -``` - -> **Tip**: If a namespace named `test` is used, the option `-n test` needs to be added to the `helm` and `kubectl` command - -These commands deploy DolphinScheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. - -> **Tip**: List all releases using `helm list` - -## Access DolphinScheduler UI - -If `ingress.enabled` in `values.yaml` is set to `true`, you just access `http://${ingress.host}/dolphinscheduler` in browser. - -> **Tip**: If there is a problem with ingress access, please contact the Kubernetes administrator and refer to the [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) - -Otherwise, you need to execute port-forward command like: - -```bash -$ kubectl port-forward --address 0.0.0.0 svc/dolphinscheduler-api 12345:12345 -$ kubectl port-forward --address 0.0.0.0 -n test svc/dolphinscheduler-api 12345:12345 # with test namespace -``` - -> **Tip**: If the error of `unable to do port forwarding: socat not found` appears, you need to install `socat` at first - -And then access the web: http://192.168.xx.xx:12345/dolphinscheduler - -The default username is `admin` and the default password is `dolphinscheduler123` - -> **Tip**: For quick start in docker, you can create a tenant named `ds` and associate the user `admin` with the tenant `ds` - -## Uninstalling the Chart - -To uninstall/delete the `dolphinscheduler` deployment: - -```bash -$ helm uninstall dolphinscheduler -``` - -The command removes all the Kubernetes components but PVC's associated with the chart and deletes the release. - -To delete the PVC's associated with `dolphinscheduler`: - -```bash -$ kubectl delete pvc -l app.kubernetes.io/instance=dolphinscheduler -``` - -> **Note**: Deleting the PVC's will delete all data as well. Please be cautious before doing it. - -## Configuration - -The Configuration file is `values.yaml`, and the following tables lists the configurable parameters of the DolphinScheduler chart and their default values. - -| Parameter | Description | Default | -| --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- | -| `timezone` | World time and date for cities in all time zones | `Asia/Shanghai` | -| | | | -| `image.repository` | Docker image repository for the DolphinScheduler | `apache/dolphinscheduler` | -| `image.tag` | Docker image version for the DolphinScheduler | `latest` | -| `image.pullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | -| `image.pullSecret` | Image pull secret. An optional reference to secret in the same namespace to use for pulling any of the images | `nil` | -| | | | -| `postgresql.enabled` | If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL | `true` | -| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` | -| `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` | -| `postgresql.postgresqlDatabase` | The database for internal PostgreSQL | `dolphinscheduler` | -| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | -| `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | -| `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it | `postgresql` | -| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it | `org.postgresql.Driver` | -| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it | `localhost` | -| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database port will use it | `5432` | -| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database username will use it | `root` | -| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database password will use it | `root` | -| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database database will use it | `dolphinscheduler` | -| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database params will use it | `characterEncoding=utf8` | -| | | | -| `zookeeper.enabled` | If not exists external Zookeeper, by default, the DolphinScheduler will use a internal Zookeeper | `true` | -| `zookeeper.fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr,ruok,wchs,cons` | -| `zookeeper.service.port` | ZooKeeper port | `2181` | -| `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` | -| `zookeeper.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | -| `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `zookeeper.zookeeperRoot` | Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` | -| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enabled` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` | -| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enabled` value to false. Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` | -| | | | -| `common.configmap.DOLPHINSCHEDULER_ENV` | System env path, self configuration, please read `values.yaml` | `[]` | -| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | User data directory path, self configuration, please make sure the directory exists and have read write permissions | `/tmp/dolphinscheduler` | -| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource storage type: HDFS, S3, NONE | `HDFS` | -| `common.configmap.RESOURCE_UPLOAD_PATH` | Resource store on HDFS/S3 path, please make sure the directory exists on hdfs and have read write permissions | `/dolphinscheduler` | -| `common.configmap.FS_DEFAULT_FS` | Resource storage file system like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler` | `file:///` | -| `common.configmap.FS_S3A_ENDPOINT` | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `s3.xxx.amazonaws.com` | -| `common.configmap.FS_S3A_ACCESS_KEY` | S3 access key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` | -| `common.configmap.FS_S3A_SECRET_KEY` | S3 secret key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` | -| `common.fsFileResourcePersistence.enabled` | Set `common.fsFileResourcePersistence.enabled` to `true` to mount a new file resource volume for `api` and `worker` | `false` | -| `common.fsFileResourcePersistence.accessModes` | `PersistentVolumeClaim` Access Modes, must be `ReadWriteMany` | `[ReadWriteMany]` | -| `common.fsFileResourcePersistence.storageClassName` | Resource Persistent Volume Storage Class, must support the access mode: ReadWriteMany | `-` | -| `common.fsFileResourcePersistence.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | -| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | -| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` | -| `master.annotations` | The `annotations` for master server | `{}` | -| `master.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | -| `master.tolerations` | If specified, the pod's tolerations | `{}` | -| `master.resources` | The `resource` limit and request config for master server | `{}` | -| `master.configmap.DOLPHINSCHEDULER_OPTS` | The java options for master server | `""` | -| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread number | `100` | -| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` | -| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` | -| `master.configmap.MASTER_TASK_COMMIT_RETRYTIMES` | Master commit task retry times | `5` | -| `master.configmap.MASTER_TASK_COMMIT_INTERVAL` | Master commit task interval | `1000` | -| `master.configmap.MASTER_MAX_CPULOAD_AVG` | Only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 | `100` | -| `master.configmap.MASTER_RESERVED_MEMORY` | Only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G | `0.1` | -| `master.configmap.MASTER_LISTEN_PORT` | Master listen port | `5678` | -| `master.livenessProbe.enabled` | Turn on and off liveness probe | `true` | -| `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | -| `master.livenessProbe.periodSeconds` | How often to perform the probe | `30` | -| `master.livenessProbe.timeoutSeconds` | When the probe times out | `5` | -| `master.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `master.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `master.readinessProbe.enabled` | Turn on and off readiness probe | `true` | -| `master.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | -| `master.readinessProbe.periodSeconds` | How often to perform the probe | `30` | -| `master.readinessProbe.timeoutSeconds` | When the probe times out | `5` | -| `master.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `master.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `master.persistentVolumeClaim.enabled` | Set `master.persistentVolumeClaim.enabled` to `true` to mount a new volume for `master` | `false` | -| `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | -| `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | -| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | -| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` | -| `worker.annotations` | The `annotations` for worker server | `{}` | -| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | -| `worker.tolerations` | If specified, the pod's tolerations | `{}` | -| `worker.resources` | The `resource` limit and request config for worker server | `{}` | -| `worker.configmap.DOLPHINSCHEDULER_OPTS` | The java options for worker server | `""` | -| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread number | `100` | -| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` | -| `worker.configmap.WORKER_MAX_CPULOAD_AVG` | Only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 | `100` | -| `worker.configmap.WORKER_RESERVED_MEMORY` | Only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G | `0.1` | -| `worker.configmap.WORKER_LISTEN_PORT` | Worker listen port | `1234` | -| `worker.configmap.WORKER_GROUPS` | Worker groups | `default` | -| `worker.configmap.WORKER_HOST_WEIGHT` | Worker host weight | `100` | -| `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` | -| `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | -| `worker.livenessProbe.periodSeconds` | How often to perform the probe | `30` | -| `worker.livenessProbe.timeoutSeconds` | When the probe times out | `5` | -| `worker.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `worker.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `worker.readinessProbe.enabled` | Turn on and off readiness probe | `true` | -| `worker.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | -| `worker.readinessProbe.periodSeconds` | How often to perform the probe | `30` | -| `worker.readinessProbe.timeoutSeconds` | When the probe times out | `5` | -| `worker.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `worker.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `worker.persistentVolumeClaim.enabled` | Set `worker.persistentVolumeClaim.enabled` to `true` to enable `persistentVolumeClaim` for `worker` | `false` | -| `worker.persistentVolumeClaim.dataPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.dataPersistentVolume.enabled` to `true` to mount a data volume for `worker` | `false` | -| `worker.persistentVolumeClaim.dataPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | -| `worker.persistentVolumeClaim.dataPersistentVolume.storageClassName` | `Worker` data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `worker.persistentVolumeClaim.dataPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| `worker.persistentVolumeClaim.logsPersistentVolume.enabled` | Set `worker.persistentVolumeClaim.logsPersistentVolume.enabled` to `true` to mount a logs volume for `worker` | `false` | -| `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | -| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | -| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` | -| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | -| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | -| `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | -| `alert.annotations` | The `annotations` for alert server | `{}` | -| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | -| `alert.tolerations` | If specified, the pod's tolerations | `{}` | -| `alert.resources` | The `resource` limit and request config for alert server | `{}` | -| `alert.configmap.DOLPHINSCHEDULER_OPTS` | The java options for alert server | `""` | -| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin directory | `lib/plugin/alert` | -| `alert.livenessProbe.enabled` | Turn on and off liveness probe | `true` | -| `alert.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | -| `alert.livenessProbe.periodSeconds` | How often to perform the probe | `30` | -| `alert.livenessProbe.timeoutSeconds` | When the probe times out | `5` | -| `alert.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `alert.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `alert.readinessProbe.enabled` | Turn on and off readiness probe | `true` | -| `alert.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | -| `alert.readinessProbe.periodSeconds` | How often to perform the probe | `30` | -| `alert.readinessProbe.timeoutSeconds` | When the probe times out | `5` | -| `alert.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `alert.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `alert.persistentVolumeClaim.enabled` | Set `alert.persistentVolumeClaim.enabled` to `true` to mount a new volume for `alert` | `false` | -| `alert.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | -| `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | -| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` | -| `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | -| `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | -| `api.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | -| `api.annotations` | The `annotations` for api server | `{}` | -| `api.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | -| `api.tolerations` | If specified, the pod's tolerations | `{}` | -| `api.resources` | The `resource` limit and request config for api server | `{}` | -| `api.configmap.DOLPHINSCHEDULER_OPTS` | The java options for api server | `""` | -| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` | -| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | -| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` | -| `api.livenessProbe.timeoutSeconds` | When the probe times out | `5` | -| `api.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `api.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `api.readinessProbe.enabled` | Turn on and off readiness probe | `true` | -| `api.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` | -| `api.readinessProbe.periodSeconds` | How often to perform the probe | `30` | -| `api.readinessProbe.timeoutSeconds` | When the probe times out | `5` | -| `api.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` | -| `api.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` | -| `api.persistentVolumeClaim.enabled` | Set `api.persistentVolumeClaim.enabled` to `true` to mount a new volume for `api` | `false` | -| `api.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | -| `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | -| `ingress.enabled` | Enable ingress | `false` | -| `ingress.host` | Ingress host | `dolphinscheduler.org` | -| `ingress.path` | Ingress path | `/dolphinscheduler` | -| `ingress.tls.enabled` | Enable ingress tls | `false` | -| `ingress.tls.secretName` | Ingress tls secret name | `dolphinscheduler-tls` | - -## FAQ - -### How to use MySQL as the DolphinScheduler's database instead of PostgreSQL? - -> Because of the commercial license, we cannot directly use the driver and client of MySQL. -> -> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image as follows. - -1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`) - -2. Create a new `Dockerfile` to add MySQL driver and client: - -``` -FROM apache/dolphinscheduler:latest -COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib -RUN apk add --update --no-cache mysql-client -``` - -3. Build a new docker image including MySQL driver and client: - -``` -docker build -t apache/dolphinscheduler:mysql . -``` - -4. Push the docker image `apache/dolphinscheduler:mysql` to a docker registry - -5. Modify image `repository` and update `tag` to `mysql` in `values.yaml` - -6. Modify postgresql `enabled` to `false` - -7. Modify externalDatabase (especially modify `host`, `username` and `password`): - -``` -externalDatabase: - type: "mysql" - driver: "com.mysql.jdbc.Driver" - host: "localhost" - port: "3306" - username: "root" - password: "root" - database: "dolphinscheduler" - params: "useUnicode=true&characterEncoding=UTF-8" -``` - -8. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**) - -### How to support MySQL datasource in `Datasource manage`? - -> Because of the commercial license, we cannot directly use the driver of MySQL. -> -> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows. - -1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`) - -2. Create a new `Dockerfile` to add MySQL driver: - -``` -FROM apache/dolphinscheduler:latest -COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib -``` - -3. Build a new docker image including MySQL driver: - -``` -docker build -t apache/dolphinscheduler:mysql-driver . -``` - -4. Push the docker image `apache/dolphinscheduler:mysql-driver` to a docker registry - -5. Modify image `repository` and update `tag` to `mysql-driver` in `values.yaml` - -6. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**) - -7. Add a MySQL datasource in `Datasource manage` - -### How to support Oracle datasource in `Datasource manage`? - -> Because of the commercial license, we cannot directly use the driver of Oracle. -> -> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows. - -1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`) - -2. Create a new `Dockerfile` to add Oracle driver: - -``` -FROM apache/dolphinscheduler:latest -COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib -``` - -3. Build a new docker image including Oracle driver: - -``` -docker build -t apache/dolphinscheduler:oracle-driver . -``` - -4. Push the docker image `apache/dolphinscheduler:oracle-driver` to a docker registry - -5. Modify image `repository` and update `tag` to `oracle-driver` in `values.yaml` - -6. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**) - -7. Add a Oracle datasource in `Datasource manage` - -For more information please refer to the [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) documentation. diff --git a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt index 81a30d567..ac15c07b5 100644 --- a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt +++ b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt @@ -17,16 +17,34 @@ ** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed ** -Access DolphinScheduler by: +Access DolphinScheduler UI URL by: {{- if .Values.ingress.enabled }} - DolphinScheduler URL: http://{{ .Values.ingress.host }}/dolphinscheduler + DolphinScheduler UI URL: http{{ if .Values.ingress.tls.enabled }}s{{ end }}://{{ .Values.ingress.host }}/dolphinscheduler -{{- else }} +{{- else if eq .Values.api.service.type "ClusterIP" }} - kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345 + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345 - DolphinScheduler URL: http://127.0.0.1:12345/dolphinscheduler + DolphinScheduler UI URL: http://127.0.0.1:12345/dolphinscheduler + +{{- else if eq .Values.api.service.type "NodePort" }} + + NODE_IP=$(kubectl get no -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + NODE_PORT=$(kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}") + echo http://$NODE_IP:$NODE_PORT/dolphinscheduler + + DolphinScheduler UI URL: http://$NODE_IP:$NODE_PORT/dolphinscheduler + +{{- else if eq .Values.api.service.type "LoadBalancer" }} + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -w' + + SERVICE_IP=$(kubectl get svc {{ template "dolphinscheduler.fullname" . }}-api -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:12345/dolphinscheduler + + DolphinScheduler UI URL: http://$SERVICE_IP:12345/dolphinscheduler {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl index 21ee92e06..69ac6d13d 100644 --- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -30,7 +30,52 @@ If release name contains chart name it will be used as a full name. Create a default docker image fullname. */}} {{- define "dolphinscheduler.image.fullname" -}} -{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}} +{{- .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- end -}} + +{{/* +Create a default common labels. +*/}} +{{- define "dolphinscheduler.common.labels" -}} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/version: {{ .Chart.AppVersion }} +{{- end -}} + +{{/* +Create a master labels. +*/}} +{{- define "dolphinscheduler.master.labels" -}} +app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master +app.kubernetes.io/component: master +{{ include "dolphinscheduler.common.labels" . }} +{{- end -}} + +{{/* +Create a worker labels. +*/}} +{{- define "dolphinscheduler.worker.labels" -}} +app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker +app.kubernetes.io/component: worker +{{ include "dolphinscheduler.common.labels" . }} +{{- end -}} + +{{/* +Create an alert labels. +*/}} +{{- define "dolphinscheduler.alert.labels" -}} +app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert +app.kubernetes.io/component: alert +{{ include "dolphinscheduler.common.labels" . }} +{{- end -}} + +{{/* +Create an api labels. +*/}} +{{- define "dolphinscheduler.api.labels" -}} +app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api +app.kubernetes.io/component: api +{{ include "dolphinscheduler.common.labels" . }} {{- end -}} {{/* @@ -57,4 +102,135 @@ Create a default fully qualified zookkeeper quorum. {{- define "dolphinscheduler.zookeeper.quorum" -}} {{- $port := default "2181" (.Values.zookeeper.service.port | toString) -}} {{- printf "%s:%s" (include "dolphinscheduler.zookeeper.fullname" .) $port -}} +{{- end -}} + +{{/* +Create a database environment variables. +*/}} +{{- define "dolphinscheduler.database.env_vars" -}} +- name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} +- name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} +- name: DATABASE_HOST + {{- if .Values.postgresql.enabled }} + value: {{ template "dolphinscheduler.postgresql.fullname" . }} + {{- else }} + value: {{ .Values.externalDatabase.host | quote }} + {{- end }} +- name: DATABASE_PORT + {{- if .Values.postgresql.enabled }} + value: "5432" + {{- else }} + value: {{ .Values.externalDatabase.port | quote }} + {{- end }} +- name: DATABASE_USERNAME + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlUsername }} + {{- else }} + value: {{ .Values.externalDatabase.username | quote }} + {{- end }} +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + {{- if .Values.postgresql.enabled }} + name: {{ template "dolphinscheduler.postgresql.fullname" . }} + key: postgresql-password + {{- else }} + name: {{ include "dolphinscheduler.fullname" . }}-externaldb + key: database-password + {{- end }} +- name: DATABASE_DATABASE + {{- if .Values.postgresql.enabled }} + value: {{ .Values.postgresql.postgresqlDatabase }} + {{- else }} + value: {{ .Values.externalDatabase.database | quote }} + {{- end }} +- name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} +{{- end -}} + +{{/* +Create a zookeeper environment variables. +*/}} +{{- define "dolphinscheduler.zookeeper.env_vars" -}} +- name: ZOOKEEPER_QUORUM + {{- if .Values.zookeeper.enabled }} + value: {{ template "dolphinscheduler.zookeeper.quorum" . }} + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperQuorum }} + {{- end }} +- name: ZOOKEEPER_ROOT + {{- if .Values.zookeeper.enabled }} + value: {{ .Values.zookeeper.zookeeperRoot }} + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperRoot }} + {{- end }} +{{- end -}} + +{{/* +Create a common fs_s3a environment variables. +*/}} +{{- define "dolphinscheduler.fs_s3a.env_vars" -}} +{{- if eq (default "HDFS" .Values.common.configmap.RESOURCE_STORAGE_TYPE) "S3" -}} +- name: FS_S3A_SECRET_KEY + valueFrom: + secretKeyRef: + key: fs-s3a-secret-key + name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a +{{- end -}} +{{- end -}} + +{{/* +Create a sharedStoragePersistence volume. +*/}} +{{- define "dolphinscheduler.sharedStorage.volume" -}} +{{- if .Values.common.sharedStoragePersistence.enabled -}} +- name: {{ include "dolphinscheduler.fullname" . }}-shared + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-shared +{{- end -}} +{{- end -}} + +{{/* +Create a sharedStoragePersistence volumeMount. +*/}} +{{- define "dolphinscheduler.sharedStorage.volumeMount" -}} +{{- if .Values.common.sharedStoragePersistence.enabled -}} +- mountPath: {{ .Values.common.sharedStoragePersistence.mountPath | quote }} + name: {{ include "dolphinscheduler.fullname" . }}-shared +{{- end -}} +{{- end -}} + +{{/* +Create a fsFileResourcePersistence volume. +*/}} +{{- define "dolphinscheduler.fsFileResource.volume" -}} +{{- if .Values.common.fsFileResourcePersistence.enabled -}} +- name: {{ include "dolphinscheduler.fullname" . }}-fs-file + persistentVolumeClaim: + claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file +{{- end -}} +{{- end -}} + +{{/* +Create a fsFileResourcePersistence volumeMount. +*/}} +{{- define "dolphinscheduler.fsFileResource.volumeMount" -}} +{{- if .Values.common.fsFileResourcePersistence.enabled -}} +- mountPath: {{ default "/dolphinscheduler" .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} + name: {{ include "dolphinscheduler.fullname" . }}-fs-file +{{- end -}} {{- end -}} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml index b5ffadd4b..9ef96fbdd 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml @@ -21,9 +21,9 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-alert labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} data: - DOLPHINSCHEDULER_OPTS: {{ .Values.alert.configmap.DOLPHINSCHEDULER_OPTS | quote }} - ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }} + {{- range $key, $value := .Values.alert.configmap }} + {{ $key }}: {{ $value | quote }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml index 3845f1f67..70b818fc4 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml @@ -21,8 +21,9 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-api labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} data: - DOLPHINSCHEDULER_OPTS: {{ .Values.api.configmap.DOLPHINSCHEDULER_OPTS | quote }} + {{- range $key, $value := .Values.api.configmap }} + {{ $key }}: {{ $value | quote }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml index f350055a3..aa8042b0f 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml @@ -21,17 +21,9 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-common labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} data: - DOLPHINSCHEDULER_ENV: |- - {{- range .Values.common.configmap.DOLPHINSCHEDULER_ENV }} - {{ . }} + {{- range $key, $value := (omit .Values.common.configmap "FS_S3A_SECRET_KEY") }} + {{ $key }}: {{ $value | quote }} {{- end }} - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} - RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }} - RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} - FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }} - FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }} - FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml index 7c1be7717..34823f66a 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml @@ -21,16 +21,9 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-master labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} data: - DOLPHINSCHEDULER_OPTS: {{ .Values.master.configmap.DOLPHINSCHEDULER_OPTS | quote }} - MASTER_EXEC_THREADS: {{ .Values.master.configmap.MASTER_EXEC_THREADS | quote }} - MASTER_EXEC_TASK_NUM: {{ .Values.master.configmap.MASTER_EXEC_TASK_NUM | quote }} - MASTER_HEARTBEAT_INTERVAL: {{ .Values.master.configmap.MASTER_HEARTBEAT_INTERVAL | quote }} - MASTER_TASK_COMMIT_RETRYTIMES: {{ .Values.master.configmap.MASTER_TASK_COMMIT_RETRYTIMES | quote }} - MASTER_TASK_COMMIT_INTERVAL: {{ .Values.master.configmap.MASTER_TASK_COMMIT_INTERVAL | quote }} - MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }} - MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }} - MASTER_LISTEN_PORT: {{ .Values.master.configmap.MASTER_LISTEN_PORT | quote }} + {{- range $key, $value := .Values.master.configmap }} + {{ $key }}: {{ $value | quote }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml index 61d208e8f..68a292d61 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml @@ -21,15 +21,9 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-worker labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} data: - DOLPHINSCHEDULER_OPTS: {{ .Values.worker.configmap.DOLPHINSCHEDULER_OPTS | quote }} - WORKER_EXEC_THREADS: {{ .Values.worker.configmap.WORKER_EXEC_THREADS | quote }} - WORKER_HEARTBEAT_INTERVAL: {{ .Values.worker.configmap.WORKER_HEARTBEAT_INTERVAL | quote }} - WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }} - WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }} - WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }} - WORKER_GROUPS: {{ .Values.worker.configmap.WORKER_GROUPS | quote }} - WORKER_HOST_WEIGHT: {{ .Values.worker.configmap.WORKER_HOST_WEIGHT | quote }} + {{- range $key, $value := .Values.worker.configmap }} + {{ $key }}: {{ $value | quote }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index 35e2e4a07..70e292a11 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -19,18 +19,12 @@ kind: Deployment metadata: name: {{ include "dolphinscheduler.fullname" . }}-alert labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: alert + {{- include "dolphinscheduler.alert.labels" . | nindent 4 }} spec: replicas: {{ .Values.alert.replicas }} selector: matchLabels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: alert + {{- include "dolphinscheduler.alert.labels" . | nindent 6 }} strategy: type: {{ .Values.alert.strategy.type | quote }} rollingUpdate: @@ -39,23 +33,23 @@ spec: template: metadata: labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: alert + {{- include "dolphinscheduler.alert.labels" . | nindent 8 }} {{- if .Values.alert.annotations }} annotations: {{- toYaml .Values.alert.annotations | nindent 8 }} {{- end }} spec: {{- if .Values.alert.affinity }} - affinity: {{- toYaml .Values.alert.affinity | nindent 8 }} + affinity: + {{- toYaml .Values.alert.affinity | nindent 8 }} {{- end }} {{- if .Values.alert.nodeSelector }} - nodeSelector: {{- toYaml .Values.alert.nodeSelector | nindent 8 }} + nodeSelector: + {{- toYaml .Values.alert.nodeSelector | nindent 8 }} {{- end }} {{- if .Values.alert.tolerations }} - tolerations: {{- toYaml . | nindent 8 }} + tolerations: + {{- toYaml .Values.alert.tolerations | nindent 8 }} {{- end }} {{- if .Values.image.pullSecret }} imagePullSecrets: @@ -73,84 +67,20 @@ spec: env: - name: TZ value: {{ .Values.timezone }} - - name: DOLPHINSCHEDULER_OPTS - valueFrom: - configMapKeyRef: - key: DOLPHINSCHEDULER_OPTS - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: ALERT_PLUGIN_DIR - valueFrom: - configMapKeyRef: - key: ALERT_PLUGIN_DIR - name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: DATABASE_TYPE - {{- if .Values.postgresql.enabled }} - value: "postgresql" - {{- else }} - value: {{ .Values.externalDatabase.type | quote }} - {{- end }} - - name: DATABASE_DRIVER - {{- if .Values.postgresql.enabled }} - value: "org.postgresql.Driver" - {{- else }} - value: {{ .Values.externalDatabase.driver | quote }} - {{- end }} - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} - - name: DATABASE_USERNAME - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlUsername }} - {{- else }} - value: {{ .Values.externalDatabase.username | quote }} - {{- end }} - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ template "dolphinscheduler.postgresql.fullname" . }} - key: postgresql-password - {{- else }} - name: {{ include "dolphinscheduler.fullname" . }}-externaldb - key: database-password - {{- end }} - - name: DATABASE_DATABASE - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlDatabase }} - {{- else }} - value: {{ .Values.externalDatabase.database | quote }} - {{- end }} - - name: DATABASE_PARAMS - {{- if .Values.postgresql.enabled }} - value: "characterEncoding=utf8" - {{- else }} - value: {{ .Values.externalDatabase.params | quote }} - {{- end }} + {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-common + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-alert {{- if .Values.alert.resources }} resources: - limits: - memory: {{ .Values.alert.resources.limits.memory }} - cpu: {{ .Values.alert.resources.limits.cpu }} - requests: - memory: {{ .Values.alert.resources.requests.memory }} - cpu: {{ .Values.alert.resources.requests.cpu }} + {{- toYaml .Values.alert.resources | nindent 12 }} {{- end }} {{- if .Values.alert.livenessProbe.enabled }} livenessProbe: exec: - command: - - sh - - /root/checkpoint.sh - - AlertServer + command: ["bash", "/root/checkpoint.sh", "AlertServer"] initialDelaySeconds: {{ .Values.alert.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.alert.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.alert.livenessProbe.timeoutSeconds }} @@ -160,10 +90,7 @@ spec: {{- if .Values.alert.readinessProbe.enabled }} readinessProbe: exec: - command: - - sh - - /root/checkpoint.sh - - AlertServer + command: ["bash", "/root/checkpoint.sh", "AlertServer"] initialDelaySeconds: {{ .Values.alert.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.alert.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.alert.readinessProbe.timeoutSeconds }} diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 1dbc513d9..65b5dea81 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -19,18 +19,12 @@ kind: Deployment metadata: name: {{ include "dolphinscheduler.fullname" . }}-api labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: api + {{- include "dolphinscheduler.api.labels" . | nindent 4 }} spec: replicas: {{ .Values.api.replicas }} selector: matchLabels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: api + {{- include "dolphinscheduler.api.labels" . | nindent 6 }} strategy: type: {{ .Values.api.strategy.type | quote }} rollingUpdate: @@ -39,23 +33,23 @@ spec: template: metadata: labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: api - {{- if .Values.alert.annotations }} + {{- include "dolphinscheduler.api.labels" . | nindent 8 }} + {{- if .Values.api.annotations }} annotations: - {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- toYaml .Values.api.annotations | nindent 8 }} {{- end }} spec: {{- if .Values.api.affinity }} - affinity: {{- toYaml .Values.api.affinity | nindent 8 }} + affinity: + {{- toYaml .Values.api.affinity | nindent 8 }} {{- end }} {{- if .Values.api.nodeSelector }} - nodeSelector: {{- toYaml .Values.api.nodeSelector | nindent 8 }} + nodeSelector: + {{- toYaml .Values.api.nodeSelector | nindent 8 }} {{- end }} {{- if .Values.api.tolerations }} - tolerations: {{- toYaml . | nindent 8 }} + tolerations: + {{- toYaml .Values.api.tolerations | nindent 8 }} {{- end }} {{- if .Values.image.pullSecret }} imagePullSecrets: @@ -73,120 +67,22 @@ spec: env: - name: TZ value: {{ .Values.timezone }} - - name: DOLPHINSCHEDULER_OPTS - valueFrom: - configMapKeyRef: - key: DOLPHINSCHEDULER_OPTS - name: {{ include "dolphinscheduler.fullname" . }}-api - - name: DATABASE_TYPE - {{- if .Values.postgresql.enabled }} - value: "postgresql" - {{- else }} - value: {{ .Values.externalDatabase.type | quote }} - {{- end }} - - name: DATABASE_DRIVER - {{- if .Values.postgresql.enabled }} - value: "org.postgresql.Driver" - {{- else }} - value: {{ .Values.externalDatabase.driver | quote }} - {{- end }} - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} - - name: DATABASE_USERNAME - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlUsername }} - {{- else }} - value: {{ .Values.externalDatabase.username | quote }} - {{- end }} - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ template "dolphinscheduler.postgresql.fullname" . }} - key: postgresql-password - {{- else }} - name: {{ include "dolphinscheduler.fullname" . }}-externaldb - key: database-password - {{- end }} - - name: DATABASE_DATABASE - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlDatabase }} - {{- else }} - value: {{ .Values.externalDatabase.database | quote }} - {{- end }} - - name: DATABASE_PARAMS - {{- if .Values.postgresql.enabled }} - value: "characterEncoding=utf8" - {{- else }} - value: {{ .Values.externalDatabase.params | quote }} - {{- end }} - - name: ZOOKEEPER_QUORUM - {{- if .Values.zookeeper.enabled }} - value: {{ template "dolphinscheduler.zookeeper.quorum" . }} - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperQuorum }} - {{- end }} - - name: ZOOKEEPER_ROOT - {{- if .Values.zookeeper.enabled }} - value: {{ .Values.zookeeper.zookeeperRoot }} - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperRoot }} - {{- end }} - - name: RESOURCE_STORAGE_TYPE - valueFrom: - configMapKeyRef: - key: RESOURCE_STORAGE_TYPE - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: RESOURCE_UPLOAD_PATH - valueFrom: - configMapKeyRef: - key: RESOURCE_UPLOAD_PATH - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_DEFAULT_FS - valueFrom: - configMapKeyRef: - key: FS_DEFAULT_FS - name: {{ include "dolphinscheduler.fullname" . }}-common - {{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} - - name: FS_S3A_ENDPOINT - valueFrom: - configMapKeyRef: - key: FS_S3A_ENDPOINT - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ACCESS_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_ACCESS_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_SECRET_KEY - valueFrom: - secretKeyRef: - key: fs-s3a-secret-key - name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a - {{- end }} + {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} + {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }} + {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-common + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-api {{- if .Values.api.resources }} resources: - limits: - memory: {{ .Values.api.resources.limits.memory | quote }} - cpu: {{ .Values.api.resources.limits.cpu | quote }} - requests: - memory: {{ .Values.api.resources.requests.memory | quote }} - cpu: {{ .Values.api.resources.requests.cpu | quote }} + {{- toYaml .Values.api.resources | nindent 12 }} {{- end }} {{- if .Values.api.livenessProbe.enabled }} livenessProbe: - tcpSocket: - port: 12345 + exec: + command: ["bash", "/root/checkpoint.sh", "ApiApplicationServer"] initialDelaySeconds: {{ .Values.api.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.api.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.api.livenessProbe.timeoutSeconds }} @@ -195,8 +91,8 @@ spec: {{- end }} {{- if .Values.api.readinessProbe.enabled }} readinessProbe: - tcpSocket: - port: 12345 + exec: + command: ["bash", "/root/checkpoint.sh", "ApiApplicationServer"] initialDelaySeconds: {{ .Values.api.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.api.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.api.readinessProbe.timeoutSeconds }} @@ -206,10 +102,8 @@ spec: volumeMounts: - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-api - {{- if .Values.common.fsFileResourcePersistence.enabled }} - - mountPath: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} - name: {{ include "dolphinscheduler.fullname" . }}-fs-file - {{- end }} + {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} + {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }} volumes: - name: {{ include "dolphinscheduler.fullname" . }}-api {{- if .Values.api.persistentVolumeClaim.enabled }} @@ -218,8 +112,5 @@ spec: {{- else }} emptyDir: {} {{- end }} - {{- if .Values.common.fsFileResourcePersistence.enabled }} - - name: {{ include "dolphinscheduler.fullname" . }}-fs-file - persistentVolumeClaim: - claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file - {{- end }} \ No newline at end of file + {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} + {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml index 4f76f2b30..7a8d6ac8b 100644 --- a/docker/kubernetes/dolphinscheduler/templates/ingress.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/ingress.yaml @@ -27,8 +27,7 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }} labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} spec: rules: - host: {{ .Values.ingress.host }} diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml index fd5e88afb..6c803ad8b 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml @@ -21,8 +21,7 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-alert labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} spec: accessModes: {{- range .Values.alert.persistentVolumeClaim.accessModes }} diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml index 48292966a..032c72fe7 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml @@ -21,8 +21,7 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-api labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} spec: accessModes: {{- range .Values.api.persistentVolumeClaim.accessModes }} diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml index 1e277eb06..8c99f42b8 100644 --- a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-fs-file.yaml @@ -21,8 +21,7 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-fs-file labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-file - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} annotations: "helm.sh/resource-policy": keep spec: diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml new file mode 100644 index 000000000..4bccdb119 --- /dev/null +++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-shared.yaml @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.common.sharedStoragePersistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-shared + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-shared + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + annotations: + "helm.sh/resource-policy": keep +spec: + accessModes: + {{- range .Values.common.sharedStoragePersistence.accessModes }} + - {{ . | quote }} + {{- end }} + storageClassName: {{ .Values.common.sharedStoragePersistence.storageClassName | quote }} + resources: + requests: + storage: {{ .Values.common.sharedStoragePersistence.storage | quote }} +{{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml b/docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml index c15ef14ff..4d299efbc 100644 --- a/docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/secret-external-database.yaml @@ -21,8 +21,7 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-externaldb labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-externaldb - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} type: Opaque data: database-password: {{ .Values.externalDatabase.password | b64enc | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml b/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml index 722d31fd2..5ba2c5038 100644 --- a/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml @@ -14,15 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} +{{- if eq (default "HDFS" .Values.common.configmap.RESOURCE_STORAGE_TYPE) "S3" }} apiVersion: v1 kind: Secret metadata: name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} type: Opaque data: fs-s3a-secret-key: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | b64enc | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 72b69f19e..735d3cd08 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -19,40 +19,34 @@ kind: StatefulSet metadata: name: {{ include "dolphinscheduler.fullname" . }}-master labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: master + {{- include "dolphinscheduler.master.labels" . | nindent 4 }} spec: podManagementPolicy: {{ .Values.master.podManagementPolicy }} replicas: {{ .Values.master.replicas }} selector: matchLabels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: master + {{- include "dolphinscheduler.master.labels" . | nindent 6 }} serviceName: {{ template "dolphinscheduler.fullname" . }}-master-headless template: metadata: labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: master - {{- if .Values.alert.annotations }} + {{- include "dolphinscheduler.master.labels" . | nindent 8 }} + {{- if .Values.master.annotations }} annotations: - {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- toYaml .Values.master.annotations | nindent 8 }} {{- end }} spec: {{- if .Values.master.affinity }} - affinity: {{- toYaml .Values.master.affinity | nindent 8 }} + affinity: + {{- toYaml .Values.master.affinity | nindent 8 }} {{- end }} {{- if .Values.master.nodeSelector }} - nodeSelector: {{- toYaml .Values.master.nodeSelector | nindent 8 }} + nodeSelector: + {{- toYaml .Values.master.nodeSelector | nindent 8 }} {{- end }} {{- if .Values.master.tolerations }} - tolerations: {{- toYaml . | nindent 8 }} + tolerations: + {{- toYaml .Values.master.tolerations | nindent 8 }} {{- end }} {{- if .Values.image.pullSecret }} imagePullSecrets: @@ -65,141 +59,27 @@ spec: args: - "master-server" ports: - - containerPort: {{ .Values.master.configmap.MASTER_LISTEN_PORT }} + - containerPort: 5678 name: "master-port" env: - name: TZ value: {{ .Values.timezone }} - - name: DOLPHINSCHEDULER_OPTS - valueFrom: - configMapKeyRef: - key: DOLPHINSCHEDULER_OPTS - name: {{ include "dolphinscheduler.fullname" . }}-master - - name: MASTER_EXEC_THREADS - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master - key: MASTER_EXEC_THREADS - - name: MASTER_EXEC_TASK_NUM - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master - key: MASTER_EXEC_TASK_NUM - - name: MASTER_HEARTBEAT_INTERVAL - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master - key: MASTER_HEARTBEAT_INTERVAL - - name: MASTER_TASK_COMMIT_RETRYTIMES - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master - key: MASTER_TASK_COMMIT_RETRYTIMES - - name: MASTER_TASK_COMMIT_INTERVAL - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master - key: MASTER_TASK_COMMIT_INTERVAL - - name: MASTER_MAX_CPULOAD_AVG - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master - key: MASTER_MAX_CPULOAD_AVG - - name: MASTER_RESERVED_MEMORY - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master - key: MASTER_RESERVED_MEMORY - - name: MASTER_LISTEN_PORT - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-master - key: MASTER_LISTEN_PORT - - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-common - key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - - name: DATABASE_TYPE - {{- if .Values.postgresql.enabled }} - value: "postgresql" - {{- else }} - value: {{ .Values.externalDatabase.type | quote }} - {{- end }} - - name: DATABASE_DRIVER - {{- if .Values.postgresql.enabled }} - value: "org.postgresql.Driver" - {{- else }} - value: {{ .Values.externalDatabase.driver | quote }} - {{- end }} - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} - - name: DATABASE_USERNAME - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlUsername }} - {{- else }} - value: {{ .Values.externalDatabase.username | quote }} - {{- end }} - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ template "dolphinscheduler.postgresql.fullname" . }} - key: postgresql-password - {{- else }} - name: {{ include "dolphinscheduler.fullname" . }}-externaldb - key: database-password - {{- end }} - - name: DATABASE_DATABASE - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlDatabase }} - {{- else }} - value: {{ .Values.externalDatabase.database | quote }} - {{- end }} - - name: DATABASE_PARAMS - {{- if .Values.postgresql.enabled }} - value: "characterEncoding=utf8" - {{- else }} - value: {{ .Values.externalDatabase.params | quote }} - {{- end }} - - name: ZOOKEEPER_QUORUM - {{- if .Values.zookeeper.enabled }} - value: {{ template "dolphinscheduler.zookeeper.quorum" . }} - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperQuorum }} - {{- end }} - - name: ZOOKEEPER_ROOT - {{- if .Values.zookeeper.enabled }} - value: {{ .Values.zookeeper.zookeeperRoot }} - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperRoot }} - {{- end }} + {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} + {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }} + {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-common + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-master {{- if .Values.master.resources }} resources: - limits: - memory: {{ .Values.master.resources.limits.memory | quote }} - cpu: {{ .Values.master.resources.limits.cpu | quote }} - requests: - memory: {{ .Values.master.resources.requests.memory | quote }} - cpu: {{ .Values.master.resources.requests.cpu | quote }} + {{- toYaml .Values.master.resources | nindent 12 }} {{- end }} {{- if .Values.master.livenessProbe.enabled }} livenessProbe: exec: - command: - - sh - - /root/checkpoint.sh - - MasterServer + command: ["bash", "/root/checkpoint.sh", "MasterServer"] initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }} @@ -209,10 +89,7 @@ spec: {{- if .Values.master.readinessProbe.enabled }} readinessProbe: exec: - command: - - sh - - /root/checkpoint.sh - - MasterServer + command: ["bash", "/root/checkpoint.sh", "MasterServer"] initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.master.readinessProbe.timeoutSeconds }} @@ -222,6 +99,7 @@ spec: volumeMounts: - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-master + {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} volumes: - name: {{ include "dolphinscheduler.fullname" . }}-master {{- if .Values.master.persistentVolumeClaim.enabled }} @@ -230,19 +108,19 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} {{- if .Values.master.persistentVolumeClaim.enabled }} volumeClaimTemplates: - metadata: name: {{ include "dolphinscheduler.fullname" . }}-master labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: - {{- range .Values.master.persistentVolumeClaim.accessModes }} + {{- range .Values.master.persistentVolumeClaim.accessModes }} - {{ . | quote }} - {{- end }} + {{- end }} storageClassName: {{ .Values.master.persistentVolumeClaim.storageClassName | quote }} resources: requests: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index d616b5421..60b67538b 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -19,40 +19,34 @@ kind: StatefulSet metadata: name: {{ include "dolphinscheduler.fullname" . }}-worker labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: worker + {{- include "dolphinscheduler.worker.labels" . | nindent 4 }} spec: podManagementPolicy: {{ .Values.worker.podManagementPolicy }} replicas: {{ .Values.worker.replicas }} selector: matchLabels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: worker + {{- include "dolphinscheduler.worker.labels" . | nindent 6 }} serviceName: {{ template "dolphinscheduler.fullname" . }}-worker-headless template: metadata: labels: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: worker - {{- if .Values.alert.annotations }} + {{- include "dolphinscheduler.worker.labels" . | nindent 8 }} + {{- if .Values.worker.annotations }} annotations: - {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- toYaml .Values.worker.annotations | nindent 8 }} {{- end }} spec: {{- if .Values.worker.affinity }} - affinity: {{- toYaml .Values.worker.affinity | nindent 8 }} + affinity: + {{- toYaml .Values.worker.affinity | nindent 8 }} {{- end }} {{- if .Values.worker.nodeSelector }} - nodeSelector: {{- toYaml .Values.worker.nodeSelector | nindent 8 }} + nodeSelector: + {{- toYaml .Values.worker.nodeSelector | nindent 8 }} {{- end }} {{- if .Values.worker.tolerations }} - tolerations: {{- toYaml . | nindent 8 }} + tolerations: + {{- toYaml .Values.worker.tolerations | nindent 8 }} {{- end }} {{- if .Values.image.pullSecret }} imagePullSecrets: @@ -65,172 +59,33 @@ spec: args: - "worker-server" ports: - - containerPort: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }} + - containerPort: 1234 name: "worker-port" - containerPort: 50051 name: "logger-port" env: - name: TZ value: {{ .Values.timezone }} - - name: DOLPHINSCHEDULER_OPTS - valueFrom: - configMapKeyRef: - key: DOLPHINSCHEDULER_OPTS - name: {{ include "dolphinscheduler.fullname" . }}-worker - - name: WORKER_EXEC_THREADS - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_EXEC_THREADS - - name: WORKER_HEARTBEAT_INTERVAL - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_HEARTBEAT_INTERVAL - - name: WORKER_MAX_CPULOAD_AVG - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_MAX_CPULOAD_AVG - - name: WORKER_RESERVED_MEMORY - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_RESERVED_MEMORY - - name: WORKER_LISTEN_PORT - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_LISTEN_PORT - - name: WORKER_GROUPS - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_GROUPS - - name: WORKER_WEUGHT - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-worker - key: WORKER_HOST_WEIGHT - - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - valueFrom: - configMapKeyRef: - name: {{ include "dolphinscheduler.fullname" . }}-common - key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - name: ALERT_LISTEN_HOST value: {{ include "dolphinscheduler.fullname" . }}-alert - - name: DATABASE_TYPE - {{- if .Values.postgresql.enabled }} - value: "postgresql" - {{- else }} - value: {{ .Values.externalDatabase.type | quote }} - {{- end }} - - name: DATABASE_DRIVER - {{- if .Values.postgresql.enabled }} - value: "org.postgresql.Driver" - {{- else }} - value: {{ .Values.externalDatabase.driver | quote }} - {{- end }} - - name: DATABASE_HOST - {{- if .Values.postgresql.enabled }} - value: {{ template "dolphinscheduler.postgresql.fullname" . }} - {{- else }} - value: {{ .Values.externalDatabase.host | quote }} - {{- end }} - - name: DATABASE_PORT - {{- if .Values.postgresql.enabled }} - value: "5432" - {{- else }} - value: {{ .Values.externalDatabase.port | quote }} - {{- end }} - - name: DATABASE_USERNAME - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlUsername }} - {{- else }} - value: {{ .Values.externalDatabase.username | quote }} - {{- end }} - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - {{- if .Values.postgresql.enabled }} - name: {{ template "dolphinscheduler.postgresql.fullname" . }} - key: postgresql-password - {{- else }} - name: {{ include "dolphinscheduler.fullname" . }}-externaldb - key: database-password - {{- end }} - - name: DATABASE_DATABASE - {{- if .Values.postgresql.enabled }} - value: {{ .Values.postgresql.postgresqlDatabase }} - {{- else }} - value: {{ .Values.externalDatabase.database | quote }} - {{- end }} - - name: DATABASE_PARAMS - {{- if .Values.postgresql.enabled }} - value: "characterEncoding=utf8" - {{- else }} - value: {{ .Values.externalDatabase.params | quote }} - {{- end }} - - name: ZOOKEEPER_QUORUM - {{- if .Values.zookeeper.enabled }} - value: {{ template "dolphinscheduler.zookeeper.quorum" . }} - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperQuorum }} - {{- end }} - - name: ZOOKEEPER_ROOT - {{- if .Values.zookeeper.enabled }} - value: {{ .Values.zookeeper.zookeeperRoot }} - {{- else }} - value: {{ .Values.externalZookeeper.zookeeperRoot }} - {{- end }} - - name: RESOURCE_STORAGE_TYPE - valueFrom: - configMapKeyRef: - key: RESOURCE_STORAGE_TYPE - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: RESOURCE_UPLOAD_PATH - valueFrom: - configMapKeyRef: - key: RESOURCE_UPLOAD_PATH - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_DEFAULT_FS - valueFrom: - configMapKeyRef: - key: FS_DEFAULT_FS - name: {{ include "dolphinscheduler.fullname" . }}-common - {{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }} - - name: FS_S3A_ENDPOINT - valueFrom: - configMapKeyRef: - key: FS_S3A_ENDPOINT - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_ACCESS_KEY - valueFrom: - configMapKeyRef: - key: FS_S3A_ACCESS_KEY - name: {{ include "dolphinscheduler.fullname" . }}-common - - name: FS_S3A_SECRET_KEY - valueFrom: - secretKeyRef: - key: fs-s3a-secret-key - name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a - {{- end }} + {{- include "dolphinscheduler.database.env_vars" . | nindent 12 }} + {{- include "dolphinscheduler.zookeeper.env_vars" . | nindent 12 }} + {{- include "dolphinscheduler.fs_s3a.env_vars" . | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-common + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-worker + - configMapRef: + name: {{ include "dolphinscheduler.fullname" . }}-alert {{- if .Values.worker.resources }} resources: - limits: - memory: {{ .Values.worker.resources.limits.memory | quote }} - cpu: {{ .Values.worker.resources.limits.cpu | quote }} - requests: - memory: {{ .Values.worker.resources.requests.memory | quote }} - cpu: {{ .Values.worker.resources.requests.cpu | quote }} + {{- toYaml .Values.worker.resources | nindent 12 }} {{- end }} {{- if .Values.worker.livenessProbe.enabled }} livenessProbe: exec: - command: - - sh - - /root/checkpoint.sh - - WorkerServer + command: ["bash", "/root/checkpoint.sh", "WorkerServer"] initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }} @@ -240,10 +95,7 @@ spec: {{- if .Values.worker.readinessProbe.enabled }} readinessProbe: exec: - command: - - sh - - /root/checkpoint.sh - - WorkerServer + command: ["bash", "/root/checkpoint.sh", "WorkerServer"] initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} @@ -251,17 +103,12 @@ spec: failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} {{- end }} volumeMounts: - - mountPath: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} + - mountPath: {{ default "/tmp/dolphinscheduler" .Values.common.configmap.DATA_BASEDIR_PATH | quote }} name: {{ include "dolphinscheduler.fullname" . }}-worker-data - mountPath: "/opt/dolphinscheduler/logs" name: {{ include "dolphinscheduler.fullname" . }}-worker-logs - - mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh" - subPath: "dolphinscheduler_env.sh" - name: {{ include "dolphinscheduler.fullname" . }}-common-env - {{- if .Values.common.fsFileResourcePersistence.enabled }} - - mountPath: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} - name: {{ include "dolphinscheduler.fullname" . }}-fs-file - {{- end }} + {{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }} + {{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }} volumes: - name: {{ include "dolphinscheduler.fullname" . }}-worker-data {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} @@ -277,18 +124,8 @@ spec: {{- else }} emptyDir: {} {{- end }} - - name: {{ include "dolphinscheduler.fullname" . }}-common-env - configMap: - defaultMode: 0777 - name: {{ include "dolphinscheduler.fullname" . }}-common - items: - - key: DOLPHINSCHEDULER_ENV - path: dolphinscheduler_env.sh - {{- if .Values.common.fsFileResourcePersistence.enabled }} - - name: {{ include "dolphinscheduler.fullname" . }}-fs-file - persistentVolumeClaim: - claimName: {{ include "dolphinscheduler.fullname" . }}-fs-file - {{- end }} + {{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }} + {{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }} {{- if .Values.worker.persistentVolumeClaim.enabled }} volumeClaimTemplates: {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} @@ -296,8 +133,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-data labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-data - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: {{- range .Values.worker.persistentVolumeClaim.dataPersistentVolume.accessModes }} @@ -313,8 +149,7 @@ spec: name: {{ include "dolphinscheduler.fullname" . }}-worker-logs labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-logs - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 10 }} spec: accessModes: {{- range .Values.worker.persistentVolumeClaim.logsPersistentVolume.accessModes }} diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml index 404c2e482..a24dfe6f6 100644 --- a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-alert.yaml @@ -20,8 +20,7 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-alert labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} spec: ports: - port: 50052 @@ -29,7 +28,4 @@ spec: protocol: TCP name: alert-port selector: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: alert \ No newline at end of file + {{- include "dolphinscheduler.alert.labels" . | nindent 4 }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml index 5daa7c47d..8d7fe8037 100644 --- a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-api.yaml @@ -20,16 +20,35 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-api labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} + {{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.annotations }} + annotations: + {{- range $key, $value := .Values.api.service.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} spec: + type: {{ .Values.api.service.type }} + {{- if and (eq .Values.api.service.type "ClusterIP") .Values.api.service.clusterIP }} + clusterIP: {{ .Values.api.service.clusterIP }} + {{- end }} ports: - port: 12345 targetPort: api-port + {{- if and (eq .Values.api.service.type "NodePort") .Values.api.service.nodePort }} + nodePort: {{ .Values.api.service.nodePort }} + {{- end }} protocol: TCP name: api-port + {{- if .Values.api.service.externalIPs }} + externalIPs: + {{- toYaml .Values.api.service.externalIPs | nindent 4 }} + {{- end }} + {{- if and (eq .Values.api.service.type "ExternalName") .Values.api.service.externalName }} + externalName: {{ .Values.api.service.externalName }} + {{- end }} + {{- if and (eq .Values.api.service.type "LoadBalancer") .Values.api.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.api.service.loadBalancerIP }} + {{- end }} selector: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: api \ No newline at end of file + {{- include "dolphinscheduler.api.labels" . | nindent 4 }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml index 0df9d0172..9fb21ca66 100644 --- a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-master-headless.yaml @@ -20,17 +20,13 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-master-headless labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master-headless - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} spec: clusterIP: "None" ports: - - port: {{ .Values.master.configmap.MASTER_LISTEN_PORT }} + - port: 5678 targetPort: master-port protocol: TCP name: master-port selector: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: master \ No newline at end of file + {{- include "dolphinscheduler.master.labels" . | nindent 4 }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml index 5dd5478e2..34664fefb 100644 --- a/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml @@ -20,12 +20,11 @@ metadata: name: {{ include "dolphinscheduler.fullname" . }}-worker-headless labels: app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker-headless - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "dolphinscheduler.common.labels" . | nindent 4 }} spec: clusterIP: "None" ports: - - port: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }} + - port: 1234 targetPort: worker-port protocol: TCP name: worker-port @@ -34,7 +33,4 @@ spec: protocol: TCP name: logger-port selector: - app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/component: worker \ No newline at end of file + {{- include "dolphinscheduler.worker.labels" . | nindent 4 }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index af7da4623..aa64b61e4 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -23,7 +23,7 @@ timezone: "Asia/Shanghai" image: repository: "apache/dolphinscheduler" - tag: "latest" + tag: "1.3.6-SNAPSHOT" pullPolicy: "IfNotPresent" pullSecret: "" @@ -54,8 +54,6 @@ externalDatabase: zookeeper: enabled: true fourlwCommandsWhitelist: "srvr,ruok,wchs,cons" - service: - port: "2181" persistence: enabled: false size: "20Gi" @@ -69,26 +67,48 @@ externalZookeeper: zookeeperRoot: "/dolphinscheduler" common: - ## ConfigMap + ## Configmap configmap: - DOLPHINSCHEDULER_ENV: - - "export HADOOP_HOME=/opt/soft/hadoop" - - "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop" - - "export SPARK_HOME1=/opt/soft/spark1" - - "export SPARK_HOME2=/opt/soft/spark2" - #- "export PYTHON_HOME=/opt/soft/python" - - "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk" - - "export HIVE_HOME=/opt/soft/hive" - - "export FLINK_HOME=/opt/soft/flink" - - "export DATAX_HOME=/opt/soft/datax" - - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH" - DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" + DOLPHINSCHEDULER_OPTS: "" + DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" RESOURCE_STORAGE_TYPE: "HDFS" RESOURCE_UPLOAD_PATH: "/dolphinscheduler" FS_DEFAULT_FS: "file:///" FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com" FS_S3A_ACCESS_KEY: "xxxxxxx" FS_S3A_SECRET_KEY: "xxxxxxx" + HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE: "false" + JAVA_SECURITY_KRB5_CONF_PATH: "/opt/krb5.conf" + LOGIN_USER_KEYTAB_USERNAME: "hdfs@HADOOP.COM" + LOGIN_USER_KEYTAB_PATH: "/opt/hdfs.keytab" + KERBEROS_EXPIRE_TIME: "2" + HDFS_ROOT_USER: "hdfs" + RESOURCE_MANAGER_HTTPADDRESS_PORT: "8088" + YARN_RESOURCEMANAGER_HA_RM_IDS: "" + YARN_APPLICATION_STATUS_ADDRESS: "http://ds1:%s/ws/v1/cluster/apps/%s" + YARN_JOB_HISTORY_STATUS_ADDRESS: "http://ds1:19888/ws/v1/history/mapreduce/jobs/%s" + DATASOURCE_ENCRYPTION_ENABLE: "false" + DATASOURCE_ENCRYPTION_SALT: "!@#$%^&*" + SUDO_ENABLE: "true" + # dolphinscheduler env + HADOOP_HOME: "/opt/soft/hadoop" + HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" + SPARK_HOME1: "/opt/soft/spark1" + SPARK_HOME2: "/opt/soft/spark2" + PYTHON_HOME: "/usr/bin/python" + JAVA_HOME: "/usr/local/openjdk-8" + HIVE_HOME: "/opt/soft/hive" + FLINK_HOME: "/opt/soft/flink" + DATAX_HOME: "/opt/soft/datax" + ## Shared storage persistence mounted into api, master and worker, such as Hadoop, Spark, Flink and DataX binary package + sharedStoragePersistence: + enabled: false + mountPath: "/opt/soft" + accessModes: + - "ReadWriteMany" + ## storageClassName must support the access mode: ReadWriteMany + storageClassName: "-" + storage: "20Gi" ## If RESOURCE_STORAGE_TYPE is HDFS and FS_DEFAULT_FS is file:///, fsFileResourcePersistence should be enabled for resource storage fsFileResourcePersistence: enabled: false @@ -126,17 +146,18 @@ master: # requests: # memory: "2Gi" # cpu: "500m" - ## ConfigMap + ## Configmap configmap: - DOLPHINSCHEDULER_OPTS: "" + MASTER_SERVER_OPTS: "-Xms1g -Xmx1g -Xmn512m" MASTER_EXEC_THREADS: "100" MASTER_EXEC_TASK_NUM: "20" + MASTER_DISPATCH_TASK_NUM: "3" + MASTER_HOST_SELECTOR: "LowerWeight" MASTER_HEARTBEAT_INTERVAL: "10" MASTER_TASK_COMMIT_RETRYTIMES: "5" MASTER_TASK_COMMIT_INTERVAL: "1000" - MASTER_MAX_CPULOAD_AVG: "100" - MASTER_RESERVED_MEMORY: "0.1" - MASTER_LISTEN_PORT: "5678" + MASTER_MAX_CPULOAD_AVG: "-1" + MASTER_RESERVED_MEMORY: "0.3" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -194,16 +215,16 @@ worker: # requests: # memory: "2Gi" # cpu: "500m" - ## ConfigMap + ## Configmap configmap: - DOLPHINSCHEDULER_OPTS: "" + LOGGER_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m" + WORKER_SERVER_OPTS: "-Xms1g -Xmx1g -Xmn512m" WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10" - WORKER_MAX_CPULOAD_AVG: "100" - WORKER_RESERVED_MEMORY: "0.1" - WORKER_LISTEN_PORT: "1234" - WORKER_GROUPS: "default" WORKER_HOST_WEIGHT: "100" + WORKER_MAX_CPULOAD_AVG: "-1" + WORKER_RESERVED_MEMORY: "0.3" + WORKER_GROUPS: "default" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -275,9 +296,9 @@ alert: # requests: # memory: "1Gi" # cpu: "500m" - ## ConfigMap + ## Configmap configmap: - DOLPHINSCHEDULER_OPTS: "" + ALERT_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m" ALERT_PLUGIN_DIR: "lib/plugin/alert" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes @@ -338,9 +359,9 @@ api: # requests: # memory: "1Gi" # cpu: "500m" - ## ConfigMap + ## Configmap configmap: - DOLPHINSCHEDULER_OPTS: "" + API_SERVER_OPTS: "-Xms512m -Xmx512m -Xmn256m" ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -367,6 +388,22 @@ api: - "ReadWriteOnce" storageClassName: "-" storage: "20Gi" + service: + ## type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer + type: "ClusterIP" + ## clusterIP is the IP address of the service and is usually assigned randomly by the master + clusterIP: "" + ## nodePort is the port on each node on which this service is exposed when type=NodePort + nodePort: "" + ## externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service + externalIPs: [] + ## externalName is the external reference that kubedns or equivalent will return as a CNAME record for this service, requires Type to be ExternalName + externalName: "" + ## loadBalancerIP when service.type is LoadBalancer. LoadBalancer will get created with the IP specified in this field + loadBalancerIP: "" + ## annotations may need to be set when service.type is LoadBalancer + ## service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:EXAMPLE_CERT + annotations: {} ingress: enabled: false diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 8041fa565..077f7ac26 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -31,7 +31,7 @@ public class Constants { public static final String ALERT_PROPERTIES_PATH = "/alert.properties"; /** default alert plugin dir **/ - public static final String ALERT_PLUGIN_PATH = "./lib/plugin/alert"; + public static final String ALERT_PLUGIN_PATH = "lib/plugin/alert"; public static final int ALERT_SCAN_INTERVAL = 5000; diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 9a6481c7c..c41ae6123 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -21,7 +21,7 @@ #eg : Alert Server Listener port #alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server . -#alert.plugin.dir=./lib/plugin/alert +#alert.plugin.dir=lib/plugin/alert #maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository diff --git a/dolphinscheduler-api/src/main/resources/application-api.properties b/dolphinscheduler-api/src/main/resources/application-api.properties index a635cccd6..d78db2d63 100644 --- a/dolphinscheduler-api/src/main/resources/application-api.properties +++ b/dolphinscheduler-api/src/main/resources/application-api.properties @@ -41,7 +41,7 @@ server.jetty.max-http-form-post-size=5000000 # messages encoding spring.messages.encoding=UTF-8 -# i18n classpath folder , file prefix messages， if have many files, use "," seperator +# i18n classpath folder, file prefix messages. if have many files, use "," seperator spring.messages.basename=i18n/messages # Authentication types (supported types: PASSWORD) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 4ef30a25e..4320bd971 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -93,11 +93,19 @@ public final class Constants { public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key"; + /** + * hadoop configuration + */ + public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE"; + + public static final String HADOOP_RM_STATE_STANDBY = "STANDBY"; + + public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port"; + /** * yarn.resourcemanager.ha.rm.ids */ public static final String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids"; - public static final String YARN_RESOURCEMANAGER_HA_XX = "xx"; /** @@ -153,7 +161,6 @@ public final class Constants { * development.state */ public static final String DEVELOPMENT_STATE = "development.state"; - public static final String DEVELOPMENT_STATE_DEFAULT_VALUE = "true"; /** * sudo enable @@ -266,15 +273,6 @@ public final class Constants { public static final String AT_SIGN = "@"; - public static final String WORKER_MAX_CPULOAD_AVG = "worker.max.cpuload.avg"; - - public static final String WORKER_RESERVED_MEMORY = "worker.reserved.memory"; - - public static final String MASTER_MAX_CPULOAD_AVG = "master.max.cpuload.avg"; - - public static final String MASTER_RESERVED_MEMORY = "master.reserved.memory"; - - /** * date format of yyyy-MM-dd HH:mm:ss */ @@ -473,15 +471,6 @@ public final class Constants { */ public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate"; - /** - * hadoop configuration - */ - public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE"; - - public static final String HADOOP_RM_STATE_STANDBY = "STANDBY"; - - public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port"; - /** * data source config @@ -834,11 +823,6 @@ public final class Constants { */ public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; - /** - * default worker group id - */ - public static final int DEFAULT_WORKER_ID = -1; - /** * loginUserFromKeytab path */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java index 4314089ef..4f0953f82 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory; */ public class NetUtils { - private static final Pattern STS_PATTERN = Pattern.compile("-\\d+$"); // StatefulSet pattern private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$"); private static final String NETWORK_PRIORITY_DEFAULT = "default"; private static final String NETWORK_PRIORITY_INNER = "inner"; @@ -81,13 +80,8 @@ public class NetUtils { if (inetAddress != null) { if (Constants.KUBERNETES_MODE) { String canonicalHost = inetAddress.getCanonicalHostName(); - if (!canonicalHost.contains(".") || IP_PATTERN.matcher(canonicalHost).matches()) { - String host = inetAddress.getHostName(); - if (STS_PATTERN.matcher(host).find()) { - return String.format("%s.%s", host, host.replaceFirst("\\d+$", "headless")); - } - } else if (canonicalHost.contains(".")) { - String[] items = canonicalHost.split("\\."); + String[] items = canonicalHost.split("\\."); + if (items.length == 6 && "svc".equals(items[3])) { return String.format("%s.%s", items[0], items[1]); } return canonicalHost; diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 1377335b7..726c799c7 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -15,16 +15,16 @@ # limitations under the License. # -# resource storage type : HDFS, S3, NONE +# user data local directory path, please make sure the directory exists and have read write permissions +data.basedir.path=/tmp/dolphinscheduler + +# resource storage type: HDFS, S3, NONE resource.storage.type=NONE -# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.upload.path=/dolphinscheduler -# user data local directory path, please make sure the directory exists and have read write permissions -#data.basedir.path=/tmp/dolphinscheduler - -# whether kerberos starts +# whether to startup kerberos hadoop.security.authentication.startup.state=false # java.security.krb5.conf path @@ -36,13 +36,16 @@ login.user.keytab.username=hdfs-mycluster@ESZ.COM # login user from keytab path login.user.keytab.path=/opt/hdfs.headless.keytab -#resource.view.suffixs +# kerberos expire time, the unit is hour +kerberos.expire.time=2 + +# resource view suffixs #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js -# if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path +# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path hdfs.root.user=hdfs -# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir +# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir fs.defaultFS=hdfs://mycluster:8020 # if resource.storage.type=S3, s3 endpoint @@ -54,31 +57,32 @@ fs.s3a.access.key=A3DXS30FO22544RE # if resource.storage.type=S3, s3 secret key fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK -# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=8088 + +# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. -yarn.application.status.address=http://10.172.33.1:%s/ws/v1/cluster/apps/%s +# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname +yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s -# if custom you resourcemanager port ,you need to replace 8088 else default value. -resource.manager.httpaddress.port=8088 - -# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000) +# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s -# system env path, If you want to set your own path, you need to set this env file to an absolute path -#dolphinscheduler.env.path=env/dolphinscheduler_env.sh -development.state=false - -# kerberos tgt expire time, unit is hours -kerberos.expire.time=2 +# datasource encryption enable +datasource.encryption.enable=false # datasource encryption salt -datasource.encryption.enable=false datasource.encryption.salt=!@#$%^&* -# Network IP gets priority, default inner outer +# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions +sudo.enable=true + +# network IP gets priority, default: inner outer #dolphin.scheduler.network.priority.strategy=default -# use sudo or not, if set true ,executing user is tenant user and deploy user need sudo permissions ; if set false, executing user is the deploy user, don't need sudo permissions. -sudo.enable=true +# system env path +#dolphinscheduler.env.path=env/dolphinscheduler_env.sh + +# development state +development.state=false diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java index f912933d8..59dfa2f73 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java @@ -51,11 +51,24 @@ public class NetUtilsTest { assertEquals("172.17.0.15", NetUtils.getHost(address)); CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address)); + + address = mock(InetAddress.class); + when(address.getCanonicalHostName()).thenReturn("busybox-1.default-subdomain.my-namespace.svc.cluster-domain.example"); + when(address.getHostName()).thenReturn("busybox-1"); + CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); + assertEquals("busybox-1.default-subdomain", NetUtils.getHost(address)); + + address = mock(InetAddress.class); + when(address.getCanonicalHostName()).thenReturn("dolphinscheduler.cluster-domain.example"); + when(address.getHostName()).thenReturn("dolphinscheduler"); + CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); + assertEquals("dolphinscheduler.cluster-domain.example", NetUtils.getHost(address)); + address = mock(InetAddress.class); when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0"); when(address.getHostName()).thenReturn("dolphinscheduler-worker-0"); CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); - assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address)); + assertEquals("dolphinscheduler-worker-0", NetUtils.getHost(address)); } @Test diff --git a/dolphinscheduler-dao/src/main/resources/datasource.properties b/dolphinscheduler-dao/src/main/resources/datasource.properties index 0deb7fe00..951e0f9a1 100644 --- a/dolphinscheduler-dao/src/main/resources/datasource.properties +++ b/dolphinscheduler-dao/src/main/resources/datasource.properties @@ -18,14 +18,14 @@ # postgresql spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler -spring.datasource.username=test -spring.datasource.password=test +spring.datasource.username=root +spring.datasource.password=root # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 -#spring.datasource.username=xxxx -#spring.datasource.password=xxxx +#spring.datasource.username=ds_user +#spring.datasource.password=dolphinscheduler # connection configuration #spring.datasource.initialSize=5 @@ -66,4 +66,4 @@ spring.datasource.password=test # open PSCache, specify count PSCache for every connection #spring.datasource.poolPreparedStatements=true -#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 \ No newline at end of file +#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index ef5236264..8020a9b24 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -14,9 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master.config; import org.apache.dolphinscheduler.common.Constants; + import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; @@ -25,21 +27,27 @@ import org.springframework.stereotype.Component; @PropertySource(value = "master.properties") public class MasterConfig { + @Value("${master.listen.port:5678}") + private int listenPort; + @Value("${master.exec.threads:100}") private int masterExecThreads; @Value("${master.exec.task.num:20}") private int masterExecTaskNum; + @Value("${master.dispatch.task.num:3}") + private int masterDispatchTaskNumber; + + @Value("${master.host.selector:LowerWeight}") + private String hostSelector; + @Value("${master.heartbeat.interval:10}") private int masterHeartbeatInterval; @Value("${master.task.commit.retryTimes:5}") private int masterTaskCommitRetryTimes; - @Value("${master.dispatch.task.num:3}") - private int masterDispatchTaskNumber; - @Value("${master.task.commit.interval:1000}") private int masterTaskCommitInterval; @@ -49,12 +57,6 @@ public class MasterConfig { @Value("${master.reserved.memory:0.3}") private double masterReservedMemory; - @Value("${master.host.selector:lowerWeight}") - private String hostSelector; - - @Value("${master.listen.port:5678}") - private int listenPort; - public int getListenPort() { return listenPort; } @@ -112,7 +114,7 @@ public class MasterConfig { } public double getMasterMaxCpuloadAvg() { - if (masterMaxCpuloadAvg == -1){ + if (masterMaxCpuloadAvg == -1) { return Constants.DEFAULT_MASTER_CPU_LOAD; } return masterMaxCpuloadAvg; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index 22c19af89..2c71a97f3 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -29,6 +29,9 @@ import org.springframework.stereotype.Component; @PropertySource(value = "worker.properties") public class WorkerConfig { + @Value("${worker.listen.port:1234}") + private int listenPort; + @Value("${worker.exec.threads:100}") private int workerExecThreads; @@ -47,9 +50,6 @@ public class WorkerConfig { @Value("${worker.reserved.memory:0.3}") private double workerReservedMemory; - @Value("${worker.listen.port:1234}") - private int listenPort; - @Value("#{'${worker.groups:default}'.split(',')}") private Set workerGroups; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java index 8340b788b..7e961964b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutor.java @@ -14,23 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.worker.task; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.function.Consumer; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * python command executor @@ -46,7 +52,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { * python */ public static final String PYTHON = "python"; - + private static final Pattern PYTHON_PATH_PATTERN = Pattern.compile("/bin/python[\\d.]*$"); /** * constructor @@ -115,54 +121,64 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { @Override protected String commandInterpreter() { String pythonHome = getPythonHome(taskExecutionContext.getEnvFile()); - if (StringUtils.isEmpty(pythonHome)){ + return getPythonCommand(pythonHome); + } + + /** + * get python command + * + * @param pythonHome python home + * @return python command + */ + public static String getPythonCommand(String pythonHome) { + if (StringUtils.isEmpty(pythonHome)) { return PYTHON; } + File file = new File(pythonHome); + if (file.exists() && file.isFile()) { + return pythonHome; + } + if (PYTHON_PATH_PATTERN.matcher(pythonHome).find()) { + return pythonHome; + } return pythonHome + "/bin/python"; } - - /** - * get the absolute path of the Python are installed - * note : - * - * for example : - * your PYTHON_HOM is /opt/python3.7 + * get python home * * @param envPath env path * @return python home */ - private static String getPythonHome(String envPath){ + public static String getPythonHome(String envPath) { BufferedReader br = null; StringBuilder sb = new StringBuilder(); try { br = new BufferedReader(new InputStreamReader(new FileInputStream(envPath))); String line; - while ((line = br.readLine()) != null){ - if (line.contains(Constants.PYTHON_HOME)){ + while ((line = br.readLine()) != null) { + if (line.contains(Constants.PYTHON_HOME)) { sb.append(line); break; } } String result = sb.toString(); - if (org.apache.commons.lang.StringUtils.isEmpty(result)){ + if (StringUtils.isEmpty(result)) { return null; } String[] arrs = result.split(Constants.EQUAL_SIGN); - if (arrs.length == 2){ + if (arrs.length == 2) { return arrs[1]; } - - }catch (IOException e){ - logger.error("read file failure",e); - }finally { + } catch (IOException e) { + logger.error("read file failure", e); + } finally { try { - if (br != null){ + if (br != null) { br.close(); } } catch (IOException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } } return null; diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index f3ed554a8..0bc5455c9 100755 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -53,7 +53,7 @@ deployUser="dolphinscheduler" # Note: find and load the Alert Plugin Jar from this dir. alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert" -# resource storage type：HDFS,S3,NONE +# resource storage type: HDFS, S3, NONE resourceStorageType="NONE" # if resourceStorageType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. @@ -66,13 +66,16 @@ s3Endpoint="http://192.168.xx.xx:9010" s3AccessKey="xxxxxxxxxx" s3SecretKey="xxxxxxxxxx" -# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty +# resourcemanager port, the default value is 8088 if not specified +resourceManagerHttpAddressPort=8088 + +# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty yarnHaIps="192.168.xx.xx,192.168.xx.xx" -# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname. +# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname singleYarnIp="yarnIp1" -# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。/dolphinscheduler is recommended +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resourceUploadPath="/dolphinscheduler" # who have permissions to create directory under HDFS/S3 root path diff --git a/dolphinscheduler-server/src/main/resources/master.properties b/dolphinscheduler-server/src/main/resources/master.properties index bb9877cb2..cc4562213 100644 --- a/dolphinscheduler-server/src/main/resources/master.properties +++ b/dolphinscheduler-server/src/main/resources/master.properties @@ -15,29 +15,32 @@ # limitations under the License. # -# master execute thread number +# master listen port +#master.listen.port=5678 + +# master execute thread number to limit process instances in parallel #master.exec.threads=100 -# master execute task number in parallel +# master execute task number in parallel per process instance #master.exec.task.num=20 -# master dispatch task number +# master dispatch task number per batch #master.dispatch.task.num=3 -# master heartbeat interval +# master host selector to select a suitable worker, default value: LowerWeight. Optional values include Random, RoundRobin, LowerWeight +#master.host.selector=LowerWeight + +# master heartbeat interval, the unit is second #master.heartbeat.interval=10 # master commit task retry times #master.task.commit.retryTimes=5 -# master commit task interval +# master commit task interval, the unit is millisecond #master.task.commit.interval=1000 -# only less than cpu avg load, master server can work. default value -1 : the number of cpu cores * 2 +# master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2 #master.max.cpuload.avg=-1 -# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. +# master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G #master.reserved.memory=0.3 - -# master listen port -#master.listen.port=5678 diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index c5c379c69..2c37e5309 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -15,10 +15,13 @@ # limitations under the License. # -# worker execute thread number +# worker listener port +#worker.listen.port=1234 + +# worker execute thread number to limit task instances in parallel #worker.exec.threads=100 -# worker heartbeat interval +# worker heartbeat interval, the unit is second #worker.heartbeat.interval=10 # worker host weight to dispatch tasks, default value 100 @@ -27,17 +30,14 @@ # worker tenant auto create #worker.tenant.auto.create=false -# only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2 +# worker max cpuload avg, only higher than the system cpu load average, worker server can be dispatched tasks. default value -1: the number of cpu cores * 2 #worker.max.cpuload.avg=-1 -# only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. +# worker reserved memory, only lower than system available memory, worker server can be dispatched tasks. default value 0.3, the unit is G #worker.reserved.memory=0.3 -# worker listener port -#worker.listen.port=1234 - -# default worker groups, if this worker belongs different groups, you can config the following like that 'worker.groups=default,test' +# default worker groups separated by comma, like 'worker.groups=default,test' #worker.groups=default -# alert server listener host -alert.listen.host=localhost +# alert server listen host +#alert.listen.host=localhost diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java deleted file mode 100644 index 34f5d2f66..000000000 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/EnvFileTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; - -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EnvFileTest { - - private static final Logger logger = LoggerFactory.getLogger(EnvFileTest.class); - - @Test - public void test() { - String path = System.getProperty("user.dir")+"/script/env/dolphinscheduler_env.sh"; - String pythonHome = getPythonHome(path); - logger.info(pythonHome); - } - - /** - * get python home - * @param path - * @return - */ - private static String getPythonHome(String path){ - BufferedReader br = null; - String line = null; - StringBuilder sb = new StringBuilder(); - try { - br = new BufferedReader(new InputStreamReader(new FileInputStream(path))); - while ((line = br.readLine()) != null){ - if (line.contains(Constants.PYTHON_HOME)) { - sb.append(line); - break; - } - } - String result = sb.toString(); - if (StringUtils.isEmpty(result)){ - return null; - } - String[] arrs = result.split("="); - if (arrs.length == 2){ - return arrs[1]; - } - - }catch (IOException e){ - logger.error("read file failed",e); - }finally { - try { - if (br != null){ - br.close(); - } - } catch (IOException e) { - logger.error(e.getMessage(),e); - } - } - return null; - } -} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java new file mode 100644 index 000000000..f4e9080a6 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.task; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PythonCommandExecutorTest { + + private static final Logger logger = LoggerFactory.getLogger(PythonCommandExecutorTest.class); + + @Test + public void testGetPythonHome() { + String path = System.getProperty("user.dir") + "/script/env/dolphinscheduler_env.sh"; + if (path.contains("dolphinscheduler-server/")) { + path = path.replace("dolphinscheduler-server/", ""); + } + String pythonHome = PythonCommandExecutor.getPythonHome(path); + logger.info(pythonHome); + Assert.assertNotNull(pythonHome); + } + + @Test + public void testGetPythonCommand() { + String pythonCommand = PythonCommandExecutor.getPythonCommand(null); + Assert.assertEquals(PythonCommandExecutor.PYTHON, pythonCommand); + pythonCommand = PythonCommandExecutor.getPythonCommand(""); + Assert.assertEquals(PythonCommandExecutor.PYTHON, pythonCommand); + pythonCommand = PythonCommandExecutor.getPythonCommand("/usr/bin/python"); + Assert.assertEquals("/usr/bin/python", pythonCommand); + pythonCommand = PythonCommandExecutor.getPythonCommand("/usr/local/bin/python2"); + Assert.assertEquals("/usr/local/bin/python2", pythonCommand); + pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/python/bin/python3.8"); + Assert.assertEquals("/opt/python/bin/python3.8", pythonCommand); + pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/soft/python"); + Assert.assertEquals("/opt/soft/python/bin/python", pythonCommand); + pythonCommand = PythonCommandExecutor.getPythonCommand("/opt/soft/python-3.8"); + Assert.assertEquals("/opt/soft/python-3.8/bin/python", pythonCommand); + } + +} diff --git a/install.sh b/install.sh index 9de49748c..89cc85da5 100755 --- a/install.sh +++ b/install.sh @@ -43,8 +43,10 @@ sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.propert sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties +sed -i ${txt} "s#resource.manager.httpaddress.port.*#resource.manager.httpaddress.port=${resourceManagerHttpAddressPort}#g" conf/common.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties -sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties +sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:%s/ws/v1/cluster/apps/%s#g" conf/common.properties +sed -i ${txt} "s#yarn.job.history.status.address.*#yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s#g" conf/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties sed -i ${txt} "s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g" conf/common.properties sed -i ${txt} "s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g" conf/common.properties @@ -56,9 +58,9 @@ sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeep sed -i ${txt} "s#\#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties -sed -i ${txt} "s#alert.plugin.dir.*#alert.plugin.dir=${alertPluginDir}#g" conf/alert.properties +sed -i ${txt} "s#\#alert.plugin.dir.*#alert.plugin.dir=${alertPluginDir}#g" conf/alert.properties -sed -i ${txt} "s#alert.listen.host.*#alert.listen.host=${alertServer}#g" conf/worker.properties +sed -i ${txt} "s#\#alert.listen.host.*#alert.listen.host=${alertServer}#g" conf/worker.properties # 2.create directory echo "2.create directory" diff --git a/pom.xml b/pom.xml index 062be48f8..1feba64ab 100644 --- a/pom.xml +++ b/pom.xml @@ -970,7 +970,6 @@ **/server/worker/shell/ShellCommandExecutorTest.java **/server/worker/sql/SqlExecutorTest.java **/server/worker/task/spark/SparkTaskTest.java - **/server/worker/task/EnvFileTest.java **/server/worker/task/spark/SparkTaskTest.java **/server/worker/task/datax/DataxTaskTest.java @@ -979,9 +978,9 @@ **/server/worker/task/shell/ShellTaskTest.java **/server/worker/task/TaskManagerTest.java **/server/worker/task/AbstractCommandExecutorTest.java + **/server/worker/task/PythonCommandExecutorTest.java **/server/worker/task/ShellTaskReturnTest.java **/server/worker/task/sql/SqlTaskTest.java - **/server/worker/EnvFileTest.java **/server/worker/runner/TaskExecuteThreadTest.java **/server/worker/runner/WorkerManagerThreadTest.java **/service/quartz/cron/CronUtilsTest.java diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh index 7ab8cbcf1..9ca949943 100755 --- a/script/create-dolphinscheduler.sh +++ b/script/create-dolphinscheduler.sh @@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 08aaa14d0..cf3aeebe3 100755 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -29,10 +29,11 @@ shift command=$1 shift +echo "Begin $startStop $command......" BIN_DIR=`dirname $0` BIN_DIR=`cd "$BIN_DIR"; pwd` -DOLPHINSCHEDULER_HOME=$BIN_DIR/.. +DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd` source /etc/profile @@ -56,51 +57,47 @@ pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME +export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS" + if [ "$command" = "api-server" ]; then - HEAP_INITIAL_SIZE=1g - HEAP_MAX_SIZE=1g - HEAP_NEW_GENERATION_SIZE=512m LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer + HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" + export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS" elif [ "$command" = "master-server" ]; then - HEAP_INITIAL_SIZE=4g - HEAP_MAX_SIZE=4g - HEAP_NEW_GENERATION_SIZE=2g LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer + HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g" + export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS" elif [ "$command" = "worker-server" ]; then - HEAP_INITIAL_SIZE=2g - HEAP_MAX_SIZE=2g - HEAP_NEW_GENERATION_SIZE=1g LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer + HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g" + export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS" elif [ "$command" = "alert-server" ]; then - HEAP_INITIAL_SIZE=1g - HEAP_MAX_SIZE=1g - HEAP_NEW_GENERATION_SIZE=512m LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml" CLASS=org.apache.dolphinscheduler.alert.AlertServer + HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" + export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS" elif [ "$command" = "logger-server" ]; then - HEAP_INITIAL_SIZE=1g - HEAP_MAX_SIZE=1g - HEAP_NEW_GENERATION_SIZE=512m CLASS=org.apache.dolphinscheduler.server.log.LoggerServer + HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m" + export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS" elif [ "$command" = "zookeeper-server" ]; then #note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml" CLASS=org.apache.dolphinscheduler.service.zk.ZKServer else - echo "Error: No command named \`$command' was found." + echo "Error: No command named '$command' was found." exit 1 fi -export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION_SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS" - case $startStop in (start) - exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" if [ "$DOCKER" = "true" ]; then - echo "start in docker" + echo start $command in docker + export DOLPHINSCHEDULER_OPTS="$DOLPHINSCHEDULER_OPTS -XX:-UseContainerSupport" + exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" $JAVA_HOME/bin/java $exec_command else [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR" @@ -113,6 +110,7 @@ case $startStop in fi echo starting $command, logging to $log + exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & echo $! > $pid diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh index 6944e3a13..42ba839d5 100755 --- a/script/upgrade-dolphinscheduler.sh +++ b/script/upgrade-dolphinscheduler.sh @@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler -- GitLab