From 06c4ddf8e36ca79b45484281e09a0bf9e0e1b969 Mon Sep 17 00:00:00 2001
From: dailidong <dailidong66@gmail.com>
Date: Tue, 12 May 2020 16:49:40 +0800
Subject: [PATCH] add worker group config (#2681)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* update README about DolphinScheduler

* Update issue templates

* update

* regularize api pom xml
update rpc maven compile to 1.8

* regularize api pom xml

* change commons.lang3.StringUtils to common.utils.StringUtils

* update pom.xml

* update

* correct equals method

* jasper-runtime is needed when api server start

* jasper-runtime jar is needed when api server start

* combine logback config of master/worker/alert/api server to one logback.xml

* remove tomcat runtime jar

* add UT

* add license

* remove jasper-runtime jar, not need anymore

* sovle jar conflict， remove servlet-api 2.5 jar

* remove servlet 2.5

* add embedded database h2 and update embedded zookeeper

* delete logs

* add license

* change log level to INFO

* simplify and optimize config

* add development state

* simplify common config

* add worker group config

* add worker group config

Co-authored-by: lidongdai <escheduler@outlook.com>
Co-authored-by: dailidong <dolphinschedule@gmail.com>
---
 .../src/main/resources/config/install_config.conf    | 12 ++++++------
 .../src/main/resources/worker.properties             |  4 ++--
 script/scp-hosts.sh                                  | 11 +++++++++++
 script/start-all.sh                                  |  3 +--
 script/stop-all.sh                                   |  3 +--
 5 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf
index fab6b32d5..f6b09b9cb 100644
--- a/dolphinscheduler-server/src/main/resources/config/install_config.conf
+++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf
@@ -118,7 +118,7 @@ apiServerPort="12345"
 
 # install hosts
 # Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
-ips="ark0,ark1,ark2,ark3,ark4"
+ips="ds1,ds2,ds3,ds4,ds5"
 
 # ssh port, default 22
 # Note: if ssh port is not default, modify here
@@ -126,19 +126,19 @@ sshPort="22"
 
 # run master machine
 # Note: list of hosts hostname for deploying master
-masters="ark0,ark1"
+masters="ds1,ds2"
 
 # run worker machine
-# note: list of machine hostnames for deploying workers
-workers="ark2,ark3,ark4"
+# note: need to write the worker group name of each worker, the default value is "default"
+workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default")
 
 # run alert machine
 # note: list of machine hostnames for deploying alert server
-alertServer="ark3"
+alertServer="ds3"
 
 # run api machine
 # note: list of machine hostnames for deploying api server
-apiServers="ark1"
+apiServers="ds1"
 
 # whether to start monitoring self-starting scripts
 monitorServerState="false"
diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties
index d078f26ca..36bc13274 100644
--- a/dolphinscheduler-server/src/main/resources/worker.properties
+++ b/dolphinscheduler-server/src/main/resources/worker.properties
@@ -21,7 +21,7 @@
 # worker heartbeat interval
 #worker.heartbeat.interval=10
 
-# submit the number of tasks at a time
+# submit the number of tasks at a time  TODO
 #worker.fetch.task.num = 3
 
 # only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
@@ -34,4 +34,4 @@
 #worker.listen.port: 1234
 
 # default worker group
-#worker.group=default
\ No newline at end of file
+worker.group=default
\ No newline at end of file
diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh
index f4949f311..adee7d76c 100644
--- a/script/scp-hosts.sh
+++ b/script/scp-hosts.sh
@@ -20,6 +20,12 @@ workDir=`dirname $0`
 workDir=`cd ${workDir};pwd`
 source $workDir/../conf/config/install_config.conf
 
+txt=""
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # Mac OSX
+    txt="''"
+fi
+
 hostsArr=(${ips//,/ })
 for host in ${hostsArr[@]}
 do
@@ -33,6 +39,11 @@ do
 
   for dsDir in bin conf lib script sql ui install.sh
   do
+    # if worker in workersGroup
+    if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then
+      sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties
+    fi
+
     echo "start to scp $dsDir to $host/$installPath"
     scp -P $sshPort -r $workDir/../$dsDir  $host:$installPath
   done
diff --git a/script/start-all.sh b/script/start-all.sh
index 11e457205..5579a7d87 100644
--- a/script/start-all.sh
+++ b/script/start-all.sh
@@ -28,8 +28,7 @@ do
 
 done
 
-workersHost=(${workers//,/ })
-for worker in ${workersHost[@]}
+for worker in ${!workersGroup[*]}
 do
   echo "$worker worker server is starting"
 
diff --git a/script/stop-all.sh b/script/stop-all.sh
index f761579cc..e4ccf75a2 100644
--- a/script/stop-all.sh
+++ b/script/stop-all.sh
@@ -29,8 +29,7 @@ do
 
 done
 
-workersHost=(${workers//,/ })
-for worker in ${workersHost[@]}
+for worker in ${!workersGroup[*]}
 do
   echo "$worker worker server is stopping"
   ssh -p $sshPort $worker  "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"
-- 
GitLab