Merge pull request #4711 from chengshiwen/1.3.5-fix-docker-sql

[1.3.5-prepare][Fix][Mail] Fix sql task do not execute and send mail failed exception under default mail config

docker/build/Dockerfile

@@ -53,7 +53,7 @@ RUN dos2unix /root/checkpoint.sh && \
     dos2unix /opt/dolphinscheduler/bin/*.sh && \
     rm -rf /bin/sh && \
     ln -s /bin/bash /bin/sh && \
-    mkdir -p /tmp/xls && \
+    mkdir -p /var/mail /tmp/xls && \
     echo "Set disable_coredump false" >> /etc/sudo.conf
 
 # 4. expose port

docker/build/README.md

@@ -65,6 +65,8 @@ apache/dolphinscheduler:latest master-server
 ```
 $ docker run -d --name dolphinscheduler-worker \
 -e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 -v dolphinscheduler-resource-local:/dolphinscheduler \
 apache/dolphinscheduler:latest worker-server
 ```

docker/build/README_zh_CN.md

@@ -64,6 +64,8 @@ apache/dolphinscheduler:latest master-server
 ```
 $ docker run -d --name dolphinscheduler-worker \
 -e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
 -v dolphinscheduler-resource-local:/dolphinscheduler \
 apache/dolphinscheduler:latest worker-server
 ```

docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh

@@ -15,6 +15,14 @@
 # limitations under the License.
 #
 
-export PYTHON_HOME=/usr/bin/python2
+export HADOOP_HOME=/opt/soft/hadoop
+export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
+export SPARK_HOME1=/opt/soft/spark1
+export SPARK_HOME2=/opt/soft/spark2
+export PYTHON_HOME=/usr/bin/python
 export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
-export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH
+export HIVE_HOME=/opt/soft/hive
+export FLINK_HOME=/opt/soft/flink
+export DATAX_HOME=/opt/soft/datax/bin/datax.py
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH

docker/build/startup.sh

@@ -143,6 +143,7 @@ case "$1" in
     ;;
     (worker-server)
         waitZK
+        waitDatabase
         initWorkerServer
         initLoggerServer
         LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log

docker/docker-swarm/docker-compose.yml

@@ -173,6 +173,20 @@ services:
       WORKER_RESERVED_MEMORY: "0.1"
       WORKER_GROUP: "default"
       DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
+      XLS_FILE_PATH: "/tmp/xls"
+      MAIL_SERVER_HOST: ""
+      MAIL_SERVER_PORT: ""
+      MAIL_SENDER: ""
+      MAIL_USER: ""
+      MAIL_PASSWD: ""
+      MAIL_SMTP_STARTTLS_ENABLE: "false"
+      MAIL_SMTP_SSL_ENABLE: "false"
+      MAIL_SMTP_SSL_TRUST: ""
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
       RESOURCE_STORAGE_TYPE: HDFS
       RESOURCE_UPLOAD_PATH: /dolphinscheduler

docker/docker-swarm/docker-stack.yml

@@ -167,6 +167,20 @@ services:
       WORKER_RESERVED_MEMORY: "0.1"
       WORKER_GROUP: "default"
       DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
+      XLS_FILE_PATH: "/tmp/xls"
+      MAIL_SERVER_HOST: ""
+      MAIL_SERVER_PORT: ""
+      MAIL_SENDER: ""
+      MAIL_USER: ""
+      MAIL_PASSWD: ""
+      MAIL_SMTP_STARTTLS_ENABLE: "false"
+      MAIL_SMTP_SSL_ENABLE: "false"
+      MAIL_SMTP_SSL_TRUST: ""
+      DATABASE_HOST: dolphinscheduler-postgresql
+      DATABASE_PORT: 5432
+      DATABASE_USERNAME: root
+      DATABASE_PASSWORD: root
+      DATABASE_DATABASE: dolphinscheduler
       ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
       RESOURCE_STORAGE_TYPE: HDFS
       RESOURCE_UPLOAD_PATH: /dolphinscheduler

docker/docker-swarm/dolphinscheduler_env.sh

@@ -15,6 +15,14 @@
 # limitations under the License.
 #
 
-export PYTHON_HOME=/usr/bin/python2
+export HADOOP_HOME=/opt/soft/hadoop
+export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
+export SPARK_HOME1=/opt/soft/spark1
+export SPARK_HOME2=/opt/soft/spark2
+export PYTHON_HOME=/usr/bin/python
 export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
-export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH
\ No newline at end of file
+export HIVE_HOME=/opt/soft/hive
+export FLINK_HOME=/opt/soft/flink
+export DATAX_HOME=/opt/soft/datax/bin/datax.py
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH

docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@@ -99,6 +99,51 @@ spec:
                 configMapKeyRef:
                   name: {{ include "dolphinscheduler.fullname" . }}-worker
                   key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
+            - name: XLS_FILE_PATH
+              valueFrom:
+                configMapKeyRef:
+                  key: XLS_FILE_PATH
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: MAIL_SERVER_HOST
+              valueFrom:
+                configMapKeyRef:
+                  key: MAIL_SERVER_HOST
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: MAIL_SERVER_PORT
+              valueFrom:
+                configMapKeyRef:
+                  key: MAIL_SERVER_PORT
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: MAIL_SENDER
+              valueFrom:
+                configMapKeyRef:
+                  key: MAIL_SENDER
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: MAIL_USER
+              valueFrom:
+                configMapKeyRef:
+                  key: MAIL_USER
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: MAIL_PASSWD
+              valueFrom:
+                configMapKeyRef:
+                  key: MAIL_PASSWD
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: MAIL_SMTP_STARTTLS_ENABLE
+              valueFrom:
+                configMapKeyRef:
+                  key: MAIL_SMTP_STARTTLS_ENABLE
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: MAIL_SMTP_SSL_ENABLE
+              valueFrom:
+                configMapKeyRef:
+                  key: MAIL_SMTP_SSL_ENABLE
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
+            - name: MAIL_SMTP_SSL_TRUST
+              valueFrom:
+                configMapKeyRef:
+                  key: MAIL_SMTP_SSL_TRUST
+                  name: {{ include "dolphinscheduler.fullname" . }}-alert
             - name: DATABASE_HOST
               {{- if .Values.postgresql.enabled }}
               value: {{ template "dolphinscheduler.postgresql.fullname" . }}

dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java

@@ -96,9 +96,6 @@ public class EmailAlertPlugin implements AlertPlugin {
         retMaps = emailManager.send(receviersList, receviersCcList, alert.getTitle(), alert.getContent(),
                 alert.getShowType());
 
-        //send flag
-        boolean flag = false;
-
         if (retMaps == null) {
             retMaps = new HashMap<>();
             retMaps.put(Constants.MESSAGE, "alert send error.");
@@ -107,9 +104,12 @@ public class EmailAlertPlugin implements AlertPlugin {
             return retMaps;
         }
 
-        flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS)));
-
-        if (flag) {
+        boolean enabled = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.MAIL_ENABLED)));
+        boolean status = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS)));
+        if (!enabled) {
+            logger.warn("mail wasn't sent since the mail config isn't set");
+            retMaps.put(Constants.MESSAGE, "mail wasn't sent since the mail config isn't set");
+        } else if (status) {
             logger.info("alert send success");
             retMaps.put(Constants.MESSAGE, "email send success.");
             if (EnterpriseWeChatUtils.isEnable()) {
@@ -121,7 +121,6 @@ public class EmailAlertPlugin implements AlertPlugin {
                     logger.error(e.getMessage(), e);
                 }
             }
-
         } else {
             retMaps.put(Constants.MESSAGE, "alert send error.");
             logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE));

dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java

@@ -28,8 +28,6 @@ public class Constants {
      */
     public static final String ALERT_PROPERTIES_PATH = "/alert.properties";
 
-    public static final String DATA_SOURCE_PROPERTIES_PATH = "/dao/data_source.properties";
-
     public static final String SINGLE_SLASH = "/";
 
     /**
@@ -41,6 +39,8 @@ public class Constants {
 
     public static final String MESSAGE = "message";
 
+    public static final String MAIL_ENABLED = "mail.enabled";
+
     public static final String MAIL_PROTOCOL = "mail.protocol";
 
     public static final String MAIL_SERVER_HOST = "mail.server.host";

dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java

@@ -51,11 +51,7 @@ public class MailUtils {
 
     public static final String MAIL_PASSWD = PropertyUtils.getString(Constants.MAIL_PASSWD);
 
-    public static final Boolean MAIL_USE_START_TLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE);
-
-    public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE);
-
-    public static final String xlsFilePath = PropertyUtils.getString(Constants.XLS_FILE_PATH,"/tmp/xls");
+    public static final String XLS_FILE_PATH = PropertyUtils.getString(Constants.XLS_FILE_PATH, "/tmp/xls");
 
     public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE);
 
@@ -89,6 +85,15 @@ public class MailUtils {
      */
     public static Map<String,Object> sendMails(Collection<String> receivers, Collection<String> receiversCc, String title, String content, String showType) {
         Map<String,Object> retMap = new HashMap<>();
+
+        // if mail is default config, no need to process
+        if (StringUtils.isEmpty(MAIL_SERVER_HOST) || "xxx.xxx.com".equals(MAIL_SERVER_HOST)) {
+            retMap.put(Constants.MAIL_ENABLED, false);
+            retMap.put(Constants.STATUS, true);
+            return retMap;
+        }
+
+        retMap.put(Constants.MAIL_ENABLED, true);
         retMap.put(Constants.STATUS, false);
 
         // if there is no receivers && no receiversCc, no need to process
@@ -260,13 +265,13 @@ public class MailUtils {
         part1.setContent(partContent, Constants.TEXT_HTML_CHARSET_UTF_8);
         // set attach file
         MimeBodyPart part2 = new MimeBodyPart();
-        File file = new File(xlsFilePath + Constants.SINGLE_SLASH +  title + Constants.EXCEL_SUFFIX_XLS);
+        File file = new File(XLS_FILE_PATH + Constants.SINGLE_SLASH +  title + Constants.EXCEL_SUFFIX_XLS);
         if (!file.getParentFile().exists()) {
             file.getParentFile().mkdirs();
         }
         // make excel file
 
-        ExcelUtils.genExcelFile(content,title,xlsFilePath);
+        ExcelUtils.genExcelFile(content,title,XLS_FILE_PATH);
 
         part2.attachFile(file);
         part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B"));

dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java

@@ -75,6 +75,7 @@ public class EmailAlertPluginTest {
         List<String> list = new ArrayList<String>(){{ add("xx@xx.com"); }};
         alertInfo.addProp("receivers", list);
         Map<String, Object> ret = plugin.process(alertInfo);
-        assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS))));
+        assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.MAIL_ENABLED))));
+        assertTrue(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS))));
     }
 }
\ No newline at end of file

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java

@@ -49,6 +49,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static org.apache.dolphinscheduler.alert.utils.Constants.MAIL_ENABLED;
 import static org.apache.dolphinscheduler.common.Constants.*;
 import static org.apache.dolphinscheduler.common.enums.DbType.HIVE;
 /**
@@ -452,7 +453,10 @@ public class SqlTask extends AbstractTask {
         if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){
             Map<String, Object> mailResult = MailUtils.sendMails(receiversList,
                     receiversCcList, title, content, ShowType.valueOf(showTypeName).getDescp());
-            if(!(boolean) mailResult.get(STATUS)){
+            if(!(boolean) mailResult.get(MAIL_ENABLED)){
+                logger.info("mail info : {} {}", title, content);
+                logger.warn("mail wasn't sent since the mail config isn't set");
+            }else if(!(boolean) mailResult.get(STATUS)){
                 throw new RuntimeException("send mail failed!");
             }
         }else{