[Feature-10498] Mask the password in the log of sqoop task (#11589)

docs/docs/en/architecture/design.md

@@ -197,7 +197,7 @@ In the early schedule design, if there is no priority design and use the fair sc
 - For details, please refer to the logback configuration of Master and Worker, as shown in the following example:
 
 ```xml
-<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
+<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
 <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
     <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
     <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

docs/docs/zh/architecture/design.md

@@ -195,7 +195,7 @@
 - 详情可参考Master和Worker的logback配置，如下示例：
 
 ```xml
-<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
+<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
 <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
     <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
     <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java

@@ -15,11 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.service.log;
+package org.apache.dolphinscheduler.common.log;
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
 
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -33,10 +37,9 @@ import com.google.common.base.Strings;
  */
 public class SensitiveDataConverter extends MessageConverter {
 
-    /**
-     * password pattern
-     */
-    private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX);
+    private static Pattern multilinePattern;
+    private static HashSet<String> maskPatterns =
+            new HashSet<>(Arrays.asList(DataSourceConstants.DATASOURCE_PASSWORD_REGEX));
 
     @Override
     public String convert(ILoggingEvent event) {
@@ -45,41 +48,25 @@ public class SensitiveDataConverter extends MessageConverter {
         String requestLogMsg = event.getFormattedMessage();
 
         // desensitization log
-        return convertMsg(requestLogMsg);
+        return maskSensitiveData(requestLogMsg);
     }
 
-    /**
-     * deal with sensitive log
-     *
-     * @param oriLogMsg original log
-     */
-    private String convertMsg(final String oriLogMsg) {
-
-        String tempLogMsg = oriLogMsg;
-
-        if (!Strings.isNullOrEmpty(tempLogMsg)) {
-            tempLogMsg = passwordHandler(pwdPattern, tempLogMsg);
-        }
-        return tempLogMsg;
+    public static void addMaskPattern(String maskPattern) {
+        maskPatterns.add(maskPattern);
     }
 
-    /**
-     * password regex
-     *
-     * @param logMsg original log
-     */
-    static String passwordHandler(Pattern pwdPattern, String logMsg) {
-
-        Matcher matcher = pwdPattern.matcher(logMsg);
+    public static String maskSensitiveData(final String logMsg) {
+        if (StringUtils.isEmpty(logMsg)) {
+            return logMsg;
+        }
+        multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE);
 
         StringBuffer sb = new StringBuffer(logMsg.length());
+        Matcher matcher = multilinePattern.matcher(logMsg);
 
         while (matcher.find()) {
-
             String password = matcher.group();
-
             String maskPassword = Strings.repeat(Constants.STAR, password.length());
-
             matcher.appendReplacement(sb, maskPassword);
         }
         matcher.appendTail(sb);

dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java → dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java

@@ -15,13 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.service.log;
-
-import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler;
-
-import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
-
-import java.util.regex.Pattern;
+package org.apache.dolphinscheduler.common.log;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -32,11 +26,6 @@ public class SensitiveDataConverterTest {
 
     private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class);
 
-    /**
-     * password pattern
-     */
-    private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX);
-
     private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\","
             + "\"database\":\"carbond\","
             + "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\","
@@ -49,21 +38,17 @@ public class SensitiveDataConverterTest {
             + "\"user\":\"view\","
             + "\"password\":\"*****\"}";
 
-    @Test
-    public void convert() {
-        Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
-    }
-
     /**
      * mask sensitive logMsg - sql task datasource password
      */
     @Test
     public void testPwdLogMsgConverter() {
-        logger.info("parameter : {}", logMsg);
-        logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg));
+        final String maskedLog = SensitiveDataConverter.maskSensitiveData(logMsg);
+
+        logger.info("original parameter : {}", logMsg);
+        logger.info("masked parameter : {}", maskedLog);
 
-        Assertions.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg));
-        Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
+        Assertions.assertEquals(maskLogMsg, maskedLog);
 
     }
 

dolphinscheduler-master/src/main/resources/logback-spring.xml

@@ -28,7 +28,7 @@
     </appender>
 
     <conversionRule conversionWord="message"
-                    converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
+                    converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
     <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
         <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
         <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml

@@ -48,7 +48,7 @@
     <logger name="org.apache.hadoop" level="WARN"/>
 
     <conversionRule conversionWord="message"
-                    converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
+                    converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
     <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
         <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
         <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">

dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java

@@ -72,4 +72,5 @@ public final class SqoopConstants {
     public static final String UPDATE_KEY = "--update-key";
     public static final String UPDATE_MODE = "--update-mode";
 
+    public static final String SQOOP_PASSWORD_REGEX = "(?<=(--password \")).+?(?=\")";
 }

dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java

@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.plugin.task.sqoop;
 
+import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@@ -67,6 +68,8 @@ public class SqoopTask extends AbstractYarnTask {
 
         sqoopTaskExecutionContext =
                 sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper());
+
+        SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX);
     }
 
     @Override

dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.task.sqoop;
+
+import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class SqoopTaskTest {
+
+    @Test
+    public void testSqoopPasswordMask() {
+        final String originalScript =
+                "sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"mypassword\" --table student --target-dir /sqoop_test --as-textfile";
+
+        final String maskScript =
+                "sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"**********\" --table student --target-dir /sqoop_test --as-textfile";
+
+        SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX);
+        Assertions.assertEquals(maskScript, SensitiveDataConverter.maskSensitiveData(originalScript));
+    }
+}

dolphinscheduler-worker/src/main/resources/logback-spring.xml

@@ -29,7 +29,7 @@
     </appender>
 
     <conversionRule conversionWord="message"
-                    converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
+                    converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
     <appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
         <filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
         <Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">