[1.3.5-prepare][cherry-pick]#4473 Hive JDBC partial permission parame… (#4608)

* [1.3.5-prepare][cherry-pick]#4473 Hive JDBC partial permission parameter concatenation JDBC URL error
issue #4472
pr #4473

* remove unused class

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -983,6 +983,13 @@ public final class Constants {
     public static final int NORAML_NODE_STATUS = 0;
     public static final int ABNORMAL_NODE_STATUS = 1;
 
+    /**
+     * datasource encryption salt
+     */
+    public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*";
+    public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable";
+    public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt";
+
 
     /**
      * exec shell scripts

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@@ -18,12 +18,13 @@ package org.apache.dolphinscheduler.common.utils;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.net.URL;
 
 /**
@@ -33,7 +34,7 @@ public class CommonUtils {
   private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
 
   private CommonUtils() {
-    throw new IllegalStateException("CommonUtils class");
+        throw new UnsupportedOperationException("Construct CommonUtils");
   }
 
   /**

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HiveConfUtils.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.utils;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * hive conf utils
+ */
+public class HiveConfUtils {
+
+    private HiveConfUtils() {
+        throw new UnsupportedOperationException("Construct HiveConfUtils");
+    }
+
+    private static class HiveConfHandler {
+        private static HiveConf singleton;
+
+        private static Map<String,Object> hiveConfVars;
+
+        static {
+            singleton = new HiveConf();
+            hiveConfVars = new HashMap<>();
+            Arrays.stream(ConfVars.values()).forEach(confVar -> hiveConfVars.put(confVar.varname,confVar));
+        }
+    }
+
+    /**
+     * get HiveConf instance
+     * @return HiveConf hiveConf
+     */
+    public static HiveConf getInstance() {
+        return HiveConfHandler.singleton;
+    }
+
+    /**
+     * get hive conf vars
+     * @return
+     */
+    public static Map<String,Object> getHiveConfVars() {
+        return HiveConfHandler.hiveConfVars;
+    }
+
+    /**
+     * Determine if it belongs to a hive conf property
+     * @param conf config
+     * @return boolean result
+     */
+    public static boolean isHiveConfVar(String conf) {
+        // the default hive conf var name
+        String confKey = conf.split("=")[0];
+        Map<String, Object> hiveConfVars = HiveConfUtils.getHiveConfVars();
+        if (hiveConfVars.get(confKey) != null) {
+            return true;
+        }
+
+        // the security authorization hive conf var name
+        HiveConf hiveConf = HiveConfUtils.getInstance();
+        String hiveAuthorizationSqlStdAuthConfigWhitelist = hiveConf.getVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST);
+        Pattern modWhiteListPattern = Pattern.compile(hiveAuthorizationSqlStdAuthConfigWhitelist);
+        Matcher matcher = modWhiteListPattern.matcher(confKey);
+        return matcher.matches();
+    }
+
+}
\ No newline at end of file

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
+import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
+
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
 import org.apache.commons.io.IOUtils;
@@ -28,7 +30,8 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
 
-import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * property utils
@@ -44,7 +47,7 @@ public class PropertyUtils {
     private static final Properties properties = new Properties();
 
     private PropertyUtils() {
-        throw new IllegalStateException("PropertyUtils class");
+        throw new UnsupportedOperationException("Construct PropertyUtils");
     }
 
     static {
@@ -72,7 +75,7 @@ public class PropertyUtils {
      * @return  judge whether resource upload startup
      */
     public static Boolean getResUploadStartupState(){
-        String resUploadStartupType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE);
+        String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
         ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
         return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3;
     }
@@ -87,6 +90,16 @@ public class PropertyUtils {
         return properties.getProperty(key.trim());
     }
 
+    /**
+     * get property value with upper case
+     *
+     * @param key property name
+     * @return property value  with upper case
+     */
+    public static String getUpperCaseString(String key) {
+        return properties.getProperty(key.trim()).toUpperCase();
+    }
+
     /**
      * get property value
      *
@@ -240,4 +253,12 @@ public class PropertyUtils {
         }
         return matchedProperties;
     }
+
+    /**
+     *
+     */
+    public static void setValue(String key, String value) {
+        properties.setProperty(key, value);
+    }
+
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * hive conf utils test
+ */
+public class HiveConfUtilsTest {
+
+    /**
+     * test is hive conf var
+     */
+    @Test
+    public void testIsHiveConfVar() {
+
+        String conf = "hive.exec.script.wrapper=123";
+        boolean hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
+        Assert.assertTrue(hiveConfVar);
+
+        conf = "hive.test.v1=v1";
+        hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
+        Assert.assertFalse(hiveConfVar);
+
+        conf = "tez.queue.name=tezQueue";
+        hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
+        Assert.assertTrue(hiveConfVar);
+
+    }
+}

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java

@@ -16,11 +16,13 @@
  */
 package org.apache.dolphinscheduler.dao.datasource;
 
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
 import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -97,7 +99,7 @@ public abstract class BaseDataSource {
    * append database
    * @param jdbcUrl jdbc url
    */
-  private void appendDatabase(StringBuilder jdbcUrl) {
+    protected void appendDatabase(StringBuilder jdbcUrl) {
     if (dbTypeSelector() == DbType.SQLSERVER) {
       jdbcUrl.append(";databaseName=").append(getDatabase());
     } else {
@@ -149,31 +151,20 @@ public abstract class BaseDataSource {
     }
   }
 
+    /**
+     * the data source test connection
+     * @return Connection Connection
+     * @throws Exception Exception
+     */
+    public Connection getConnection() throws Exception {
+        Class.forName(driverClassSelector());
+        return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
+    }
+
   protected String filterOther(String otherParams){
     return otherParams;
   }
 
-  /**
-   * test whether the data source can be connected successfully
-   */
-  public void isConnectable() {
-    Connection con = null;
-    try {
-      Class.forName(driverClassSelector());
-      con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
-    } catch (ClassNotFoundException | SQLException e) {
-      logger.error("Get connection error: {}", e.getMessage());
-    } finally {
-      if (con != null) {
-        try {
-          con.close();
-        } catch (SQLException e) {
-          logger.error(e.getMessage(), e);
-        }
-      }
-    }
-  }
-
   public String getUser() {
     return user;
   }
@@ -182,6 +173,10 @@ public abstract class BaseDataSource {
     this.user = user;
   }
 
+    /**
+     * password need decode
+     * @return
+     */
   public String getPassword() {
     return password;
   }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java

@@ -18,6 +18,11 @@ package org.apache.dolphinscheduler.dao.datasource;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.utils.CommonUtils;
+import org.apache.dolphinscheduler.common.utils.HiveConfUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
+
+import java.sql.Connection;
 
 /**
  * data source of hive
@@ -40,4 +45,58 @@ public class HiveDataSource extends BaseDataSource {
   public DbType dbTypeSelector() {
     return DbType.HIVE;
   }
+
+    /**
+     * build hive jdbc params,append : ?hive_conf_list
+     *
+     * hive jdbc url template:
+     *
+     * jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
+     *
+     * @param otherParams otherParams
+     * @return filter otherParams
+     */
+    @Override
+    protected String filterOther(String otherParams) {
+        if (StringUtils.isBlank(otherParams)) {
+            return "";
+        }
+
+        StringBuilder hiveConfListSb = new StringBuilder();
+        hiveConfListSb.append("?");
+        StringBuilder sessionVarListSb = new StringBuilder();
+
+        String[] otherArray = otherParams.split(";", -1);
+
+        for (String conf : otherArray) {
+            if (HiveConfUtils.isHiveConfVar(conf)) {
+                hiveConfListSb.append(conf).append(";");
+            } else {
+                sessionVarListSb.append(conf).append(";");
+            }
+        }
+
+        // remove the last ";"
+        if (sessionVarListSb.length() > 0) {
+            sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);
+        }
+
+        if (hiveConfListSb.length() > 0) {
+            hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);
+        }
+
+        return sessionVarListSb.toString() + hiveConfListSb.toString();
+    }
+
+    /**
+     * the data source test connection
+     * @return Connection Connection
+     * @throws Exception Exception
+     */
+    @Override
+    public Connection getConnection() throws Exception {
+        CommonUtils.loadKerberosConf();
+        return super.getConnection();
+    }
+
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java

@@ -49,27 +49,6 @@ public class SQLServerDataSource extends BaseDataSource {
         return jdbcUrl;
     }
 
-    /**
-     * test whether the data source can be connected successfully
-     */
-    @Override
-    public void isConnectable() {
-        Connection con = null;
-        try {
-            Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER);
-            con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
-        } catch (Exception e) {
-            logger.error("error", e);
-        } finally {
-            if (con != null) {
-                try {
-                    con.close();
-                } catch (SQLException e) {
-                    logger.error("SQL Server datasource try conn close conn error", e);
-                }
-            }
-        }
-    }
   /**
    * @return driver class
    */

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.datasource;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * test data source of hive
+ */
+public class HiveDataSourceTest {
+
+    @Test
+    public void testFilterOther() {
+        BaseDataSource hiveDataSource = new HiveDataSource();
+
+        // not contain hive_site_conf
+        String other = hiveDataSource.filterOther("charset=UTF-8");
+        Assert.assertEquals("charset=UTF-8", other);
+
+        // not contain
+        other = hiveDataSource.filterOther("");
+        Assert.assertEquals("", other);
+
+        // only contain hive_site_conf
+        other = hiveDataSource.filterOther("hive.mapred.mode=strict");
+        Assert.assertEquals("?hive.mapred.mode=strict", other);
+
+        // contain hive_site_conf at the first
+        other = hiveDataSource.filterOther("hive.mapred.mode=strict;charset=UTF-8");
+        Assert.assertEquals("charset=UTF-8?hive.mapred.mode=strict", other);
+
+        // contain hive_site_conf in the middle
+        other = hiveDataSource.filterOther("charset=UTF-8;hive.mapred.mode=strict;foo=bar");
+        Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other);
+
+        // contain hive_site_conf at the end
+        other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict");
+        Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other);
+
+        // contain multi hive_site_conf
+        other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict;hive.exec.parallel=true");
+        Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict;hive.exec.parallel=true", other);
+
+        // the security authorization hive conf var
+        other = hiveDataSource.filterOther("tez.queue.name=tezTest");
+        Assert.assertEquals("?tez.queue.name=tezTest", other);
+
+    }
+
+    @Test
+    public void testGetHiveJdbcUrlOther() {
+
+        BaseDataSource hiveDataSource = new HiveDataSource();
+        hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000");
+        hiveDataSource.setDatabase("test");
+        hiveDataSource.setPassword("123456");
+        hiveDataSource.setUser("test");
+        Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl());
+
+        hiveDataSource.setOther("charset=UTF-8;hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
+
+        Assert.assertEquals(
+                "jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
+                hiveDataSource.getJdbcUrl());
+
+        hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
+        Assert.assertEquals(
+                "jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
+                hiveDataSource.getJdbcUrl());
+
+    }
+
+}

pom.xml

@@ -772,6 +772,7 @@
                         <include>**/common/utils/TaskParametersUtilsTest.java</include>
                         <include>**/common/utils/HadoopUtilsTest.java</include>
                         <include>**/common/utils/HttpUtilsTest.java</include>
+                        <include>**/common/utils/HiveConfUtilsTest.java</include>
                         <include>**/common/ConstantsTest.java</include>
                         <include>**/common/utils/HadoopUtils.java</include>
                         <include>**/common/plugin/FilePluginManagerTest</include>
@@ -781,6 +782,7 @@
                         <include>**/dao/mapper/CommandMapperTest.java</include>
                         <include>**/dao/mapper/ConnectionFactoryTest.java</include>
                         <include>**/dao/mapper/DataSourceMapperTest.java</include>
+                        <include>**/dao/datasource/HiveDataSourceTest.java</include>
                         <include>**/remote/RemoveTaskLogResponseCommandTest.java</include>
                         <include>**/remote/RemoveTaskLogRequestCommandTest.java</include>
                         <include>**/remote/FastJsonSerializerTest.java</include>