diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 8af62a4822606545598c368828fa6c945d676ea9..d67fc5754c9620ebaa199d464f09f74dfc98a20d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -983,6 +983,13 @@ public final class Constants { public static final int NORAML_NODE_STATUS = 0; public static final int ABNORMAL_NODE_STATUS = 1; + /** + * datasource encryption salt + */ + public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; + public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; + public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; + /** * exec shell scripts diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 2468e62ab0e6bc9024e7288464f3d7db8a4fa1be..029413e38c84638649c1c4dad518ac18c595205a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -18,12 +18,13 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.net.URL; /** @@ -33,7 +34,7 @@ public class CommonUtils { private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); private CommonUtils() { - throw new IllegalStateException("CommonUtils class"); + throw new UnsupportedOperationException("Construct CommonUtils"); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HiveConfUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HiveConfUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..ef11a92695316069e8cbc9213d94397aae50d93b --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HiveConfUtils.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * hive conf utils + */ +public class HiveConfUtils { + + private HiveConfUtils() { + throw new UnsupportedOperationException("Construct HiveConfUtils"); + } + + private static class HiveConfHandler { + private static HiveConf singleton; + + private static Map hiveConfVars; + + static { + singleton = new HiveConf(); + hiveConfVars = new HashMap<>(); + Arrays.stream(ConfVars.values()).forEach(confVar -> hiveConfVars.put(confVar.varname,confVar)); + } + } + + /** + * get HiveConf instance + * @return HiveConf hiveConf + */ + public static HiveConf getInstance() { + return HiveConfHandler.singleton; + } + + /** + * get hive conf vars + * @return + */ + public static Map getHiveConfVars() { + return HiveConfHandler.hiveConfVars; + } + + /** + * Determine if it belongs to a hive conf property + * @param conf config + * @return boolean result + */ + public static boolean isHiveConfVar(String conf) { + // the default hive conf var name + String confKey = conf.split("=")[0]; + Map hiveConfVars = HiveConfUtils.getHiveConfVars(); + if (hiveConfVars.get(confKey) != null) { + return true; + } + + // the security authorization hive conf var name + HiveConf hiveConf = HiveConfUtils.getInstance(); + String hiveAuthorizationSqlStdAuthConfigWhitelist = hiveConf.getVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_SQL_STD_AUTH_CONFIG_WHITELIST); + Pattern modWhiteListPattern = Pattern.compile(hiveAuthorizationSqlStdAuthConfigWhitelist); + Matcher matcher = modWhiteListPattern.matcher(confKey); + return matcher.matches(); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index ba1fcd69265bd7cbcdc32eb0b2e16b8497019d21..12c00542fc79aaa5f38f668b0fbad43737ad9644 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.common.utils; +import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.commons.io.IOUtils; @@ -28,7 +30,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * property utils @@ -44,7 +47,7 @@ public class PropertyUtils { private static final Properties properties = new Properties(); private PropertyUtils() { - throw new IllegalStateException("PropertyUtils class"); + throw new UnsupportedOperationException("Construct PropertyUtils"); } static { @@ -72,7 +75,7 @@ public class PropertyUtils { * @return judge whether resource upload startup */ public static Boolean getResUploadStartupState(){ - String resUploadStartupType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); + String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3; } @@ -87,6 +90,16 @@ public class PropertyUtils { return properties.getProperty(key.trim()); } + /** + * get property value with upper case + * + * @param key property name + * @return property value with upper case + */ + public static String getUpperCaseString(String key) { + return properties.getProperty(key.trim()).toUpperCase(); + } + /** * get property value * @@ -240,4 +253,12 @@ public class PropertyUtils { } return matchedProperties; } + + /** + * + */ + public static void setValue(String key, String value) { + properties.setProperty(key, value); + } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java new file mode 100644 index 0000000000000000000000000000000000000000..85912d5ff6900a2c0ee3a120558b1e772025ebcf --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; + +/** + * hive conf utils test + */ +public class HiveConfUtilsTest { + + /** + * test is hive conf var + */ + @Test + public void testIsHiveConfVar() { + + String conf = "hive.exec.script.wrapper=123"; + boolean hiveConfVar = HiveConfUtils.isHiveConfVar(conf); + Assert.assertTrue(hiveConfVar); + + conf = "hive.test.v1=v1"; + hiveConfVar = HiveConfUtils.isHiveConfVar(conf); + Assert.assertFalse(hiveConfVar); + + conf = "tez.queue.name=tezQueue"; + hiveConfVar = HiveConfUtils.isHiveConfVar(conf); + Assert.assertTrue(hiveConfVar); + + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index 30d5d778d37fac1d071ce0548d703f9127a5cada..d9125a2f67184ccd77fbd33e17171af4b8bb7886 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -16,11 +16,13 @@ */ package org.apache.dolphinscheduler.dao.datasource; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,7 +99,7 @@ public abstract class BaseDataSource { * append database * @param jdbcUrl jdbc url */ - private void appendDatabase(StringBuilder jdbcUrl) { + protected void appendDatabase(StringBuilder jdbcUrl) { if (dbTypeSelector() == DbType.SQLSERVER) { jdbcUrl.append(";databaseName=").append(getDatabase()); } else { @@ -149,31 +151,20 @@ public abstract class BaseDataSource { } } + /** + * the data source test connection + * @return Connection Connection + * @throws Exception Exception + */ + public Connection getConnection() throws Exception { + Class.forName(driverClassSelector()); + return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); + } + protected String filterOther(String otherParams){ return otherParams; } - /** - * test whether the data source can be connected successfully - */ - public void isConnectable() { - Connection con = null; - try { - Class.forName(driverClassSelector()); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } catch (ClassNotFoundException | SQLException e) { - logger.error("Get connection error: {}", e.getMessage()); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - } - } - public String getUser() { return user; } @@ -182,6 +173,10 @@ public abstract class BaseDataSource { this.user = user; } + /** + * password need decode + * @return + */ public String getPassword() { return password; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index 055937b49ca8966ca898aa842f5383a6c0c92d29..931a5d39c7f62c411e1f81e20da6baeeabd58451 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -18,6 +18,11 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.HiveConfUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import java.sql.Connection; /** * data source of hive @@ -40,4 +45,58 @@ public class HiveDataSource extends BaseDataSource { public DbType dbTypeSelector() { return DbType.HIVE; } + + /** + * build hive jdbc params,append : ?hive_conf_list + * + * hive jdbc url template: + * + * jdbc:hive2://:,:/dbName;initFile=;sess_var_list?hive_conf_list#hive_var_list + * + * @param otherParams otherParams + * @return filter otherParams + */ + @Override + protected String filterOther(String otherParams) { + if (StringUtils.isBlank(otherParams)) { + return ""; + } + + StringBuilder hiveConfListSb = new StringBuilder(); + hiveConfListSb.append("?"); + StringBuilder sessionVarListSb = new StringBuilder(); + + String[] otherArray = otherParams.split(";", -1); + + for (String conf : otherArray) { + if (HiveConfUtils.isHiveConfVar(conf)) { + hiveConfListSb.append(conf).append(";"); + } else { + sessionVarListSb.append(conf).append(";"); + } + } + + // remove the last ";" + if (sessionVarListSb.length() > 0) { + sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); + } + + if (hiveConfListSb.length() > 0) { + hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); + } + + return sessionVarListSb.toString() + hiveConfListSb.toString(); + } + + /** + * the data source test connection + * @return Connection Connection + * @throws Exception Exception + */ + @Override + public Connection getConnection() throws Exception { + CommonUtils.loadKerberosConf(); + return super.getConnection(); + } + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java index e4b8f4bf1348e60f28c6d8c226add8620f79bdea..6664fb608db133f18d024a1b3a25c861e0c0aa96 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java @@ -49,27 +49,6 @@ public class SQLServerDataSource extends BaseDataSource { return jdbcUrl; } - /** - * test whether the data source can be connected successfully - */ - @Override - public void isConnectable() { - Connection con = null; - try { - Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } catch (Exception e) { - logger.error("error", e); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("SQL Server datasource try conn close conn error", e); - } - } - } - } /** * @return driver class */ diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..3920fd39b21b2b294cc6a5ac10160f927b0dfe0c --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.datasource; + +import org.junit.Assert; +import org.junit.Test; + +/** + * test data source of hive + */ +public class HiveDataSourceTest { + + @Test + public void testFilterOther() { + BaseDataSource hiveDataSource = new HiveDataSource(); + + // not contain hive_site_conf + String other = hiveDataSource.filterOther("charset=UTF-8"); + Assert.assertEquals("charset=UTF-8", other); + + // not contain + other = hiveDataSource.filterOther(""); + Assert.assertEquals("", other); + + // only contain hive_site_conf + other = hiveDataSource.filterOther("hive.mapred.mode=strict"); + Assert.assertEquals("?hive.mapred.mode=strict", other); + + // contain hive_site_conf at the first + other = hiveDataSource.filterOther("hive.mapred.mode=strict;charset=UTF-8"); + Assert.assertEquals("charset=UTF-8?hive.mapred.mode=strict", other); + + // contain hive_site_conf in the middle + other = hiveDataSource.filterOther("charset=UTF-8;hive.mapred.mode=strict;foo=bar"); + Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other); + + // contain hive_site_conf at the end + other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict"); + Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other); + + // contain multi hive_site_conf + other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict;hive.exec.parallel=true"); + Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict;hive.exec.parallel=true", other); + + // the security authorization hive conf var + other = hiveDataSource.filterOther("tez.queue.name=tezTest"); + Assert.assertEquals("?tez.queue.name=tezTest", other); + + } + + @Test + public void testGetHiveJdbcUrlOther() { + + BaseDataSource hiveDataSource = new HiveDataSource(); + hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000"); + hiveDataSource.setDatabase("test"); + hiveDataSource.setPassword("123456"); + hiveDataSource.setUser("test"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl()); + + hiveDataSource.setOther("charset=UTF-8;hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2"); + + Assert.assertEquals( + "jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2", + hiveDataSource.getJdbcUrl()); + + hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2"); + Assert.assertEquals( + "jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2", + hiveDataSource.getJdbcUrl()); + + } + +} diff --git a/pom.xml b/pom.xml index 0f67fff2bb0cc022dfbec17863f4b397babe21f3..6833fd427c1adcabe44a29a01177956f37f3316a 100644 --- a/pom.xml +++ b/pom.xml @@ -772,6 +772,7 @@ **/common/utils/TaskParametersUtilsTest.java **/common/utils/HadoopUtilsTest.java **/common/utils/HttpUtilsTest.java + **/common/utils/HiveConfUtilsTest.java **/common/ConstantsTest.java **/common/utils/HadoopUtils.java **/common/plugin/FilePluginManagerTest @@ -781,6 +782,7 @@ **/dao/mapper/CommandMapperTest.java **/dao/mapper/ConnectionFactoryTest.java **/dao/mapper/DataSourceMapperTest.java + **/dao/datasource/HiveDataSourceTest.java **/remote/RemoveTaskLogResponseCommandTest.java **/remote/RemoveTaskLogRequestCommandTest.java **/remote/FastJsonSerializerTest.java