diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index 055937b49ca8966ca898aa842f5383a6c0c92d29..e64b0395ecdd6380261b1442f224775e576b7998 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -14,10 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; /** * data source of hive @@ -40,4 +46,51 @@ public class HiveDataSource extends BaseDataSource { public DbType dbTypeSelector() { return DbType.HIVE; } + + /** + * build hive jdbc params,append : ?hive_conf_list + * + * hive jdbc url template: + * + * jdbc:hive2://:,:/dbName;initFile=;sess_var_list?hive_conf_list#hive_var_list + * + * @param otherParams otherParams + * @return filter otherParams + */ + @Override + protected String filterOther(String otherParams) { + if (StringUtils.isBlank(otherParams)) { + return ""; + } + + StringBuilder hiveConfListSb = new StringBuilder(); + hiveConfListSb.append("?"); + StringBuilder sessionVarListSb = new StringBuilder(); + + String[] otherArray = otherParams.split(";", -1); + + // get the default hive conf var name + Set hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname) + .collect(Collectors.toSet()); + + for (String conf : otherArray) { + if (hiveConfSet.contains(conf.split("=")[0])) { + hiveConfListSb.append(conf).append(";"); + } else { + sessionVarListSb.append(conf).append(";"); + } + } + + // remove the last ";" + if (sessionVarListSb.length() > 0) { + sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); + } + + if (hiveConfListSb.length() > 0) { + hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); + } + + return sessionVarListSb.toString() + hiveConfListSb.toString(); + } + } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..36a505872fa27ffc0bb230afbf14dab5d3ce3ff7 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.datasource; + +import org.junit.Assert; +import org.junit.Test; + +/** + * test data source of hive + */ +public class HiveDataSourceTest { + + @Test + public void testfilterOther() { + BaseDataSource hiveDataSource = new HiveDataSource(); + + // not contain hive_site_conf + String other = hiveDataSource.filterOther("charset=UTF-8"); + Assert.assertEquals("charset=UTF-8", other); + + // not contain + other = hiveDataSource.filterOther(""); + Assert.assertEquals("", other); + + // only contain hive_site_conf + other = hiveDataSource.filterOther("hive.mapred.mode=strict"); + Assert.assertEquals("?hive.mapred.mode=strict", other); + + // contain hive_site_conf at the first + other = hiveDataSource.filterOther("hive.mapred.mode=strict;charset=UTF-8"); + Assert.assertEquals("charset=UTF-8?hive.mapred.mode=strict", other); + + // contain hive_site_conf in the middle + other = hiveDataSource.filterOther("charset=UTF-8;hive.mapred.mode=strict;foo=bar"); + Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other); + + // contain hive_site_conf at the end + other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict"); + Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other); + + // contain multi hive_site_conf + other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict;hive.exec.parallel=true"); + Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict;hive.exec.parallel=true", other); + } + + @Test + public void testGetHiveJdbcUrlOther() { + + BaseDataSource hiveDataSource = new HiveDataSource(); + hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000"); + hiveDataSource.setDatabase("test"); + hiveDataSource.setPassword("123456"); + hiveDataSource.setUser("test"); + Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl()); + + hiveDataSource.setOther("charset=UTF-8;hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2"); + + Assert.assertEquals( + "jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2", + hiveDataSource.getJdbcUrl()); + + hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2"); + + Assert.assertEquals( + "jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2", + hiveDataSource.getJdbcUrl()); + + } + +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index b012fff0ed3aba75dc003cb57e109ae4855581ac..c8449cb21b19ebe6b91918250c842d99e59e3dc9 100644 --- a/pom.xml +++ b/pom.xml @@ -852,6 +852,7 @@ **/dao/utils/DagHelperTest.java **/dao/AlertDaoTest.java **/dao/datasource/OracleDataSourceTest.java + **/dao/datasource/HiveDataSourceTest.java **/dao/upgrade/ProcessDefinitionDaoTest.java **/dao/upgrade/WokrerGrouopDaoTest.java **/dao/upgrade/UpgradeDaoTest.java