diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 58e8ac7780d3475668228892af3bdaa25d636ae1..96d2369b80ff83db52f1f9293fe54b711341d289 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -91,30 +91,48 @@ public class CommonUtils { /** * load kerberos configuration * - * @throws Exception errors + * @param configuration + * @return load kerberos config return true + * @throws IOException errors */ - public static void loadKerberosConf() throws Exception { - loadKerberosConf(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), + public static boolean loadKerberosConf(Configuration configuration) throws IOException { + return loadKerberosConf(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), configuration); } /** * load kerberos configuration + * * @param javaSecurityKrb5Conf javaSecurityKrb5Conf * @param loginUserKeytabUsername loginUserKeytabUsername * @param loginUserKeytabPath loginUserKeytabPath * @throws IOException errors */ public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException { + loadKerberosConf(javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath, new Configuration()); + } + + /** + * load kerberos configuration + * + * @param javaSecurityKrb5Conf javaSecurityKrb5Conf + * @param loginUserKeytabUsername loginUserKeytabUsername + * @param loginUserKeytabPath loginUserKeytabPath + * @param configuration configuration + * @return load kerberos config return true + * @throws IOException errors + */ + public static boolean loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath, Configuration configuration) throws IOException { if (CommonUtils.getKerberosStartupState()) { System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH))); - Configuration configuration = new Configuration(); configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS); UserGroupInformation.setConfiguration(configuration); UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)), StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH))); + return true; } + return false; } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index b76f2b19778f5046ad894c8cb1e28c4a8a6b8507..2dbc4848e9b1456ebcd426525c516ee6efc6517f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -127,14 +127,8 @@ public class HadoopUtils implements Closeable { ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType); if (resUploadType == ResUploadType.HDFS) { - if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) { - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, - PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + if (CommonUtils.loadKerberosConf(configuration)) { hdfsUser = ""; - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); } String defaultFS = configuration.get(Constants.FS_DEFAULTFS); @@ -156,20 +150,15 @@ public class HadoopUtils implements Closeable { logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); } - if (fs == null) { - if (StringUtils.isNotEmpty(hdfsUser)) { - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Boolean run() throws Exception { - fs = FileSystem.get(configuration); - return true; - } - }); - } else { - logger.warn("hdfs.root.user is not set value!"); + if (StringUtils.isNotEmpty(hdfsUser)) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); + ugi.doAs((PrivilegedExceptionAction) () -> { fs = FileSystem.get(configuration); - } + return true; + }); + } else { + logger.warn("hdfs.root.user is not set value!"); + fs = FileSystem.get(configuration); } } else if (resUploadType == ResUploadType.S3) { System.setProperty(Constants.AWS_S3_V4, Constants.STRING_TRUE); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java index cb038e95035482b76b20e2b0df2a7727ad5359ea..92f0d7bd49c2b47178c12424d3919ad25e87567e 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java @@ -14,20 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; + +import java.net.InetAddress; +import java.net.UnknownHostException; + import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.InetAddress; -import java.net.UnknownHostException; - /** * configuration test */ +@RunWith(PowerMockRunner.class) +@PrepareForTest(value = { PropertyUtils.class, UserGroupInformation.class}) public class CommonUtilsTest { private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class); @Test @@ -35,21 +46,43 @@ public class CommonUtilsTest { logger.info(CommonUtils.getSystemEnvPath()); Assert.assertTrue(true); } + @Test public void isDevelopMode() { logger.info("develop mode: {}",CommonUtils.isDevelopMode()); Assert.assertTrue(true); } + @Test - public void getKerberosStartupState(){ - logger.info("kerberos startup state: {}",CommonUtils.getKerberosStartupState()); - Assert.assertTrue(true); + public void getKerberosStartupState() { + boolean kerberosStartupState = CommonUtils.getKerberosStartupState(); + logger.info("kerberos startup state: {}",kerberosStartupState); + Assert.assertFalse(kerberosStartupState); + PowerMockito.mockStatic(PropertyUtils.class); + PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS"); + PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE); + kerberosStartupState = CommonUtils.getKerberosStartupState(); + logger.info("kerberos startup state: {}",kerberosStartupState); + Assert.assertTrue(kerberosStartupState); + } + @Test - public void loadKerberosConf(){ + public void loadKerberosConf() { try { - CommonUtils.loadKerberosConf(); - Assert.assertTrue(true); + PowerMockito.mockStatic(PropertyUtils.class); + PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS"); + PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE); + PowerMockito.when(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)).thenReturn("/opt/krb5.conf"); + PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)).thenReturn("hdfs-mycluster@ESZ.COM"); + PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)).thenReturn("/opt/hdfs.headless.keytab"); + + PowerMockito.mockStatic(UserGroupInformation.class); + boolean result = CommonUtils.loadKerberosConf(new Configuration()); + Assert.assertTrue(result); + + CommonUtils.loadKerberosConf(null, null, null); + } catch (Exception e) { Assert.fail("load Kerberos Conf failed"); } @@ -80,11 +113,11 @@ public class CommonUtilsTest { } @Test - public void test(){ - InetAddress IP = null; + public void test() { + InetAddress ip; try { - IP = InetAddress.getLocalHost(); - logger.info(IP.getHostAddress()); + ip = InetAddress.getLocalHost(); + logger.info(ip.getHostAddress()); } catch (UnknownHostException e) { e.printStackTrace(); }