[Fix-3262][common] When you request the URL through applicationID to get the...

[Fix-3262][common]   When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled  (#3264)

* fix bug #3165   get resource.storage.type value   toUpperCase

* fix bug #3176  optimize Gets the value of this property “resource.storage.type”

* fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled

* fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled

* fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled

* fix miss

* add KerberosHttpClient fix  kerberos bug

* fix map init

* add juint test

* Extraction of public methods

* Extraction of public methods

* Fix code quality

* Fix code quality

* Fix code quality

* Fix code quality

* Fix code quality

* Fix code quality

* Fix code quality

* rebuild test

* rebuild test

* check style

* check style

* check style

* revert UT POM

* Kerberos judgment goes up to hadoopUtils

* fix merge

* Remove connection pool shutdown
Co-authored-by: Ndailidong <dailidong66@gmail.com>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@@ -417,7 +417,12 @@ public class HadoopUtils implements Closeable {
         String applicationUrl = getApplicationUrl(applicationId);
         logger.info("applicationUrl={}", applicationUrl);
 
-        String responseContent = HttpUtils.get(applicationUrl);
+        String responseContent ;
+		if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) {
+			responseContent = KerberosHttpClient.get(applicationUrl);
+		} else {
+			responseContent = HttpUtils.get(applicationUrl);
+		}
         if (responseContent != null) {
             ObjectNode jsonObject = JSONUtils.parseObject(responseContent);
             result = jsonObject.path("app").path("finalStatus").asText();

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@@ -128,40 +128,50 @@ public class HttpUtils {
 		CloseableHttpClient httpclient = HttpUtils.getInstance();
 
 		HttpGet httpget = new HttpGet(url);
-		String responseContent = null;
-		CloseableHttpResponse response = null;
-
-		try {
-			response = httpclient.execute(httpget);
-			//check response status is 200
-			if (response.getStatusLine().getStatusCode() == 200) {
-				HttpEntity entity = response.getEntity();
-				if (entity != null) {
-					responseContent = EntityUtils.toString(entity, Constants.UTF_8);
-				}else{
-					logger.warn("http entity is null");
-				}
-			}else{
-				logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
-			}
-		}catch (Exception e){
-			logger.error(e.getMessage(),e);
-		}finally {
-			try {
-				if (response != null) {
-					EntityUtils.consume(response.getEntity());
-					response.close();
-				}
-			} catch (IOException e) {
-				logger.error(e.getMessage(),e);
-			}
-
-			if (!httpget.isAborted()) {
-				httpget.releaseConnection();
-				httpget.abort();
-			}
+		return getResponseContentString(httpget,httpclient);
+    }
+
+    /**
+     * get http response content
+     *
+     * @param httpget    httpget
+     * @param httpClient httpClient
+     * @return http get request response content
+     */
+    public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) {
+	String responseContent = null;
+	CloseableHttpResponse response = null;
+	try {
+	    response = httpClient.execute(httpget);
+	    // check response status is 200
+	    if (response.getStatusLine().getStatusCode() == 200) {
+		HttpEntity entity = response.getEntity();
+		if (entity != null) {
+		    responseContent = EntityUtils.toString(entity, Constants.UTF_8);
+		} else {
+		    logger.warn("http entity is null");
 		}
-		return responseContent;
+	    } else {
+		logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
+	    }
+	} catch (IOException ioe) {
+	    logger.error(ioe.getMessage(), ioe);
+	} finally {
+	    try {
+		if (response != null) {
+		    EntityUtils.consume(response.getEntity());
+		    response.close();
+		}
+	    } catch (IOException e) {
+		logger.error(e.getMessage(), e);
+	    }
+	    if (!httpget.isAborted()) {
+		httpget.releaseConnection();
+		httpget.abort();
+	    }
+
 	}
+	return responseContent;
+    }
 
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common.utils;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.config.Lookup;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.auth.Subject;
+import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
+import java.security.Principal;
+import java.security.PrivilegedAction;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * kerberos http client
+ */
+public class KerberosHttpClient {
+
+    public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClient.class);
+
+    private String principal;
+    private String keyTabLocation;
+
+    public KerberosHttpClient(String principal, String keyTabLocation) {
+	super();
+	this.principal = principal;
+	this.keyTabLocation = keyTabLocation;
+    }
+
+    public KerberosHttpClient(String principal, String keyTabLocation, boolean isDebug) {
+	this(principal, keyTabLocation);
+	if (isDebug) {
+	    System.setProperty("sun.security.spnego.debug", "true");
+	    System.setProperty("sun.security.krb5.debug", "true");
+	}
+    }
+
+    public KerberosHttpClient(String principal, String keyTabLocation, String krb5Location, boolean isDebug) {
+	this(principal, keyTabLocation, isDebug);
+	System.setProperty("java.security.krb5.conf", krb5Location);
+    }
+
+    private static CloseableHttpClient buildSpengoHttpClient() {
+	HttpClientBuilder builder = HttpClientBuilder.create();
+	Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
+		.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
+	builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
+	BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+	credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
+	    @Override
+	    public Principal getUserPrincipal() {
+		return null;
+	    }
+
+	    @Override
+	    public String getPassword() {
+		return null;
+	    }
+	});
+	builder.setDefaultCredentialsProvider(credentialsProvider);
+	return builder.build();
+    }
+
+    public String get(final String url, final String userId) {
+	logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url);
+	Configuration config = new Configuration() {
+	    @SuppressWarnings("serial")
+	    @Override
+	    public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
+		Map<String, Object> options = new HashMap<>(9);
+		options.put("useTicketCache", "false");
+		options.put("useKeyTab", "true");
+		options.put("keyTab", keyTabLocation);
+		options.put("refreshKrb5Config", "true");
+		options.put("principal", principal);
+		options.put("storeKey", "true");
+		options.put("doNotPrompt", "true");
+		options.put("isInitiator", "true");
+		options.put("debug", "true");
+		return new AppConfigurationEntry[] {
+			new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule",
+				AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) };
+	    }
+	};
+	Set<Principal> princ = new HashSet<>(1);
+	princ.add(new KerberosPrincipal(userId));
+	Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>());
+
+	LoginContext lc;
+	try {
+	    lc = new LoginContext("", sub, null, config);
+	    lc.login();
+	    Subject serviceSubject = lc.getSubject();
+	    return Subject.doAs(serviceSubject, (PrivilegedAction<String>) () -> {
+		CloseableHttpClient httpClient = buildSpengoHttpClient();
+		HttpGet httpget = new HttpGet(url);
+		return HttpUtils.getResponseContentString(httpget, httpClient);
+	    });
+	} catch (LoginException le) {
+	    logger.error("Kerberos authentication failed ", le);
+	}
+	return null;
+    }
+
+    /**
+     * get http request content by kerberosClient
+     *
+     * @param url url
+     * @return http get request response content
+     */
+    public static String get(String url) {
+
+	String responseContent;
+	KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(
+		PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
+		PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH),
+		PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
+	responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
+	return responseContent;
+
+    }
+
+}

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java

@@ -17,7 +17,13 @@
 package org.apache.dolphinscheduler.common.utils;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -28,26 +34,53 @@ import org.slf4j.LoggerFactory;
  */
 public class HttpUtilsTest {
 
+    public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class);
+    private HadoopUtils hadoopUtils = HadoopUtils.getInstance();
 
-	public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class);
+    @Test
+    public void testGetTest() {
+	// success
+	String result = HttpUtils.get("https://github.com/manifest.json");
+	Assert.assertNotNull(result);
+	ObjectNode jsonObject = JSONUtils.parseObject(result);
+	Assert.assertEquals("GitHub", jsonObject.path("name").asText());
+	result = HttpUtils.get("https://123.333.111.33/ccc");
+	Assert.assertNull(result);
+    }
 
+    @Test
+    public void testGetByKerberos() {
+	try {
+	    String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029");
+	    String responseContent;
+	    responseContent = HttpUtils.get(applicationUrl);
+	    Assert.assertNull(responseContent);
 
-	@Test
-	public void testGetTest(){
-		//success
-		String result = HttpUtils.get("https://github.com/manifest.json");
-		Assert.assertNotNull(result);
-		ObjectNode jsonObject = JSONUtils.parseObject(result);
-		Assert.assertEquals("GitHub", jsonObject.path("name").asText());
-
-		result = HttpUtils.get("https://123.333.111.33/ccc");
-		Assert.assertNull(result);
+	} catch (Exception e) {
+	    logger.error(e.getMessage(), e);
 	}
 
+    }
+
+    @Test
+    public void testGetResponseContentString() {
+	CloseableHttpClient httpclient = HttpClients.createDefault();
+	HttpGet httpget = new HttpGet("https://github.com/manifest.json");
+	/** set timeout、request time、socket timeout */
+	RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
+		.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
+		.setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build();
+	httpget.setConfig(requestConfig);
+	String responseContent = HttpUtils.getResponseContentString(httpget, httpclient);
+	Assert.assertNotNull(responseContent);
+    }
+
+
 	@Test
 	public void testGetHttpClient() {
 		CloseableHttpClient httpClient1 = HttpUtils.getInstance();
 		CloseableHttpClient httpClient2 = HttpUtils.getInstance();
 		Assert.assertEquals(httpClient1, httpClient2);
 	}
+
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.common.utils;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * KerberosHttpClient  test
+ */
+public class KerberosHttpClientTest {
+    public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClientTest.class);
+    private HadoopUtils hadoopUtils = HadoopUtils.getInstance();
+
+    @Test
+    public void get() {
+        try {
+            String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029");
+            String responseContent;
+            KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
+                    PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
+            responseContent = kerberosHttpClient.get(applicationUrl,
+                    PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
+            Assert.assertNull(responseContent);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+}
\ No newline at end of file

pom.xml

@@ -779,6 +779,7 @@
                         <include>**/common/utils/TaskParametersUtilsTest.java</include>
                         <include>**/common/utils/HadoopUtilsTest.java</include>
                         <include>**/common/utils/HttpUtilsTest.java</include>
+                        <include>**/common/utils/KerberosHttpClientTest.java</include>
                         <include>**/common/ConstantsTest.java</include>
                         <include>**/common/utils/HadoopUtils.java</include>
                         <include>**/common/utils/RetryerUtilsTest.java</include>