[TS-572]<fix>: fix noHttpResponseException in httpClient (#8583)

src/connector/jdbc/deploy-pom.xml

@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.35</version>
+  <version>2.0.36</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>

src/connector/jdbc/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.35</version>
+    <version>2.0.36</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
@@ -58,6 +58,13 @@
             <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+            <version>1.2</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
@@ -70,6 +77,18 @@
             </resource>
         </resources>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -135,7 +135,6 @@ public class TSDBDriver extends AbstractDriver {
             TSDBJNIConnector.init(props);
             return new TSDBConnection(props, this.dbMetaData);
         } catch (SQLWarning sqlWarning) {
-            sqlWarning.printStackTrace();
             return new TSDBConnection(props, this.dbMetaData);
         } catch (SQLException sqlEx) {
             throw sqlEx;

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java

@@ -50,9 +50,13 @@ public class RestfulDriver extends AbstractDriver {
             String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), StandardCharsets.UTF_8.displayName());
             loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + user + "/" + password + "";
         } catch (UnsupportedEncodingException e) {
-            e.printStackTrace();
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "unsupported UTF-8 concoding, user: " + props.getProperty(TSDBDriver.PROPERTY_KEY_USER) + ", password: " + props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD));
         }
 
+        int poolSize = Integer.valueOf(props.getProperty("httpPoolSize", HttpClientPoolUtil.DEFAULT_MAX_PER_ROUTE));
+        boolean keepAlive = Boolean.valueOf(props.getProperty("httpKeepAlive", HttpClientPoolUtil.DEFAULT_HTTP_KEEP_ALIVE));
+
+        HttpClientPoolUtil.init(poolSize, keepAlive);
         String result = HttpClientPoolUtil.execute(loginUrl);
         JSONObject jsonResult = JSON.parseObject(result);
         String status = jsonResult.getString("status");

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java

@@ -9,6 +9,7 @@ import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.*;
 import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.conn.ClientConnectionManager;
 import org.apache.http.conn.ConnectionKeepAliveStrategy;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -22,15 +23,17 @@ import org.apache.http.util.EntityUtils;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.sql.SQLException;
+import java.util.concurrent.TimeUnit;
 
 public class HttpClientPoolUtil {
 
     private static final String DEFAULT_CONTENT_TYPE = "application/json";
     private static final int DEFAULT_MAX_RETRY_COUNT = 5;
 
-    private static final int DEFAULT_MAX_TOTAL = 50;
-    private static final int DEFAULT_MAX_PER_ROUTE = 5;
+    public static final String DEFAULT_HTTP_KEEP_ALIVE = "true";
+    public static final String DEFAULT_MAX_PER_ROUTE = "20";
     private static final int DEFAULT_HTTP_KEEP_TIME = -1;
+    private static String isKeepAlive;
 
     private static final ConnectionKeepAliveStrategy DEFAULT_KEEP_ALIVE_STRATEGY = (response, context) -> {
         HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
@@ -48,37 +51,41 @@ public class HttpClientPoolUtil {
         return DEFAULT_HTTP_KEEP_TIME * 1000;
     };
 
-    private static final CloseableHttpClient httpClient;
+    private static CloseableHttpClient httpClient;
 
-    static {
-        PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
-        connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL);
-        connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE);
-
-        httpClient = HttpClients.custom()
-                .setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY)
-                .setConnectionManager(connectionManager)
-                .setRetryHandler((exception, executionCount, httpContext) -> executionCount < DEFAULT_MAX_RETRY_COUNT)
-                .build();
+    public static void init(Integer connPoolSize, boolean keepAlive) {
+        if (httpClient == null) {
+            synchronized (HttpClientPoolUtil.class) {
+                if (httpClient == null) {
+                    isKeepAlive = keepAlive ? HTTP.CONN_KEEP_ALIVE : HTTP.CONN_CLOSE;
+                    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+                    connectionManager.setMaxTotal(connPoolSize * 10);
+                    connectionManager.setDefaultMaxPerRoute(connPoolSize);
+                    httpClient = HttpClients.custom()
+                            .setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY)
+                            .setConnectionManager(connectionManager)
+                            .setRetryHandler((exception, executionCount, httpContext) -> executionCount < DEFAULT_MAX_RETRY_COUNT)
+                            .build();
+                }
+            }
+        }
     }
 
     /*** execute GET request ***/
     public static String execute(String uri) throws SQLException {
         HttpEntity httpEntity = null;
         String responseBody = "";
-        try {
-            HttpRequestBase method = getRequest(uri, HttpGet.METHOD_NAME);
-            HttpContext context = HttpClientContext.create();
-            CloseableHttpResponse httpResponse = httpClient.execute(method, context);
+        HttpRequestBase method = getRequest(uri, HttpGet.METHOD_NAME);
+        HttpContext context = HttpClientContext.create();
+
+        try (CloseableHttpResponse httpResponse = httpClient.execute(method, context)) {
             httpEntity = httpResponse.getEntity();
             if (httpEntity != null) {
                 responseBody = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8);
             }
         } catch (ClientProtocolException e) {
-            e.printStackTrace();
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_Protocol_Exception, e.getMessage());
         } catch (IOException exception) {
-            exception.printStackTrace();
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_IOException, exception.getMessage());
         } finally {
             if (httpEntity != null) {
@@ -88,30 +95,27 @@ public class HttpClientPoolUtil {
         return responseBody;
     }
 
-
     /*** execute POST request ***/
     public static String execute(String uri, String data, String token) throws SQLException {
+
+        HttpEntityEnclosingRequestBase method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME);
+        method.setHeader(HTTP.CONTENT_TYPE, "text/plain");
+        method.setHeader(HTTP.CONN_DIRECTIVE, isKeepAlive);
+        method.setHeader("Authorization", "Taosd " + token);
+        method.setEntity(new StringEntity(data, StandardCharsets.UTF_8));
+        HttpContext context = HttpClientContext.create();
+
         HttpEntity httpEntity = null;
         String responseBody = "";
-        try {
-            HttpEntityEnclosingRequestBase method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME);
-            method.setHeader(HTTP.CONTENT_TYPE, "text/plain");
-            method.setHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_KEEP_ALIVE);
-            method.setHeader("Authorization", "Taosd " + token);
-
-            method.setEntity(new StringEntity(data, StandardCharsets.UTF_8));
-            HttpContext context = HttpClientContext.create();
-            CloseableHttpResponse httpResponse = httpClient.execute(method, context);
+        try (CloseableHttpResponse httpResponse = httpClient.execute(method, context)) {
             httpEntity = httpResponse.getEntity();
             if (httpEntity == null) {
                 throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_HTTP_ENTITY_IS_NULL, "httpEntity is null, sql: " + data);
             }
             responseBody = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8);
         } catch (ClientProtocolException e) {
-            e.printStackTrace();
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_Protocol_Exception, e.getMessage());
         } catch (IOException exception) {
-            exception.printStackTrace();
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_IOException, exception.getMessage());
         } finally {
             if (httpEntity != null) {
@@ -142,4 +146,12 @@ public class HttpClientPoolUtil {
         return method;
     }
 
+
+    public static void reset() {
+        synchronized (HttpClientPoolUtil.class) {
+            ClientConnectionManager cm = httpClient.getConnectionManager();
+            cm.closeExpiredConnections();
+            cm.closeIdleConnections(100, TimeUnit.MILLISECONDS);
+        }
+    }
 }
\ No newline at end of file

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java

@@ -16,7 +16,6 @@ public class TaosInfo implements TaosInfoMBean {
             MBeanServer server = ManagementFactory.getPlatformMBeanServer();
             ObjectName name = new ObjectName("TaosInfoMBean:name=TaosInfo");
             server.registerMBean(TaosInfo.getInstance(), name);
-
         } catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException e) {
             e.printStackTrace();
         }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java

@@ -49,14 +49,9 @@ public class Utils {
             try {
                 return parseMicroSecTimestamp(timeStampStr);
             } catch (DateTimeParseException ee) {
-                try {
-                    return parseNanoSecTimestamp(timeStampStr);
-                } catch (DateTimeParseException eee) {
-                    eee.printStackTrace();
-                }
+                return parseNanoSecTimestamp(timeStampStr);
             }
         }
-        return null;
     }
 
     private static LocalDateTime parseMilliSecTimestamp(String timeStampStr) throws DateTimeParseException {

src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/HttpKeepAliveTest.java

+package com.taosdata.jdbc.rs;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class HttpKeepAliveTest {
+
+    private static final String host = "127.0.0.1";
+
+    @Test
+    public void test() throws SQLException {
+        //given
+        int multi = 4000;
+        AtomicInteger exceptionCount = new AtomicInteger();
+
+        //when
+        Properties props = new Properties();
+        props.setProperty("httpKeepAlive", "false");
+        props.setProperty("httpPoolSize", "20");
+        Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata", props);
+
+        List<Thread> threads = IntStream.range(0, multi).mapToObj(i -> new Thread(
+                () -> {
+                    try (Statement stmt = connection.createStatement()) {
+                        stmt.execute("insert into log.tb_not_exists values(now, 1)");
+                        stmt.execute("select last(*) from log.dn");
+                    } catch (SQLException throwables) {
+                        exceptionCount.getAndIncrement();
+                    }
+                }
+        )).collect(Collectors.toList());
+
+        threads.forEach(Thread::start);
+
+        for (Thread thread : threads) {
+            try {
+                thread.join();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+
+        //then
+        Assert.assertEquals(multi, exceptionCount.get());
+    }
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/WasNullTest.java

@@ -6,8 +6,7 @@ import java.sql.*;
 
 public class WasNullTest {
 
-    //    private static final String host = "127.0.0.1";
-    private static final String host = "master";
+    private static final String host = "127.0.0.1";
     private Connection conn;
 
 

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/HttpClientPoolUtilTest.java

@@ -2,7 +2,6 @@ package com.taosdata.jdbc.utils;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
-import com.taosdata.jdbc.TSDBError;
 import org.junit.Test;
 
 import java.io.UnsupportedEncodingException;
@@ -18,13 +17,21 @@ public class HttpClientPoolUtilTest {
     String user = "root";
     String password = "taosdata";
     String host = "127.0.0.1";
-    String dbname = "log";
+//    String host = "master";
 
     @Test
-    public void test() {
+    public void useLog() {
         // given
-        List<Thread> threads = IntStream.range(0, 4000).mapToObj(i -> new Thread(() -> {
-            useDB();
+        int multi = 10;
+
+        // when
+        List<Thread> threads = IntStream.range(0, multi).mapToObj(i -> new Thread(() -> {
+            try {
+                String token = login(multi);
+                executeOneSql("use log", token);
+            } catch (SQLException | UnsupportedEncodingException e) {
+                e.printStackTrace();
+            }
         })).collect(Collectors.toList());
 
         threads.forEach(Thread::start);
@@ -38,31 +45,60 @@ public class HttpClientPoolUtilTest {
         }
     }
 
-    private void useDB() {
-        try {
-            user = URLEncoder.encode(user, StandardCharsets.UTF_8.displayName());
-            password = URLEncoder.encode(password, StandardCharsets.UTF_8.displayName());
-            String loginUrl = "http://" + host + ":" + 6041 + "/rest/login/" + user + "/" + password + "";
-            String result = HttpClientPoolUtil.execute(loginUrl);
-            JSONObject jsonResult = JSON.parseObject(result);
-            String status = jsonResult.getString("status");
-            String token = jsonResult.getString("desc");
-            if (!status.equals("succ")) {
-                throw new SQLException(jsonResult.getString("desc"));
+    @Test
+    public void tableNotExist() {
+        // given
+        int multi = 20;
+
+        // when
+        List<Thread> threads = IntStream.range(0, multi * 25).mapToObj(i -> new Thread(() -> {
+            try {
+//                String token = "/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04";
+                String token = login(multi);
+                executeOneSql("insert into log.tb_not_exist values(now, 1)", token);
+                executeOneSql("select last(*) from log.dn", token);
+            } catch (SQLException | UnsupportedEncodingException e) {
+                e.printStackTrace();
             }
+        })).collect(Collectors.toList());
 
-            String url = "http://" + host + ":6041/rest/sql";
-            String sql = "use " + dbname;
-            result = HttpClientPoolUtil.execute(url, sql, token);
+        threads.forEach(Thread::start);
 
-            JSONObject resultJson = JSON.parseObject(result);
-            if (resultJson.getString("status").equals("error")) {
-                throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc"));
+        for (Thread thread : threads) {
+            try {
+                thread.join();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
             }
-        } catch (UnsupportedEncodingException | SQLException e) {
-            e.printStackTrace();
         }
     }
 
+    private String login(int connPoolSize) throws SQLException, UnsupportedEncodingException {
+        user = URLEncoder.encode(user, StandardCharsets.UTF_8.displayName());
+        password = URLEncoder.encode(password, StandardCharsets.UTF_8.displayName());
+        String loginUrl = "http://" + host + ":" + 6041 + "/rest/login/" + user + "/" + password + "";
+        HttpClientPoolUtil.init(connPoolSize, false);
+        String result = HttpClientPoolUtil.execute(loginUrl);
+        JSONObject jsonResult = JSON.parseObject(result);
+        String status = jsonResult.getString("status");
+        String token = jsonResult.getString("desc");
+        if (!status.equals("succ")) {
+            throw new SQLException(jsonResult.getString("desc"));
+        }
+        return token;
+    }
+
+    private boolean executeOneSql(String sql, String token) throws SQLException {
+        String url = "http://" + host + ":6041/rest/sql";
+        String result = HttpClientPoolUtil.execute(url, sql, token);
+        JSONObject resultJson = JSON.parseObject(result);
+        if (resultJson.getString("status").equals("error")) {
+//            HttpClientPoolUtil.reset();
+//            throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc"));
+            return false;
+        }
+        return true;
+    }
+
 
 }
\ No newline at end of file

src/connector/jdbc/src/test/resources/commons-logging.properties

+#org.apache.commons.logging.Log=org.apache.commons.logging.impl.SimpleLog
+

src/connector/jdbc/src/test/resources/simplelog.properties

+org.apache.commons.logging.simplelog.defaultlog=TRACE
+org.apache.commons.logging.simplelog.showlogname=true
+org.apache.commons.logging.simplelog.showShortLogname=restful
+org.apache.commons.logging.simplelog.showdatetime=true
+org.apache.commons.logging.simplelog.dateTimeFormat=yyyy-mm-dd hh:MM:ss.SSS
\ No newline at end of file

tests/examples/JDBC/connectionPools/pom.xml

@@ -18,7 +18,7 @@
         <dependency>
             <groupId>com.taosdata.jdbc</groupId>
             <artifactId>taos-jdbcdriver</artifactId>
-            <version>2.0.18</version>
+            <version>2.0.34</version>
         </dependency>
         <!-- druid -->
         <dependency>