hotfix/TS-498: change taosd keepAlive to 30s (#8521)

* [TS-498]<test>: add a test case for HttpClientPoolUtil

* [TS-498]<fix>: fixed 'no sql input' bug

* throw exception when httpEntity is null

* [TS-498]<fix>: change taosd keepAlive to 30s
Co-authored-by: Nxywang <xywang@taosdata.com>

src/common/inc/tglobal.h

@@ -62,10 +62,11 @@ extern int32_t  tsCompressMsgSize;
 extern int32_t  tsMaxNumOfDistinctResults;
 extern char     tsTempDir[];
 
-//query buffer management
+// query buffer management
 extern int32_t tsQueryBufferSize;  // maximum allowed usage buffer size in MB for each data node during query processing
-extern int64_t  tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing
-extern int32_t  tsRetrieveBlockingModel;// retrieve threads will be blocked
+extern int64_t
+    tsQueryBufferSizeBytes;  // maximum allowed usage buffer size in byte for each data node during query processing
+extern int32_t tsRetrieveBlockingModel;  // retrieve threads will be blocked
 
 extern int8_t tsKeepOriginalColumnName;
 
@@ -126,6 +127,7 @@ extern int8_t   tsHttpEnableCompress;
 extern int8_t   tsHttpEnableRecordSql;
 extern int8_t   tsTelegrafUseFieldNum;
 extern int8_t   tsHttpDbNameMandatory;
+extern int32_t  tsHttpKeepAlive;
 
 // mqtt
 extern int8_t tsEnableMqttModule;

src/common/src/tglobal.c

@@ -14,18 +14,18 @@
  */
 
 #define _DEFAULT_SOURCE
+#include "tglobal.h"
+#include "monitor.h"
 #include "os.h"
 #include "taosdef.h"
 #include "taoserror.h"
-#include "tulog.h"
+#include "tcompare.h"
 #include "tconfig.h"
-#include "tglobal.h"
-#include "monitor.h"
-#include "tsocket.h"
-#include "tutil.h"
 #include "tlocale.h"
+#include "tsocket.h"
 #include "ttimezone.h"
-#include "tcompare.h"
+#include "tulog.h"
+#include "tutil.h"
 
 // TSDB
 bool tsdbForceKeepFile = false;
@@ -53,7 +53,7 @@ int32_t  tsDnodeId = 0;
 // common
 int32_t tsRpcTimer = 300;
 int32_t tsRpcMaxTime = 600;  // seconds;
-int32_t tsRpcForceTcp    = 0;  //disable this, means query, show command use udp protocol as default
+int32_t tsRpcForceTcp = 0;   // disable this, means query, show command use udp protocol as default
 int32_t tsMaxShellConns = 50000;
 int32_t tsMaxConnections = 5000;
 int32_t tsShellActivityTimer = 3;  // second
@@ -102,7 +102,7 @@ int32_t tsMaxStreamComputDelay = 20000;
 int32_t tsStreamCompStartDelay = 10000;
 
 // the stream computing delay time after executing failed, change accordingly
-int32_t tsRetryStreamCompDelay = 10*1000;
+int32_t tsRetryStreamCompDelay = 10 * 1000;
 
 // The delayed computing ration. 10% of the whole computing time window by default.
 float tsStreamComputDelayRatio = 0.1f;
@@ -166,6 +166,7 @@ int8_t   tsHttpEnableCompress = 1;
 int8_t   tsHttpEnableRecordSql = 0;
 int8_t   tsTelegrafUseFieldNum = 0;
 int8_t   tsHttpDbNameMandatory = 0;
+int32_t  tsHttpKeepAlive = 30000;
 
 // mqtt
 int8_t tsEnableMqttModule = 0;  // not finished yet, not started it by default
@@ -260,7 +261,8 @@ int8_t tsClientMerge = 0;
 //
 // lossy compress 6
 //
-char lossyColumns[32] = "";  // "float|double" means all float and double columns can be lossy compressed.  set empty can close lossy compress.
+char lossyColumns[32] = "";  // "float|double" means all float and double columns can be lossy compressed.  set empty
+                             // can close lossy compress.
 // below option can take effect when tsLossyColumns not empty
 double   fPrecision = 1E-8;                   // float column precision
 double   dPrecision = 1E-16;                  // double column precision
@@ -306,7 +308,8 @@ bool taosCfgDynamicOptions(char *msg) {
   int32_t vint = 0;
 
   paGetToken(msg, &option, &olen);
-  if (olen == 0) return false;;
+  if (olen == 0) return false;
+  ;
 
   paGetToken(option + olen + 1, &value, &vlen);
   if (vlen == 0)
@@ -319,7 +322,7 @@ bool taosCfgDynamicOptions(char *msg) {
 
   for (int32_t i = 0; i < tsGlobalConfigNum; ++i) {
     SGlobalCfg *cfg = tsGlobalConfig + i;
-    //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue;
+    // if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue;
     if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue;
 
     int32_t cfgLen = (int32_t)strlen(cfg->option);
@@ -407,7 +410,7 @@ static void taosCheckDataDirCfg() {
 }
 
 static int32_t taosCheckTmpDir(void) {
-  if (strlen(tsTempDir) <= 0){
+  if (strlen(tsTempDir) <= 0) {
     uError("tempDir is not set");
     return -1;
   }
@@ -557,8 +560,8 @@ static void doInitGlobalConfig(void) {
   cfg.ptr = &tsMaxNumOfDistinctResults;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 10*10000;
-  cfg.maxValue = 10000*10000;
+  cfg.minValue = 10 * 10000;
+  cfg.maxValue = 10000 * 10000;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
@@ -1280,6 +1283,17 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  // pContext in cache
+  cfg.option = "httpKeepAlive";
+  cfg.ptr = &tsHttpKeepAlive;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = 3000;
+  cfg.maxValue = 3600000;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   // debug flag
   cfg.option = "numOfLogLines";
   cfg.ptr = &tsNumOfLogLines;
@@ -1361,7 +1375,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-
   cfg.option = "sdbDebugFlag";
   cfg.ptr = &sdbDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1648,12 +1661,9 @@ static void doInitGlobalConfig(void) {
 #else
   assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5);
 #endif
-
 }
 
-void taosInitGlobalCfg() {
-  pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig);
-}
+void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); }
 
 int32_t taosCheckGlobalCfg() {
   char     fqdn[TSDB_FQDN_LEN];
@@ -1711,8 +1721,8 @@ int32_t taosCheckGlobalCfg() {
   }
 
   if (tsMaxTablePerVnode < tsMinTablePerVnode) {
-    uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)",
-	   tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode);
+    uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", tsMaxTablePerVnode,
+           tsMinTablePerVnode, tsMinTablePerVnode);
     tsMaxTablePerVnode = tsMinTablePerVnode;
   }
 
@@ -1756,7 +1766,7 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) {
   char *temp = strchr(fqdn, ':');
   if (temp) {
     *temp = 0;
-    *port = atoi(temp+1);
+    *port = atoi(temp + 1);
   }
 
   if (*port == 0) {

src/connector/jdbc/pom.xml

@@ -112,6 +112,7 @@
                         <include>**/*Test.java</include>
                     </includes>
                     <excludes>
+                        <exclude>**/HttpClientPoolUtilTest.java</exclude>
                         <exclude>**/AppMemoryLeakTest.java</exclude>
                         <exclude>**/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java</exclude>
                         <exclude>**/DatetimeBefore1970Test.java</exclude>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java

@@ -31,6 +31,8 @@ public class TSDBErrorNumbers {
     public static final int ERROR_RESTFul_Client_IOException = 0x2318;
     public static final int ERROR_USER_IS_REQUIRED = 0x2319;        // user is required
     public static final int ERROR_PASSWORD_IS_REQUIRED = 0x231a;    // password is required
+    public static final int ERROR_INVALID_JSON_FORMAT = 0x231b;
+    public static final int ERROR_HTTP_ENTITY_IS_NULL = 0x231c; //http entity is null
 
 
     public static final int ERROR_UNKNOWN = 0x2350;    //unknown error
@@ -72,6 +74,8 @@ public class TSDBErrorNumbers {
         errorNumbers.add(ERROR_RESTFul_Client_IOException);
         errorNumbers.add(ERROR_USER_IS_REQUIRED);
         errorNumbers.add(ERROR_PASSWORD_IS_REQUIRED);
+        errorNumbers.add(ERROR_INVALID_JSON_FORMAT);
+        errorNumbers.add(ERROR_HTTP_ENTITY_IS_NULL);
 
         errorNumbers.add(ERROR_RESTFul_Client_Protocol_Exception);
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java

@@ -5,6 +5,7 @@ import com.taosdata.jdbc.TSDBErrorNumbers;
 import org.apache.http.HeaderElement;
 import org.apache.http.HeaderElementIterator;
 import org.apache.http.HttpEntity;
+import org.apache.http.NoHttpResponseException;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.client.config.RequestConfig;
@@ -30,12 +31,12 @@ import java.sql.SQLException;
 public class HttpClientPoolUtil {
 
     private static final String DEFAULT_CONTENT_TYPE = "application/json";
-    private static final int DEFAULT_MAX_TOTAL = 200;
-    private static final int DEFAULT_MAX_PER_ROUTE = 20;
-    private static final int DEFAULT_TIME_OUT = 15000;
-    private static final int DEFAULT_HTTP_KEEP_TIME = 15000;
     private static final int DEFAULT_MAX_RETRY_COUNT = 5;
 
+    private static final int DEFAULT_MAX_TOTAL = 50;
+    private static final int DEFAULT_MAX_PER_ROUTE = 5;
+    private static final int DEFAULT_HTTP_KEEP_TIME = -1;
+
     private static final ConnectionKeepAliveStrategy DEFAULT_KEEP_ALIVE_STRATEGY = (response, context) -> {
         HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
         while (it.hasNext()) {
@@ -52,29 +53,19 @@ public class HttpClientPoolUtil {
         return DEFAULT_HTTP_KEEP_TIME * 1000;
     };
 
-    private static final HttpRequestRetryHandler retryHandler = (exception, executionCount, httpContext) -> {
-        if (executionCount >= DEFAULT_MAX_RETRY_COUNT)
-            // do not retry if over max retry count
-            return false;
-        if (exception instanceof InterruptedIOException)
-            // timeout
-            return false;
-        if (exception instanceof UnknownHostException)
-            // unknown host
-            return false;
-        if (exception instanceof SSLException)
-            // SSL handshake exception
-            return false;
-        return true;
-    };
-
     private static CloseableHttpClient httpClient;
 
     static {
+
         PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
         connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL);
         connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE);
-        httpClient = HttpClients.custom().setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY).setConnectionManager(connectionManager).setRetryHandler(retryHandler).build();
+
+        httpClient = HttpClients.custom()
+                .setKeepAliveStrategy(DEFAULT_KEEP_ALIVE_STRATEGY)
+                .setConnectionManager(connectionManager)
+                .setRetryHandler((exception, executionCount, httpContext) -> executionCount < DEFAULT_MAX_RETRY_COUNT)
+                .build();
     }
 
     /*** execute GET request ***/
@@ -118,9 +109,10 @@ public class HttpClientPoolUtil {
             HttpContext context = HttpClientContext.create();
             CloseableHttpResponse httpResponse = httpClient.execute(method, context);
             httpEntity = httpResponse.getEntity();
-            if (httpEntity != null) {
-                responseBody = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8);
+            if (httpEntity == null) {
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_HTTP_ENTITY_IS_NULL, "httpEntity is null, sql: " + data);
             }
+            responseBody = EntityUtils.toString(httpEntity, StandardCharsets.UTF_8);
         } catch (ClientProtocolException e) {
             e.printStackTrace();
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESTFul_Client_Protocol_Exception, e.getMessage());
@@ -139,9 +131,6 @@ public class HttpClientPoolUtil {
     private static HttpRequestBase getRequest(String uri, String methodName) {
         HttpRequestBase method;
         RequestConfig requestConfig = RequestConfig.custom()
-                .setSocketTimeout(DEFAULT_TIME_OUT * 1000)
-                .setConnectTimeout(DEFAULT_TIME_OUT * 1000)
-                .setConnectionRequestTimeout(DEFAULT_TIME_OUT * 1000)
                 .setExpectContinueEnabled(false)
                 .build();
         if (HttpPut.METHOD_NAME.equalsIgnoreCase(methodName)) {

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/HttpClientPoolUtilTest.java

+package com.taosdata.jdbc.utils;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.taosdata.jdbc.TSDBDriver;
+import com.taosdata.jdbc.TSDBError;
+import org.junit.Test;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class HttpClientPoolUtilTest {
+
+    String user = "root";
+    String password = "taosdata";
+    String host = "127.0.0.1";
+    String dbname = "log";
+
+    @Test
+    public void test() {
+        // given
+        List<Thread> threads = IntStream.range(0, 4000).mapToObj(i -> new Thread(() -> {
+            useDB();
+//            try {
+//                TimeUnit.SECONDS.sleep(10);
+//            } catch (InterruptedException e) {
+//                e.printStackTrace();
+//            }
+        })).collect(Collectors.toList());
+
+        threads.forEach(Thread::start);
+
+        for (Thread thread : threads) {
+            try {
+                thread.join();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+
+    }
+
+    private void useDB() {
+        try {
+            user = URLEncoder.encode(user, StandardCharsets.UTF_8.displayName());
+            password = URLEncoder.encode(password, StandardCharsets.UTF_8.displayName());
+            String loginUrl = "http://" + host + ":" + 6041 + "/rest/login/" + user + "/" + password + "";
+            String result = HttpClientPoolUtil.execute(loginUrl);
+            JSONObject jsonResult = JSON.parseObject(result);
+            String status = jsonResult.getString("status");
+            String token = jsonResult.getString("desc");
+            if (!status.equals("succ")) {
+                throw new SQLException(jsonResult.getString("desc"));
+            }
+
+            String url = "http://" + host + ":6041/rest/sql";
+            String sql = "use " + dbname;
+            result = HttpClientPoolUtil.execute(url, sql, token);
+
+            JSONObject resultJson = JSON.parseObject(result);
+            if (resultJson.getString("status").equals("error")) {
+                throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc"));
+            }
+        } catch (UnsupportedEncodingException | SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+
+}
\ No newline at end of file

src/plugins/http/inc/httpInt.h

@@ -154,12 +154,13 @@ typedef struct HttpContext {
   char               db[/*TSDB_ACCT_ID_LEN + */TSDB_DB_NAME_LEN];
   TAOS *             taos;
   void *             ppContext;
+  pthread_mutex_t    ctxMutex;
   HttpSession       *session;
   z_stream           gzipStream;
   HttpParser        *parser;
   HttpSqlCmd         singleCmd;
   HttpSqlCmds       *multiCmds;
-  JsonBuf *    jsonBuf;
+  JsonBuf           *jsonBuf;
   HttpEncodeMethod  *encodeMethod;
   HttpDecodeMethod  *decodeMethod;
   struct HttpThread *pThread;

src/plugins/http/src/httpContext.c

@@ -67,6 +67,8 @@ static void httpDestroyContext(void *data) {
     pContext->parser = NULL;
   }
 
+  pthread_mutex_destroy(&pContext->ctxMutex);
+
   tfree(pContext);
 }
 
@@ -121,13 +123,15 @@ HttpContext *httpCreateContext(SOCKET fd) {
 
   TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pContext;
   HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &pContext,
-                                         sizeof(TSDB_CACHE_PTR_TYPE), 3000);
+                                         sizeof(TSDB_CACHE_PTR_TYPE), tsHttpKeepAlive);
   pContext->ppContext = ppContext;
   httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext);
 
   // set the ref to 0
   taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false);
 
+  pthread_mutex_init(&pContext->ctxMutex, NULL);
+
   return pContext;
 }
 

src/plugins/http/src/httpHandle.c

@@ -45,15 +45,14 @@ bool httpProcessData(HttpContext* pContext) {
     httpTrace("context:%p, fd:%d, process options request", pContext, pContext->fd);
     httpSendOptionResp(pContext, "process options request success");
   } else {
-    if (!httpDecodeRequest(pContext)) {
-      /*
-       * httpCloseContextByApp has been called when parsing the error
-       */
-      // httpCloseContextByApp(pContext);
-    } else {
+    pthread_mutex_lock(&pContext->ctxMutex);
+
+    if (httpDecodeRequest(pContext)) {
       httpClearParser(pContext->parser);
       httpProcessRequest(pContext);
     }
+
+    pthread_mutex_unlock(&pContext->ctxMutex);
   }
 
   return true;

src/plugins/http/src/httpSql.c

@@ -406,7 +406,14 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) {
   if (pContext->session == NULL) {
     httpSendErrorResp(pContext, TSDB_CODE_HTTP_SESSION_FULL);
   } else {
+    // httpProcessRequestCb called by another thread and a subsequent thread calls this
+    // function again, if this function called by httpProcessRequestCb executes memset
+    // just before the subsequent thread executes *Cmd function, nativSql will be NULL
+    pthread_mutex_lock(&pContext->ctxMutex);
+
     httpExecCmd(pContext);
+
+    pthread_mutex_unlock(&pContext->ctxMutex);
   }
 }
 

src/util/inc/tconfig.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#define TSDB_CFG_MAX_NUM    120  
+#define TSDB_CFG_MAX_NUM    121
 #define TSDB_CFG_PRINT_LEN  23
 #define TSDB_CFG_OPTION_LEN 24
 #define TSDB_CFG_VALUE_LEN  41