feat: refactor rpc code

include/libs/qcom/query.h

@@ -16,6 +16,7 @@
 #ifndef _TD_QUERY_H_
 #define _TD_QUERY_H_
 
+// clang-foramt off
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -71,7 +72,7 @@ typedef struct SIndexMeta {
 } SIndexMeta;
 
 typedef struct STbVerInfo {
-  char tbFName[TSDB_TABLE_FNAME_LEN];
+  char    tbFName[TSDB_TABLE_FNAME_LEN];
   int32_t sversion;
   int32_t tversion;
 } STbVerInfo;
@@ -141,7 +142,7 @@ typedef struct SDataBuf {
 
 typedef struct STargetInfo {
   ETargetType type;
-  char*       dbFName; // used to update db's vgroup epset
+  char*       dbFName;  // used to update db's vgroup epset
   int32_t     vgId;
 } STargetInfo;
 
@@ -149,15 +150,15 @@ typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32
 typedef int32_t (*__async_exec_fn_t)(void* param);
 
 typedef struct SRequestConnInfo {
-  void*     pTrans;
-  uint64_t  requestId;
-  int64_t   requestObjRefId;
-  SEpSet    mgmtEps;
+  void*    pTrans;
+  uint64_t requestId;
+  int64_t  requestObjRefId;
+  SEpSet   mgmtEps;
 } SRequestConnInfo;
 
 typedef struct SMsgSendInfo {
-  __async_send_cb_fn_t fp;  // async callback function
-  STargetInfo          target; // for update epset
+  __async_send_cb_fn_t fp;      // async callback function
+  STargetInfo          target;  // for update epset
   void*                param;
   uint64_t             requestId;
   uint64_t             requestObjRefId;
@@ -206,9 +207,10 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl
 char*   jobTaskStatusStr(int32_t status);
 
 SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
-void destroyQueryExecRes(SQueryExecRes* pRes);
+void    destroyQueryExecRes(SQueryExecRes* pRes);
 
-extern int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallocFp)(int32_t));
+extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen,
+                                          void* (*mallocFp)(int32_t));
 extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
 
 #define SET_META_TYPE_NULL(t)       (t) = META_TYPE_NULL_TABLE
@@ -219,7 +221,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define NEED_CLIENT_RM_TBLMETA_ERROR(_code)                                                   \
   ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST ||       \
    (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \
-   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
+   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG ||    \
    (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
 #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
   ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
@@ -227,11 +229,13 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define NEED_CLIENT_HANDLE_ERROR(_code)                                          \
   (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
    NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
-#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \
-  || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB)
+#define NEED_CLIENT_RM_TBLMETA_REQ(_type)                                                                  \
+  ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
+   (_type) == TDMT_VND_DROP_STB)
 
-#define NEED_SCHEDULER_RETRY_ERROR(_code) \
-  ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
+#define NEED_SCHEDULER_RETRY_ERROR(_code)                                           \
+  ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
+   (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
 
 #define REQUEST_TOTAL_EXEC_TIMES 2
 
@@ -308,3 +312,4 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #endif
 
 #endif /*_TD_QUERY_H_*/
+       // clang-foramt on

include/os/osSocket.h

@@ -157,7 +157,10 @@ int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
 int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
 void    taosWinSocketInit();
 
-int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
+/*
+ * set timeout(ms)
+ */
+int32_t taosCreateSocketWithTimeout(uint32_t timeout);
 
 TdSocketPtr       taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
 TdSocketPtr       taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);

source/client/src/clientImpl.c

@@ -59,7 +59,7 @@ static STscObj* taosConnectImpl(const char* user, const char* auth, const char*
                                 SAppInstInfo* pAppInfo, int connType);
 
 STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
-                            uint16_t port, int connType) {
+                               uint16_t port, int connType) {
   if (taos_init() != TSDB_CODE_SUCCESS) {
     return NULL;
   }
@@ -313,8 +313,8 @@ bool qnodeRequired(SRequestObj* pRequest) {
   }
 
   SAppInstInfo* pInfo = pRequest->pTscObj->pAppInfo;
-  bool required = false;
-  
+  bool          required = false;
+
   taosThreadMutexLock(&pInfo->qnodeMutex);
   required = (NULL == pInfo->pQnodeList);
   taosThreadMutexUnlock(&pInfo->qnodeMutex);
@@ -419,11 +419,11 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr
     }
 
     for (int32_t j = 0; j < vgNum; ++j) {
-      SVgroupInfo* pInfo = taosArrayGet(pVg, j);
+      SVgroupInfo*   pInfo = taosArrayGet(pVg, j);
       SQueryNodeLoad load = {0};
       load.addr.nodeId = pInfo->vgId;
       load.addr.epSet = pInfo->epSet;
-      
+
       taosArrayPush(nodeList, &load);
     }
   }
@@ -481,17 +481,16 @@ _return:
   return TSDB_CODE_SUCCESS;
 }
 
-
-int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData *pResultMeta) {
+int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData* pResultMeta) {
   SArray* pDbVgList = NULL;
   SArray* pQnodeList = NULL;
   int32_t code = 0;
-  
+
   switch (tsQueryPolicy) {
     case QUERY_POLICY_VNODE: {
       if (pResultMeta) {
         pDbVgList = taosArrayInit(4, POINTER_BYTES);
-        
+
         int32_t dbNum = taosArrayGetSize(pResultMeta->pDbVgroup);
         for (int32_t i = 0; i < dbNum; ++i) {
           SMetaRes* pRes = taosArrayGet(pResultMeta->pDbVgroup, i);
@@ -500,9 +499,9 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray
           }
 
           taosArrayPush(pDbVgList, &pRes->pRes);
-        } 
+        }
       }
-    
+
       code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList);
       break;
     }
@@ -523,7 +522,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray
         }
         taosThreadMutexUnlock(&pInst->qnodeMutex);
       }
-      
+
       code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList);
       break;
     }
@@ -534,7 +533,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray
 
   taosArrayDestroy(pDbVgList);
   taosArrayDestroy(pQnodeList);
-  
+
   return code;
 }
 
@@ -542,43 +541,43 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray*
   SArray* pDbVgList = NULL;
   SArray* pQnodeList = NULL;
   int32_t code = 0;
-  
+
   switch (tsQueryPolicy) {
     case QUERY_POLICY_VNODE: {
       int32_t dbNum = taosArrayGetSize(pRequest->dbList);
       if (dbNum > 0) {
-        SCatalog* pCtg = NULL;
+        SCatalog*     pCtg = NULL;
         SAppInstInfo* pInst = pRequest->pTscObj->pAppInfo;
         code = catalogGetHandle(pInst->clusterId, &pCtg);
         if (code != TSDB_CODE_SUCCESS) {
           goto _return;
         }
 
-        pDbVgList = taosArrayInit(dbNum, POINTER_BYTES);   
+        pDbVgList = taosArrayInit(dbNum, POINTER_BYTES);
         SArray* pVgList = NULL;
         for (int32_t i = 0; i < dbNum; ++i) {
-          char* dbFName = taosArrayGet(pRequest->dbList, i);
+          char*            dbFName = taosArrayGet(pRequest->dbList, i);
           SRequestConnInfo conn = {.pTrans = pInst->pTransporter,
                                    .requestId = pRequest->requestId,
                                    .requestObjRefId = pRequest->self,
-                                   .mgmtEps = getEpSet_s(&pInst->mgmtEp)};    
-                                   
+                                   .mgmtEps = getEpSet_s(&pInst->mgmtEp)};
+
           code = catalogGetDBVgInfo(pCtg, &conn, dbFName, &pVgList);
           if (code) {
             goto _return;
           }
-        
+
           taosArrayPush(pDbVgList, &pVgList);
-        } 
+        }
       }
-    
+
       code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList);
       break;
     }
     case QUERY_POLICY_HYBRID:
     case QUERY_POLICY_QNODE: {
       getQnodeList(pRequest, &pQnodeList);
-      
+
       code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList);
       break;
     }
@@ -591,11 +590,10 @@ _return:
 
   taosArrayDestroy(pDbVgList);
   taosArrayDestroy(pQnodeList);
-  
+
   return code;
 }
 
-
 int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
   tsem_init(&schdRspSem, 0, 0);
 
@@ -604,12 +602,12 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod
                            .requestId = pRequest->requestId,
                            .requestObjRefId = pRequest->self};
   SSchedulerReq    req = {.pConn = &conn,
-                          .pNodeList = pNodeList,
-                          .pDag = pDag,
-                          .sql = pRequest->sqlstr,
-                          .startTs = pRequest->metric.start,
-                          .fp = schdExecCallback,
-                          .cbParam = &res};
+                       .pNodeList = pNodeList,
+                       .pDag = pDag,
+                       .sql = pRequest->sqlstr,
+                       .startTs = pRequest->metric.start,
+                       .fp = schdExecCallback,
+                       .cbParam = &res};
 
   int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
 
@@ -656,13 +654,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
                            .requestId = pRequest->requestId,
                            .requestObjRefId = pRequest->self};
   SSchedulerReq    req = {.pConn = &conn,
-                          .pNodeList = pNodeList,
-                          .pDag = pDag,
-                          .sql = pRequest->sqlstr,
-                          .startTs = pRequest->metric.start,
-                          .fp = NULL,
-                          .cbParam = NULL,
-                          .reqKilled = &pRequest->killed};
+                       .pNodeList = pNodeList,
+                       .pDag = pDag,
+                       .sql = pRequest->sqlstr,
+                       .startTs = pRequest->metric.start,
+                       .fp = NULL,
+                       .cbParam = NULL,
+                       .reqKilled = &pRequest->killed};
 
   int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res);
   pRequest->body.resInfo.execRes = res.res;
@@ -819,8 +817,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
     }
   }
 
-  tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64,
-             pRequest->self, code, tstrerror(code), pRequest->requestId);
+  tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
+           tstrerror(code), pRequest->requestId);
 
   STscObj* pTscObj = pRequest->pTscObj;
   if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) {
@@ -862,7 +860,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
       if (TSDB_CODE_SUCCESS == code) {
         SArray* pNodeList = NULL;
         buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList);
-        
+
         code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
         taosArrayDestroy(pNodeList);
       }
@@ -915,7 +913,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
   return launchQueryImpl(pRequest, pQuery, false, NULL);
 }
 
-void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta) {
+void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) {
   int32_t code = 0;
 
   switch (pQuery->execMode) {
@@ -948,7 +946,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM
       if (TSDB_CODE_SUCCESS == code) {
         SArray* pNodeList = NULL;
         buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
-        
+
         SRequestConnInfo conn = {
             .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
         SSchedulerReq req = {.pConn = &conn,
@@ -1308,7 +1306,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
   if (pObj) {
     return pObj->id;
   }
-  
+
   return NULL;
 }
 
@@ -1554,10 +1552,10 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows){
+static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
   char* p = (char*)pResultInfo->pData;
 
-  int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
+  int32_t  len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
   int32_t* colLength = (int32_t*)(p + len);
   len += sizeof(int32_t) * numOfCols;
 
@@ -1567,7 +1565,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
 
     if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
       int32_t* offset = (int32_t*)pStart;
-      int32_t lenTmp = numOfRows * sizeof(int32_t);
+      int32_t  lenTmp = numOfRows * sizeof(int32_t);
       len += lenTmp;
       pStart += lenTmp;
 
@@ -1592,7 +1590,6 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
         } else {
           ASSERT(0);
         }
-
       }
     } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
       int32_t lenTmp = numOfRows * sizeof(int32_t);
@@ -1616,13 +1613,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
       break;
     }
   }
-  if(!needConvert) return TSDB_CODE_SUCCESS;
+  if (!needConvert) return TSDB_CODE_SUCCESS;
 
-  char* p = (char*)pResultInfo->pData;
+  char*   p = (char*)pResultInfo->pData;
   int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows);
 
   pResultInfo->convertJson = taosMemoryCalloc(1, dataLen);
-  if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
   char* p1 = pResultInfo->convertJson;
 
   int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
@@ -1691,7 +1688,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
           ASSERT(0);
         }
 
-        offset1[j]= len;
+        offset1[j] = len;
         memcpy(pStart1 + len, dst, varDataTLen(dst));
         len += varDataTLen(dst);
       }
@@ -1709,7 +1706,6 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
       pStart += len;
       pStart1 += len;
       memcpy(pStart1, pStart, colLen);
-
     }
     pStart += colLen;
     pStart1 += colLen1;
@@ -1777,7 +1773,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
     pStart += colLength[i];
   }
 
-  if(convertUcs4){
+  if (convertUcs4) {
     code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength);
   }
 

source/libs/qcom/src/queryUtil.c

@@ -19,7 +19,7 @@
 #include "tmsg.h"
 #include "trpc.h"
 #include "tsched.h"
-
+// clang-format off
 #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
 #define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS)
 
@@ -146,13 +146,15 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra
   }
 
   memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len);
-  SRpcMsg rpcMsg = {.msgType = pInfo->msgType,
-                    .pCont = pMsg,
-                    .contLen = pInfo->msgInfo.len,
-                    .info.ahandle = (void*)pInfo,
-                    .info.handle = pInfo->msgInfo.handle,
-                    .info.persistHandle = persistHandle,
-                    .code = 0};
+  SRpcMsg rpcMsg = {
+    .msgType = pInfo->msgType,
+    .pCont = pMsg,
+    .contLen = pInfo->msgInfo.len,
+    .info.ahandle = (void*)pInfo,
+    .info.handle = pInfo->msgInfo.handle,
+    .info.persistHandle = persistHandle, 
+    .code = 0
+  };
   assert(pInfo->fp != NULL);
   TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId);
   rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx);
@@ -220,3 +222,4 @@ void destroyQueryExecRes(SQueryExecRes* pRes) {
       qError("invalid exec result for request type %d", pRes->msgType);
   }
 }
+// clang-format on

source/libs/transport/inc/transComm.h

@@ -238,6 +238,32 @@ int         transSendAsync(SAsyncPool* pool, queue* mq);
       }                                                       \
     }                                                         \
   } while (0)
+
+#define ASYNC_CHECK_HANDLE(exh1, refId)                                                                               \
+  do {                                                                                                                \
+    if (refId > 0) {                                                                                                  \
+      tTrace("handle step1");                                                                                         \
+      SExHandle* exh2 = transAcquireExHandle(refMgt, refId);                                                          \
+      if (exh2 == NULL || refId != exh2->refId) {                                                                     \
+        tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1,        \
+               exh2 ? exh2->refId : 0, refId);                                                                        \
+        goto _return1;                                                                                                \
+      }                                                                                                               \
+    } else if (refId == 0) {                                                                                          \
+      tTrace("handle step2");                                                                                         \
+      SExHandle* exh2 = transAcquireExHandle(refMgt, refId);                                                          \
+      if (exh2 == NULL || refId != exh2->refId) {                                                                     \
+        tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \
+               exh2 ? exh2->refId : 0);                                                                               \
+        goto _return1;                                                                                                \
+      } else {                                                                                                        \
+        refId = exh1->refId;                                                                                          \
+      }                                                                                                               \
+    } else if (refId < 0) {                                                                                           \
+      tTrace("handle step3");                                                                                         \
+      goto _return2;                                                                                                  \
+    }                                                                                                                 \
+  } while (0)
 int  transInitBuffer(SConnBuffer* buf);
 int  transClearBuffer(SConnBuffer* buf);
 int  transDestroyBuffer(SConnBuffer* buf);

source/libs/transport/src/transCli.c

@@ -25,7 +25,6 @@ typedef struct SCliConn {
   uv_write_t   writeReq;
 
   void* hostThrd;
-  int   hThrdIdx;
 
   SConnBuffer readBuf;
   STransQueue cliMsgs;
@@ -36,6 +35,7 @@ typedef struct SCliConn {
   bool       broken;  // link broken or not
   ConnStatus status;  //
 
+  int64_t  refId;
   char*    ip;
   uint32_t port;
 
@@ -168,16 +168,24 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
     snprintf(key, sizeof(key), "%s:%d", ip, (int)port); \
   } while (0)
 
-#define CONN_HOST_THREAD_IDX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1)
+#define CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd) \
+  do {                                                \
+    if (exh == NULL) {                                \
+      idx = -1;                                       \
+    } else {                                          \
+      ASYNC_CHECK_HANDLE(exh, refId);                 \
+      pThrd = (SCliThrdObj*)exh->pThrd;               \
+    }                                                 \
+  } while (0)
 #define CONN_PERSIST_TIME(para)    (para * 1000 * 10)
 #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
 #define CONN_GET_INST_LABEL(conn)  (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label)
 #define CONN_SHOULD_RELEASE(conn, head)                                                                            \
   do {                                                                                                             \
     if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) {                                                 \
+      int      status = conn->status;                                                                              \
       uint64_t ahandle = head->ahandle;                                                                            \
       CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle);                                                                   \
-      conn->status = ConnRelease;                                                                                  \
       transClearBuffer(&conn->readBuf);                                                                            \
       transFreeMsg(transContFromHead((char*)head));                                                                \
       tDebug("%s conn %p receive release request, ref: %d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \
@@ -186,7 +194,9 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
       }                                                                                                            \
       destroyCmsg(pMsg);                                                                                           \
       cliReleaseUnfinishedMsg(conn);                                                                               \
-      addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn);                                                   \
+      if (status != ConnInPool) {                                                                                  \
+        addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn);                                                 \
+      }                                                                                                            \
       return;                                                                                                      \
     }                                                                                                              \
   } while (0)
@@ -323,23 +333,29 @@ void cliHandleResp(SCliConn* conn) {
   transClearBuffer(&conn->readBuf);
 
   if (!CONN_NO_PERSIST_BY_APP(conn)) {
-    transMsg.info.handle = conn;
+    SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
+    exh->handle = conn;
+    exh->pThrd = pThrd;
+    exh->refId = transAddExHandle(refMgt, exh);
+
+    transMsg.info.handle = exh;
+    transMsg.info.refId = exh->refId;
+    conn->refId = exh->refId;
     tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn);
   }
-  // char buf[64] = {0};
-  // TRACE_TO_STR(&transMsg.info.traceId, buf);
+
   STraceId* trace = &transMsg.info.traceId;
-  tGTrace("conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", conn, TMSG_INFO(pHead->msgType),
-          taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->localAddr.sin_addr),
-          ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code);
+  tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", CONN_GET_INST_LABEL(conn),
+          conn, TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port),
+          taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code);
 
   if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) {
-    tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn));
+    tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn);
     // transUnrefCliHandle(conn);
     return;
   }
   if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) {
-    tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn));
+    tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn);
     // transUnrefCliHandle(conn);
     return;
   }
@@ -476,10 +492,9 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
   if (QUEUE_IS_EMPTY(&plist->conn)) {
     return NULL;
   }
-  queue* h = QUEUE_HEAD(&plist->conn);
-  // //QUEUE_REMOVE(h);
+  queue*    h = QUEUE_HEAD(&plist->conn);
   SCliConn* conn = QUEUE_DATA(h, SCliConn, conn);
-  // conn->status = ConnNormal;
+  conn->status = ConnNormal;
   QUEUE_REMOVE(&conn->conn);
   QUEUE_INIT(&conn->conn);
   return conn;
@@ -559,6 +574,13 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) {
   conn->status = ConnNormal;
   conn->broken = 0;
   transRefCliHandle(conn);
+
+  SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
+  exh->handle = conn;
+  exh->pThrd = pThrd;
+  exh->refId = transAddExHandle(refMgt, exh);
+  conn->refId = exh->refId;
+
   return conn;
 }
 static void cliDestroyConn(SCliConn* conn, bool clear) {
@@ -566,6 +588,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
 
   QUEUE_REMOVE(&conn->conn);
   QUEUE_INIT(&conn->conn);
+  transRemoveExHandle(refMgt, conn->refId);
   if (clear) {
     uv_close((uv_handle_t*)conn->stream, cliDestroy);
   }
@@ -650,12 +673,10 @@ void cliSend(SCliConn* pConn) {
 
   uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
 
-  // char buf[64] = {0};
-  // TRACE_TO_STR(&pMsg->info.traceId, buf);
   STraceId* trace = &pMsg->info.traceId;
-  tGTrace("conn %p %s is sent to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType),
-          taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr),
-          ntohs(pConn->localAddr.sin_port));
+  tGTrace("%s conn %p %s is sent to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn,
+          TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port),
+          taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port));
 
   if (pHead->persist == 1) {
     CONN_SET_PERSIST_BY_APP(pConn);
@@ -663,7 +684,6 @@ void cliSend(SCliConn* pConn) {
 
   pConn->writeReq.data = pConn;
   uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
-
   return;
 _RETURN:
   return;
@@ -723,20 +743,32 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) {
 }
 
 SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) {
-  SCliConn* conn = NULL;
-  if (pMsg->msg.info.handle != NULL) {
-    conn = (SCliConn*)(pMsg->msg.info.handle);
-    if (conn != NULL) {
-      tTrace("%s conn %p reused", CONN_GET_INST_LABEL(conn), conn);
+  SCliConn*       conn = NULL;
+  SRpcHandleInfo* pInfo = &pMsg->msg.info;
+
+  SExHandle* exh = transAcquireExHandle(refMgt, pInfo->refId);
+  if (exh == NULL) {
+    if (pInfo->refId != 0) {
+      tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn);
+      assert(0);
+      return NULL;
     }
   } else {
-    STransConnCtx* pCtx = pMsg->ctx;
-    conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet));
-    if (conn != NULL) {
-      tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn);
-    } else {
-      tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool);
-    }
+    transReleaseExHandle(refMgt, pInfo->refId);
+    return exh->handle;
+  }
+
+  STransConnCtx* pCtx = pMsg->ctx;
+  conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet));
+  if (conn != NULL) {
+    exh = taosMemoryCalloc(1, sizeof(SExHandle));
+    exh->handle = conn;
+    exh->pThrd = pThrd;
+    exh->refId = transAddExHandle(refMgt, exh);
+    conn->refId = exh->refId;
+    tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn);
+  } else {
+    tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool);
   }
   return conn;
 }
@@ -765,8 +797,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
 
   SCliConn* conn = cliGetConn(pMsg, pThrd);
   if (conn != NULL) {
-    conn->hThrdIdx = pCtx->hThrdIdx;
-
     transCtxMerge(&conn->ctx, &pCtx->appCtx);
     transQueuePush(&conn->cliMsgs, pMsg);
     cliSend(conn);
@@ -775,7 +805,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
     transCtxMerge(&conn->ctx, &pCtx->appCtx);
     transQueuePush(&conn->cliMsgs, pMsg);
 
-    conn->hThrdIdx = pCtx->hThrdIdx;
     conn->ip = strdup(EPSET_GET_INUSE_IP(&pCtx->epSet));
     conn->port = EPSET_GET_INUSE_PORT(&pCtx->epSet);
 
@@ -783,7 +812,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
     if (ret) {
       tError("%s conn %p failed to set conn option, errmsg %s", transLabel(pTransInst), conn, uv_err_name(ret));
     }
-    int fd = taosCreateSocketWithTimeOutOpt(TRANS_CONN_TIMEOUT);
+    int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT);
     if (fd == -1) {
       tTrace("%s conn %p failed to create socket", transLabel(pTransInst), conn);
       cliHandleExcept(conn);
@@ -1009,7 +1038,9 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
         tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
                pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT);
       }
-      addConnToPool(pThrd->pool, pConn);
+      if (pConn->status != ConnInPool) {
+        addConnToPool(pThrd->pool, pConn);
+      }
 
       STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
       arg->param1 = pMsg;
@@ -1086,10 +1117,21 @@ void transReleaseCliHandle(void* handle) {
 }
 
 void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) {
-  STrans* pTransInst = (STrans*)shandle;
-  int     idx = CONN_HOST_THREAD_IDX((SCliConn*)pReq->info.handle);
+  STrans*         pTransInst = (STrans*)shandle;
+  SRpcHandleInfo* info = &pReq->info;
+
+  int          idx = -1;
+  SCliThrdObj* pThrd = NULL;
+  SExHandle*   exh = info->handle;
+  int64_t      refId = -1;
+  if (exh != NULL) {
+    refId = exh->refId;
+  }
+
+  CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd);
   if (idx == -1) {
     idx = cliRBChoseIdx(pTransInst);
+    pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx];
   }
   TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64());
 
@@ -1097,7 +1139,6 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra
   pCtx->epSet = *pEpSet;
   pCtx->ahandle = pReq->info.ahandle;
   pCtx->msgType = pReq->msgType;
-  pCtx->hThrdIdx = idx;
 
   if (ctx != NULL) {
     pCtx->appCtx = *ctx;
@@ -1110,19 +1151,31 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra
   cliMsg->st = taosGetTimestampUs();
   cliMsg->type = Normal;
 
-  SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx];
-
   STraceId* trace = &pReq->info.traceId;
-  tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid,
+  tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid,
           EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle);
-  ASSERT(transSendAsync(thrd->asyncPool, &(cliMsg->q)) == 0);
+  ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0);
+_return1:
+  return;
+_return2:
+  return;
 }
 
 void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) {
-  STrans* pTransInst = (STrans*)shandle;
-  int     idx = CONN_HOST_THREAD_IDX(pReq->info.handle);
+  STrans*         pTransInst = (STrans*)shandle;
+  SRpcHandleInfo* info = &pReq->info;
+  SCliThrdObj*    pThrd = NULL;
+  int             idx = -1;
+  SExHandle*      exh = info->handle;
+  int64_t         refId = -1;
+  if (exh != NULL) {
+    refId = exh->refId;
+  }
+
+  CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd);
   if (idx == -1) {
     idx = cliRBChoseIdx(pTransInst);
+    pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx];
   }
   tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t));
   tsem_init(sem, 0, 0);
@@ -1133,7 +1186,6 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM
   pCtx->epSet = *pEpSet;
   pCtx->ahandle = pReq->info.ahandle;
   pCtx->msgType = pReq->msgType;
-  pCtx->hThrdIdx = idx;
   pCtx->pSem = sem;
   pCtx->pRsp = pRsp;
 
@@ -1143,16 +1195,18 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM
   cliMsg->st = taosGetTimestampUs();
   cliMsg->type = Normal;
 
-  SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx];
-
   STraceId* trace = &pReq->info.traceId;
-  tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid,
+  tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid,
           EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle);
 
-  transSendAsync(thrd->asyncPool, &(cliMsg->q));
+  transSendAsync(pThrd->asyncPool, &(cliMsg->q));
   tsem_wait(sem);
   tsem_destroy(sem);
   taosMemoryFree(sem);
+_return1:
+  return;
+_return2:
+  return;
 }
 /*
  *
@@ -1168,7 +1222,6 @@ void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) {
   }
   for (int i = 0; i < pTransInst->numOfThreads; i++) {
     STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
-    pCtx->hThrdIdx = i;
     pCtx->cvtAddr = cvtAddr;
 
     SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg));

source/libs/transport/src/transComm.c

@@ -455,7 +455,7 @@ void transPrintEpSet(SEpSet* pEpSet) {
     return;
   }
   char buf[512] = {0};
-  int  len = snprintf(buf, sizeof(buf), "epset { ");
+  int  len = snprintf(buf, sizeof(buf), "epset:{ ");
   for (int i = 0; i < pEpSet->numOfEps; i++) {
     if (i == pEpSet->numOfEps - 1) {
       len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);

source/libs/transport/src/transSvr.c

@@ -206,32 +206,6 @@ static bool addHandleToAcceptloop(void* arg);
     }                                                   \
   } while (0)
 
-#define ASYNC_CHECK_HANDLE(exh1, refId)                                                                               \
-  do {                                                                                                                \
-    if (refId > 0) {                                                                                                  \
-      tTrace("handle step1");                                                                                         \
-      SExHandle* exh2 = transAcquireExHandle(refMgt, refId);                                                          \
-      if (exh2 == NULL || refId != exh2->refId) {                                                                     \
-        tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1,        \
-               exh2 ? exh2->refId : 0, refId);                                                                        \
-        goto _return1;                                                                                                \
-      }                                                                                                               \
-    } else if (refId == 0) {                                                                                          \
-      tTrace("handle step2");                                                                                         \
-      SExHandle* exh2 = transAcquireExHandle(refMgt, refId);                                                          \
-      if (exh2 == NULL || refId != exh2->refId) {                                                                     \
-        tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \
-               exh2 ? exh2->refId : 0);                                                                               \
-        goto _return1;                                                                                                \
-      } else {                                                                                                        \
-        refId = exh1->refId;                                                                                          \
-      }                                                                                                               \
-    } else if (refId < 0) {                                                                                           \
-      tTrace("handle step3");                                                                                         \
-      goto _return2;                                                                                                  \
-    }                                                                                                                 \
-  } while (0)
-
 void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
   SSvrConn*    conn = handle->data;
   SConnBuffer* pBuf = &conn->readBuf;

source/os/src/osSocket.c

@@ -946,7 +946,7 @@ int32_t taosGetFqdn(char *fqdn) {
 #endif  // __APPLE__
   int32_t ret = getaddrinfo(hostname, NULL, &hints, &result);
   if (!result) {
-    fprintf(stderr,"failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
+    fprintf(stderr, "failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret));
     return -1;
   }
 
@@ -1073,7 +1073,7 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) {
  * Set TCP connection timeout per-socket level.
  * ref [https://github.com/libuv/help/issues/54]
  */
-int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) {
+int32_t taosCreateSocketWithTimeout(uint32_t timeout) {
 #if defined(WINDOWS)
   SOCKET fd;
 #else
@@ -1083,11 +1083,11 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) {
     return -1;
   }
 #if defined(WINDOWS)
-  if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&conn_timeout_sec, sizeof(conn_timeout_sec))) {
+  if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) {
     return -1;
   }
 #else  // Linux like systems
-  uint32_t conn_timeout_ms = conn_timeout_sec * 1000;
+  uint32_t conn_timeout_ms = timeout * 1000;
   if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) {
     return -1;
   }