TD-1645

src/rpc/src/rpcTcp.c

@@ -62,7 +62,7 @@ typedef struct {
   char        label[TSDB_LABEL_LEN];
   int         numOfThreads;
   void *      shandle;
-  SThreadObj *pThreadObj;
+  SThreadObj **pThreadObj;
   pthread_t   thread;
 } SServerObj;
 
@@ -90,7 +90,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
   tstrncpy(pServerObj->label, label, sizeof(pServerObj->label));
   pServerObj->numOfThreads = numOfThreads;
 
-  pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads);
+  pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads);
   if (pServerObj->pThreadObj == NULL) {
     tError("TCP:%s no enough memory", label);
     terrno = TAOS_SYSTEM_ERROR(errno); 
@@ -104,19 +104,28 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
 
   // initialize parameters in case it may encounter error later 
-  pThreadObj = pServerObj->pThreadObj;
   for (int i = 0; i < numOfThreads; ++i) {
+    pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1);
+    if (pThreadObj == NULL) {
+      tError("TCP:%s no enough memory", label);
+      terrno = TAOS_SYSTEM_ERROR(errno); 
+      for (int j=0; j<i; ++j) free(pServerObj->pThreadObj[j]);
+      free(pServerObj->pThreadObj);
+      free(pServerObj);
+      return NULL;
+    }
+      
+    pServerObj->pThreadObj[i] = pThreadObj;
     pThreadObj->pollFd = -1;
     taosResetPthread(&pThreadObj->thread);
     pThreadObj->processData = fp;
     tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
     pThreadObj->shandle = shandle;
-    pThreadObj++;
   }
 
   // initialize mutex, thread, fd which may fail
-  pThreadObj = pServerObj->pThreadObj;
   for (int i = 0; i < numOfThreads; ++i) {
+    pThreadObj = pServerObj->pThreadObj[i];
     code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
     if (code < 0) {
       tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno));
@@ -137,7 +146,6 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
     }
 
     pThreadObj->threadId = i;
-    pThreadObj++;
   }
 
   pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
@@ -166,6 +174,11 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
   pThreadObj->stop = true;
   eventfd_t fd = -1;
 
+  if (pThreadObj->thread == pthread_self()) {
+    pthread_detach(pthread_self());
+    return;
+  }
+
   if (taosCheckPthreadValid(pThreadObj->thread) && pThreadObj->pollFd >= 0) {
     // signal the thread to stop, try graceful method first,
     // and use pthread_cancel when failed
@@ -184,14 +197,7 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) {
   }
 
   if (taosCheckPthreadValid(pThreadObj->thread)) pthread_join(pThreadObj->thread, NULL);
-  if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd);
   if (fd != -1) taosCloseSocket(fd);
-
-  while (pThreadObj->pHead) {
-    SFdObj *pFdObj = pThreadObj->pHead;
-    pThreadObj->pHead = pFdObj->next;
-    taosFreeFdObj(pFdObj);
-  }
 }
 
 void taosStopTcpServer(void *handle) {
@@ -210,7 +216,7 @@ void taosCleanUpTcpServer(void *handle) {
   if (pServerObj == NULL) return;
 
   for (int i = 0; i < pServerObj->numOfThreads; ++i) {
-    pThreadObj = pServerObj->pThreadObj + i;
+    pThreadObj = pServerObj->pThreadObj[i];
     taosStopTcpThread(pThreadObj);
     pthread_mutex_destroy(&(pThreadObj->mutex));
   }
@@ -249,7 +255,7 @@ static void *taosAcceptTcpConnection(void *arg) {
     taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
 
     // pick up the thread to handle this connection
-    pThreadObj = pServerObj->pThreadObj + threadId;
+    pThreadObj = pServerObj->pThreadObj[threadId];
 
     SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
     if (pFdObj) {
@@ -329,8 +335,6 @@ void taosCleanUpTcpClient(void *chandle) {
 
   taosStopTcpThread(pThreadObj);
   tDebug ("%s TCP client is cleaned up", pThreadObj->label);
-
-  taosTFree(pThreadObj);
 }
 
 void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) {
@@ -503,8 +507,21 @@ static void *taosProcessTcpData(void *param) {
       pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo);
       if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj);
     }
+
+    if (pThreadObj->stop) break; 
+  }
+
+  if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd);
+
+  while (pThreadObj->pHead) {
+    SFdObj *pFdObj = pThreadObj->pHead;
+    pThreadObj->pHead = pFdObj->next;
+    taosFreeFdObj(pFdObj);
   }
 
+  tDebug("%s TCP thread exits ...", pThreadObj->label);
+  taosTFree(pThreadObj);
+
   return NULL;
 }
 

src/rpc/src/rpcUdp.c

@@ -140,15 +140,18 @@ void taosStopUdpConnection(void *handle) {
     pConn = pSet->udpConn + i;
     if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR);
     if (pConn->fd >=0) taosCloseSocket(pConn->fd);
+    pConn->fd = -1;
   }
 
   for (int i = 0; i < pSet->threads; ++i) {
     pConn = pSet->udpConn + i;
     if (taosCheckPthreadValid(pConn->thread)) {
-      pthread_join(pConn->thread, NULL);
+      if (pConn->thread == pthread_self()) {
+        pthread_detach(pthread_self());
+      } else {
+        pthread_join(pConn->thread, NULL);
+      }
     }
-    taosTFree(pConn->buffer);
-    // tTrace("%s UDP thread is closed, index:%d", pConn->label, i);
   }
 
   tDebug("%s UDP is stopped", pSet->label);
@@ -230,6 +233,9 @@ static void *taosRecvUdpData(void *param) {
     (*(pConn->processData))(&recvInfo);
   }
 
+  taosTFree(pConn->buffer);
+  tDebug("%s UDP recv thread exits", pConn->label);
+
   return NULL;
 }