support both TCP and UDP simultaneously for both server and client

src/inc/trpc.h

@@ -23,18 +23,8 @@ extern "C" {
 #include <stdint.h>
 #include "taosdef.h"
 
-#define TAOS_CONN_UDPS     0
-#define TAOS_CONN_UDPC     1
-#define TAOS_CONN_TCPS     2
-#define TAOS_CONN_TCPC     3
-#define TAOS_CONN_HTTPS    4
-#define TAOS_CONN_HTTPC    5
-
-#define TAOS_SOCKET_TYPE_NAME_TCP  "tcp"
-#define TAOS_SOCKET_TYPE_NAME_UDP  "udp"
-
-#define TAOS_CONN_SOCKET_TYPE_S()  ((strcasecmp(tsSocketType, TAOS_SOCKET_TYPE_NAME_UDP) == 0)? TAOS_CONN_UDPS:TAOS_CONN_TCPS)
-#define TAOS_CONN_SOCKET_TYPE_C()  ((strcasecmp(tsSocketType, TAOS_SOCKET_TYPE_NAME_UDP) == 0)? TAOS_CONN_UDPC:TAOS_CONN_TCPC)
+#define TAOS_CONN_SERVER   0
+#define TAOS_CONN_CLIENT   1
 
 extern int tsRpcHeadSize;
 

src/rpc/inc/rpcClient.h

@@ -26,7 +26,7 @@ void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp,
 void taosCleanUpTcpClient(void *chandle);
 void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port);
 void taosCloseTcpClientConnection(void *chandle);
-int taosSendTcpClientData(uint32_t ip, uint16_t port, char *data, int len, void *chandle);
+int  taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);
 
 #ifdef __cplusplus
 }

src/rpc/inc/rpcHead.h

@@ -20,12 +20,29 @@
 extern "C" {
 #endif
 
+#define RPC_CONN_UDPS   0
+#define RPC_CONN_UDPC   1
+#define RPC_CONN_TCPS   2
+#define RPC_CONN_TCPC   3
+#define RPC_CONN_TCP    2
+
+typedef struct {
+  void    *msg;
+  int      msgLen;
+  uint32_t ip; 
+  uint16_t port;
+  int      connType;
+  void    *shandle;
+  void    *thandle;
+  void    *chandle;
+} SRecvInfo;
+
 #pragma pack(push, 1)
 
 typedef struct {
   char     version:4; // RPC version
   char     comp:4;    // compression algorithm, 0:no compression 1:lz4
-  char     tcp:2;     // tcp flag 
+  char     resflag:2; // reserved bits
   char     spi:3;     // security parameter index
   char     encrypt:3; // encrypt algorithm, 0: no encryption
   uint16_t tranId;    // transcation ID
@@ -33,12 +50,12 @@ typedef struct {
   uint32_t sourceId;  // source ID, an index for connection list  
   uint32_t destId;    // destination ID, an index for connection list
   uint32_t destIp;    // destination IP address, for NAT scenario
-  char     user[TSDB_UNI_LEN];
+  char     user[TSDB_UNI_LEN]; // user ID 
   uint16_t port;      // for UDP only, port may be changed
   char     empty[1];  // reserved
   uint8_t  msgType;   // message type  
   int32_t  msgLen;    // message length including the header iteslf
-  int32_t  code;
+  int32_t  code;      // code in response message
   uint8_t  content[0]; // message body starts from here
 } SRpcHead;
 
@@ -54,6 +71,7 @@ typedef struct {
 
 #pragma pack(pop)
 
+
 #ifdef __cplusplus
 }
 #endif

src/rpc/inc/rpcServer.h

@@ -25,7 +25,7 @@ extern "C" {
 void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
 void taosCleanUpTcpServer(void *param);
 void taosCloseTcpServerConnection(void *param);
-int taosSendTcpServerData(uint32_t ip, uint16_t port, char *data, int len, void *chandle);
+int  taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle);
 
 #ifdef __cplusplus
 }

src/rpc/inc/rpcUdp.h

@@ -22,10 +22,9 @@ extern "C" {
 
 #include "taosdef.h"
 
-void *taosInitUdpServer(char *ip, uint16_t port, char *label, int, void *fp, void *shandle);
-void *taosInitUdpClient(char *ip, uint16_t port, char *label, int, void *fp, void *shandle);
+void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int, void *fp, void *shandle);
 void  taosCleanUpUdpConnection(void *handle);
-int   taosSendUdpData(uint32_t ip, uint16_t port, char *data, int dataLen, void *chandle);
+int   taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle);
 void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t port);
 
 void  taosFreeMsgHdr(void *hdr);

src/rpc/src/rpcClient.c

@@ -45,16 +45,143 @@ typedef struct _tcp_client {
   int             numOfFds;
   char            label[12];
   char            ipstr[20];
-  void *          shandle;  // handle passed by upper layer during server initialization
-  void *(*processData)(char *data, int dataLen, unsigned int ip, uint16_t port, void *shandle, void *thandle,
-                       void *chandle);
-  // char   buffer[128000];
+  void           *shandle;  // handle passed by upper layer during server initialization
+  void           *(*processData)(SRecvInfo *pRecv);
 } STcpClient;
 
 #define maxTcpEvents 100
 
+static void taosCleanUpTcpFdObj(STcpFd *pFdObj);
+static void *taosReadTcpData(void *param);
+
+void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
+  STcpClient    *pTcp;
+  pthread_attr_t thattr;
+
+  pTcp = (STcpClient *)malloc(sizeof(STcpClient));
+  memset(pTcp, 0, sizeof(STcpClient));
+  strcpy(pTcp->label, label);
+  strcpy(pTcp->ipstr, ip);
+  pTcp->shandle = shandle;
+
+  if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) {
+    tError("%s failed to init TCP mutex, reason:%s", label, strerror(errno));
+    return NULL;
+  }
+
+  if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) {
+    tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno));
+    return NULL;
+  }
+
+  pTcp->pollFd = epoll_create(10);  // size does not matter
+  if (pTcp->pollFd < 0) {
+    tError("%s failed to create TCP epoll", label);
+    return NULL;
+  }
+
+  pTcp->processData = fp;
+
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  if (pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)) != 0) {
+    tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno));
+    return NULL;
+  }
+
+  tTrace("%s TCP client is initialized, ip:%s port:%hu", label, ip, port);
+
+  return pTcp;
+}
+
+void taosCleanUpTcpClient(void *chandle) {
+  STcpClient *pTcp = (STcpClient *)chandle;
+  if (pTcp == NULL) return;
+
+  while (pTcp->pHead) {
+    taosCleanUpTcpFdObj(pTcp->pHead);
+    pTcp->pHead = pTcp->pHead->next;
+  }
+
+  close(pTcp->pollFd);
+
+  pthread_cancel(pTcp->thread);
+  pthread_join(pTcp->thread, NULL);
+
+  // tTrace (":%s, all connections are cleaned up", pTcp->label);
+
+  tfree(pTcp);
+}
+
+void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
+  STcpClient *       pTcp = (STcpClient *)shandle;
+  STcpFd *           pFdObj;
+  struct epoll_event event;
+  struct in_addr     destIp;
+  int                fd;
+
+  fd = taosOpenTcpClientSocket(ip, port, pTcp->ipstr);
+  if (fd <= 0) return NULL;
+
+  pFdObj = (STcpFd *)malloc(sizeof(STcpFd));
+  if (pFdObj == NULL) {
+    tError("%s no enough resource to allocate TCP FD IDs", pTcp->label);
+    tclose(fd);
+    return NULL;
+  }
+
+  memset(pFdObj, 0, sizeof(STcpFd));
+  pFdObj->fd = fd;
+  strcpy(pFdObj->ipstr, ip);
+  inet_aton(ip, &destIp);
+  pFdObj->ip = destIp.s_addr;
+  pFdObj->port = port;
+  pFdObj->pTcp = pTcp;
+  pFdObj->thandle = thandle;
+  pFdObj->signature = pFdObj;
+
+  event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
+  event.data.ptr = pFdObj;
+  if (epoll_ctl(pTcp->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+    tError("%s failed to add TCP FD for epoll, error:%s", pTcp->label, strerror(errno));
+    tfree(pFdObj);
+    tclose(fd);
+    return NULL;
+  }
+
+  // notify the data process, add into the FdObj list
+  pthread_mutex_lock(&(pTcp->mutex));
+  pFdObj->next = pTcp->pHead;
+  if (pTcp->pHead) (pTcp->pHead)->prev = pFdObj;
+  pTcp->pHead = pFdObj;
+  pTcp->numOfFds++;
+  pthread_cond_signal(&pTcp->fdReady);
+  pthread_mutex_unlock(&(pTcp->mutex));
+
+  tTrace("%s TCP connection to %s:%hu is created, FD:%p numOfFds:%d", pTcp->label, ip, port, pFdObj, pTcp->numOfFds);
+
+  return pFdObj;
+}
+
+void taosCloseTcpClientConnection(void *chandle) {
+  STcpFd *pFdObj = (STcpFd *)chandle;
+
+  if (pFdObj == NULL) return;
+
+  taosCleanUpTcpFdObj(pFdObj);
+}
+
+int taosSendTcpClientData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
+  STcpFd *pFdObj = (STcpFd *)chandle;
+
+  if (chandle == NULL) return -1;
+
+  return (int)send(pFdObj->fd, data, (size_t)len, 0);
+}
+
 static void taosCleanUpTcpFdObj(STcpFd *pFdObj) {
   STcpClient *pTcp;
+  SRecvInfo   recvInfo;
 
   if (pFdObj == NULL) return;
   if (pFdObj->signature != pFdObj) return;
@@ -75,8 +202,6 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) {
   if (pTcp->numOfFds < 0) 
     tError("%s number of TCP FDs shall never be negative, FD:%p", pTcp->label, pFdObj);
 
-  // remove from the FdObject list
-
   if (pFdObj->prev) {
     (pFdObj->prev)->next = pFdObj->next;
   } else {
@@ -89,40 +214,28 @@ static void taosCleanUpTcpFdObj(STcpFd *pFdObj) {
 
   pthread_mutex_unlock(&pTcp->mutex);
 
-  // notify the upper layer to clean the associated context
-  if (pFdObj->thandle) (*(pTcp->processData))(NULL, 0, 0, 0, pTcp->shandle, pFdObj->thandle, NULL);
+  recvInfo.msg = NULL;
+  recvInfo.msgLen = 0;
+  recvInfo.ip = 0;
+  recvInfo.port = 0;
+  recvInfo.shandle = pTcp->shandle;
+  recvInfo.thandle = pFdObj->thandle;;
+  recvInfo.chandle = NULL;
+  recvInfo.connType = RPC_CONN_TCP;
 
+  if (pFdObj->thandle) (*(pTcp->processData))(&recvInfo);
   tTrace("%s TCP is cleaned up, FD:%p numOfFds:%d", pTcp->label, pFdObj, pTcp->numOfFds);
 
   memset(pFdObj, 0, sizeof(STcpFd));
-
   tfree(pFdObj);
 }
 
-void taosCleanUpTcpClient(void *chandle) {
-  STcpClient *pTcp = (STcpClient *)chandle;
-  if (pTcp == NULL) return;
-
-  while (pTcp->pHead) {
-    taosCleanUpTcpFdObj(pTcp->pHead);
-    pTcp->pHead = pTcp->pHead->next;
-  }
-
-  close(pTcp->pollFd);
-
-  pthread_cancel(pTcp->thread);
-  pthread_join(pTcp->thread, NULL);
-
-  // tTrace (":%s, all connections are cleaned up", pTcp->label);
-
-  tfree(pTcp);
-}
-
 static void *taosReadTcpData(void *param) {
-  STcpClient *       pTcp = (STcpClient *)param;
+  STcpClient        *pTcp = (STcpClient *)param;
   int                i, fdNum;
-  STcpFd *           pFdObj;
+  STcpFd            *pFdObj;
   struct epoll_event events[maxTcpEvents];
+  SRecvInfo          recvInfo;
 
   while (1) {
     pthread_mutex_lock(&pTcp->mutex);
@@ -186,132 +299,22 @@ static void *taosReadTcpData(void *param) {
         continue;
       }
 
-      pFdObj->thandle =
-          (*(pTcp->processData))(buffer, dataLen, pFdObj->ip, pFdObj->port, pTcp->shandle, pFdObj->thandle, pFdObj);
-
-      if (pFdObj->thandle == NULL) taosCleanUpTcpFdObj(pFdObj);
-    }
-  }
-
-  return NULL;
-}
-
-void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) {
-  STcpClient *   pTcp;
-  pthread_attr_t thattr;
-
-  pTcp = (STcpClient *)malloc(sizeof(STcpClient));
-  memset(pTcp, 0, sizeof(STcpClient));
-  strcpy(pTcp->label, label);
-  strcpy(pTcp->ipstr, ip);
-  pTcp->shandle = shandle;
-
-  if (pthread_mutex_init(&(pTcp->mutex), NULL) < 0) {
-    tError("%s failed to init TCP mutex, reason:%s", label, strerror(errno));
-    return NULL;
-  }
-
-  if (pthread_cond_init(&(pTcp->fdReady), NULL) != 0) {
-    tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno));
-    return NULL;
-  }
-
-  pTcp->pollFd = epoll_create(10);  // size does not matter
-  if (pTcp->pollFd < 0) {
-    tError("%s failed to create TCP epoll", label);
-    return NULL;
-  }
-
-  pTcp->processData = fp;
-
-  pthread_attr_init(&thattr);
-  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
-  if (pthread_create(&(pTcp->thread), &thattr, taosReadTcpData, (void *)(pTcp)) != 0) {
-    tError("%s failed to create TCP read data thread, reason:%s", label, strerror(errno));
-    return NULL;
-  }
-
-  tTrace("%s TCP client is initialized, ip:%s port:%hu", label, ip, port);
-
-  return pTcp;
-}
-
-void taosCloseTcpClientConnection(void *chandle) {
-  STcpFd *pFdObj = (STcpFd *)chandle;
-
-  if (pFdObj == NULL) return;
+      recvInfo.msg = buffer;
+      recvInfo.msgLen = dataLen;
+      recvInfo.ip = pFdObj->ip;
+      recvInfo.port = pFdObj->port;
+      recvInfo.shandle = pTcp->shandle;
+      recvInfo.thandle = pFdObj->thandle;;
+      recvInfo.chandle = pFdObj;
+      recvInfo.connType = RPC_CONN_TCP;
 
-  taosCleanUpTcpFdObj(pFdObj);
-}
+      pFdObj->thandle = (*(pTcp->processData))(&recvInfo);
 
-void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
-  STcpClient *       pTcp = (STcpClient *)shandle;
-  STcpFd *           pFdObj;
-  struct epoll_event event;
-  struct in_addr     destIp;
-  int                fd;
-
-  /*
-    if ( (strcmp(ip, "127.0.0.1") == 0 ) || (strcmp(ip, "localhost") == 0 ) ) {
-      fd = taosOpenUDClientSocket(ip, port);
-    } else {
-      fd = taosOpenTcpClientSocket(ip, port, pTcp->ipstr);
+      if (pFdObj->thandle == NULL) taosCleanUpTcpFdObj(pFdObj);
     }
-  */
-
-  fd = taosOpenTcpClientSocket(ip, port, pTcp->ipstr);
-
-  if (fd <= 0) return NULL;
-
-  pFdObj = (STcpFd *)malloc(sizeof(STcpFd));
-  if (pFdObj == NULL) {
-    tError("%s no enough resource to allocate TCP FD IDs", pTcp->label);
-    tclose(fd);
-    return NULL;
   }
 
-  memset(pFdObj, 0, sizeof(STcpFd));
-  pFdObj->fd = fd;
-  strcpy(pFdObj->ipstr, ip);
-  inet_aton(ip, &destIp);
-  pFdObj->ip = destIp.s_addr;
-  pFdObj->port = port;
-  pFdObj->pTcp = pTcp;
-  pFdObj->thandle = thandle;
-  pFdObj->signature = pFdObj;
-
-  event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP;
-  event.data.ptr = pFdObj;
-  if (epoll_ctl(pTcp->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
-    tError("%s failed to add TCP FD for epoll, error:%s", pTcp->label, strerror(errno));
-    tfree(pFdObj);
-    tclose(fd);
   return NULL;
-  }
-
-  // notify the data process, add into the FdObj list
-  pthread_mutex_lock(&(pTcp->mutex));
-
-  pFdObj->next = pTcp->pHead;
-
-  if (pTcp->pHead) (pTcp->pHead)->prev = pFdObj;
-
-  pTcp->pHead = pFdObj;
-
-  pTcp->numOfFds++;
-  pthread_cond_signal(&pTcp->fdReady);
-
-  pthread_mutex_unlock(&(pTcp->mutex));
-
-  tTrace("%s TCP connection to %s:%hu is created, FD:%p numOfFds:%d", pTcp->label, ip, port, pFdObj, pTcp->numOfFds);
-
-  return pFdObj;
 }
 
-int taosSendTcpClientData(uint32_t ip, uint16_t port, char *data, int len, void *chandle) {
-  STcpFd *pFdObj = (STcpFd *)chandle;
-
-  if (chandle == NULL) return -1;
 
-  return (int)send(pFdObj->fd, data, (size_t)len, 0);
-}

src/rpc/src/rpcMain.c

@@ -62,7 +62,8 @@ typedef struct {
   void     *idPool;   // handle to ID pool
   void     *tmrCtrl;  // handle to timer
   void     *hash;     // handle returned by hash utility
-  void     *shandle;  // returned handle from lower layer during initialization
+  void     *tcphandle;// returned handle from TCP initialization
+  void     *udphandle;// returned handle from UDP initialization
   void     *pCache;   // connection cache
   pthread_mutex_t  mutex;
   struct _RpcConn *connList;  // connection list
@@ -79,6 +80,7 @@ typedef struct {
   int16_t   numOfTry;   // number of try for different servers
   int8_t    oldIndex;   // server IP index passed by app
   int8_t    redirect;   // flag to indicate redirect
+  int8_t    connType;   // connection type
   char      msg[0];     // RpcHead starts from here
 } SRpcReqContext;
 
@@ -113,6 +115,7 @@ typedef struct _RpcConn {
   char     *pReqMsg;    // request message including header
   int       reqMsgLen;  // request message length
   SRpcInfo *pRpc;       // the associated SRpcInfo
+  int       connType;   // connection type
   SRpcReqContext *pContext; // request context
 } SRpcConn;
 
@@ -122,9 +125,16 @@ int tsRpcProgressTime = 10;  // milliseocnds
 int tsRpcMaxRetry;
 int tsRpcHeadSize;
 
+// server:0 client:1  tcp:2 udp:0
+#define RPC_CONN_UDPS   0
+#define RPC_CONN_UDPC   1
+#define RPC_CONN_TCPS   2
+#define RPC_CONN_TCPC   3
+#define RPC_CONN_TCP    2
+
 void *(*taosInitConn[])(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = {
-    taosInitUdpServer, 
-    taosInitUdpClient, 
+    taosInitUdpConnection,
+    taosInitUdpConnection,
     taosInitTcpServer, 
     taosInitTcpClient
 };
@@ -136,7 +146,7 @@ void (*taosCleanUpConn[])(void *thandle) = {
     taosCleanUpTcpClient
 };
 
-int (*taosSendData[])(uint32_t ip, uint16_t port, char *data, int len, void *chandle) = {
+int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = {
     taosSendUdpData, 
     taosSendUdpData, 
     taosSendTcpServerData, 
@@ -157,19 +167,19 @@ void (*taosCloseConn[])(void *chandle) = {
     taosCloseTcpClientConnection
 };
 
-static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort);
+static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType);
 static void      rpcCloseConn(void *thandle);
-static SRpcConn *rpcSetConnToServer(SRpcInfo *pRpc, SRpcIpSet ipSet);
+static SRpcConn *rpcSetConnToServer(SRpcReqContext *pContext);
 static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc);
 static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, char *user, char *hashstr);
 static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, char *user, char *hashstr);
 
 static void  rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext);
 static void  rpcSendQuickRsp(SRpcConn *pConn, int32_t code);
-static void  rpcSendErrorMsgToPeer(SRpcInfo *pRpc, char *pMsg, int32_t code, uint32_t ip, uint16_t port, void *chandle);
+static void  rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code);
 static void  rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen);
 
-static void *rpcProcessMsgFromPeer(void *msg, int msgLen, uint32_t ip, uint16_t port, void *shandle, void *thandle, void *chandle);
+static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv);
 static void  rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead);
 static void  rpcProcessConnError(void *param, void *id);
 static void  rpcProcessRetryTimer(void *, void *);
@@ -194,7 +204,8 @@ void *rpcOpen(SRpcInit *pInit) {
   if(pInit->label) strcpy(pRpc->label, pInit->label);
   pRpc->connType = pInit->connType;
   pRpc->idleTime = pInit->idleTime;
-  pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads;
+  // pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads;
+  pRpc->numOfThreads = 1;
   if (pInit->localIp) strcpy(pRpc->localIp, pInit->localIp);
   pRpc->localPort = pInit->localPort;
   pRpc->afp = pInit->afp;
@@ -207,9 +218,12 @@ void *rpcOpen(SRpcInit *pInit) {
   pRpc->cfp = pInit->cfp;
   pRpc->afp = pInit->afp;
 
-  pRpc->shandle = (*taosInitConn[pRpc->connType])(pRpc->localIp, pRpc->localPort, pRpc->label, 
+  pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(pRpc->localIp, pRpc->localPort, pRpc->label, 
+                                                  pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
+  pRpc->udphandle = (*taosInitConn[pRpc->connType])(pRpc->localIp, pRpc->localPort, pRpc->label, 
                                                   pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc);
-  if (pRpc->shandle == NULL) {
+
+  if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) {
     tError("%s failed to init network, %s:%d", pRpc->label, pRpc->localIp, pRpc->localPort);
     rpcClose(pRpc);
     return NULL;
@@ -237,19 +251,21 @@ void *rpcOpen(SRpcInit *pInit) {
     return NULL;
   }
 
+  if (pRpc->connType == TAOS_CONN_SERVER) {
     pRpc->hash = taosInitStrHash(pRpc->sessions, sizeof(pRpc), taosHashString);
     if (pRpc->hash == NULL) {
       tError("%s failed to init string hash", pRpc->label);
       rpcClose(pRpc);
       return NULL;
     }
-
+  } else {
     pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, tsShellActivityTimer*1000); 
     if ( pRpc->pCache == NULL ) {
       tError("%s failed to init connection cache", pRpc->label);
       rpcClose(pRpc);
       return NULL;
     }
+  }
 
   pthread_mutex_init(&pRpc->mutex, NULL);
 
@@ -261,7 +277,8 @@ void *rpcOpen(SRpcInit *pInit) {
 void rpcClose(void *param) {
   SRpcInfo *pRpc = (SRpcInfo *)param;
 
-  (*taosCleanUpConn[pRpc->connType])(pRpc->shandle);
+  (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle);
+  (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle);
 
   for (int i = 0; i < pRpc->sessions; ++i) {
     if (pRpc->connList[i].user[0]) {
@@ -313,6 +330,16 @@ void rpcSendRequest(void *shandle, SRpcIpSet *pIpSet, char type, void *pCont, in
   pContext->msgType = type;
   pContext->oldIndex = pIpSet->index;
 
+  pContext->connType = RPC_CONN_UDPC; 
+  if (contLen > 16000) pContext->connType = RPC_CONN_TCPC;
+
+  // connection type is application specific. 
+  // for TDengine, all the query, show commands shall have TCP connection
+  if (type == TSDB_MSG_TYPE_DNODE_QUERY || type == TSDB_MSG_TYPE_DNODE_RETRIEVE ||
+      type == TSDB_MSG_TYPE_STABLE_META || type == TSDB_MSG_TYPE_MULTI_TABLE_META ||
+      type == TSDB_MSG_TYPE_SHOW ) 
+    pContext->connType = RPC_CONN_TCPC;
+  
   rpcSendReqToServer(pRpc, pContext);
 
   return;
@@ -346,7 +373,6 @@ void rpcSendResponse(void *handle, int32_t code, void *pCont, int contLen) {
   pHead->version = 1;
   pHead->msgType = pConn->inType+1;
   pHead->spi = 0;
-  pHead->tcp = 0;
   pHead->encrypt = 0;
   pHead->tranId = pConn->inTranId;
   pHead->sourceId = pConn->ownId;
@@ -388,7 +414,6 @@ void rpcSendRedirectRsp(void *thandle, SRpcIpSet *pIpSet) {
 
 void rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
   SRpcConn  *pConn = (SRpcConn *)thandle;
-  SRpcInfo  *pRpc = pConn->pRpc;
 
   pInfo->clientIp = pConn->peerIp;
   pInfo->clientPort = pConn->peerPort;
@@ -396,7 +421,7 @@ void rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
   strcpy(pInfo->user, pConn->user);
 }
 
-static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort) {
+static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType) {
   SRpcConn *pConn;
 
   pConn = rpcAllocateClientConn(pRpc); 
@@ -406,12 +431,14 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort)
     pConn->peerIp = inet_addr(peerIpStr);
     pConn->peerPort = peerPort;
     strcpy(pConn->user, pRpc->user);
+    pConn->connType = connType;
 
-    if (taosOpenConn[pRpc->connType]) {
-      pConn->chandle = (*taosOpenConn[pRpc->connType])(pRpc->shandle, pConn, pConn->peerIpstr, pConn->peerPort);
+    if (taosOpenConn[connType]) {
+      void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle;
+      pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIpstr, pConn->peerPort);
       if (pConn->chandle) {
-        tTrace("%s %p, rpc connection is set up, sid:%d id:%s ip:%s:%hu localPort:%d", pRpc->label, 
-                pConn, pConn->sid, pRpc->user, pConn->peerIpstr, pConn->peerPort, pConn->localPort);
+        tTrace("%s %p, rpc connection is set up, sid:%d id:%s ip:%s:%hu connType:%d", pRpc->label, 
+                pConn, pConn->sid, pRpc->user, pConn->peerIpstr, pConn->peerPort, pConn->connType);
       } else {
         tError("%s %p, failed to set up connection to ip:%s:%hu", pRpc->label, pConn, 
                 pConn->peerIpstr, pConn->peerPort);
@@ -433,14 +460,14 @@ static void rpcCloseConn(void *thandle) {
 
   if (pConn->user[0]) {
     pConn->user[0] = 0;
-    if (taosCloseConn[pRpc->connType]) (*taosCloseConn[pRpc->connType])(pConn->chandle);
+    if (taosCloseConn[pConn->connType]) (*taosCloseConn[pConn->connType])(pConn->chandle);
 
     taosTmrStopA(&pConn->pTimer);
     taosTmrStopA(&pConn->pIdleTimer);
 
-    if ( pRpc->connType == TAOS_CONN_UDPS || pRpc->connType == TAOS_CONN_TCPS) {
+    if ( pRpc->connType == TAOS_CONN_SERVER) {
       char hashstr[40] = {0};
-      sprintf(hashstr, "%x:%x:%x", pConn->peerIp, pConn->peerUid, pConn->peerId);
+      sprintf(hashstr, "%x:%x:%x:%d", pConn->peerIp, pConn->peerUid, pConn->peerId, pConn->connType);
       taosDeleteStrHash(pRpc->hash, hashstr);
       rpcFreeOutMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg
       pConn->pRspMsg = NULL;
@@ -540,15 +567,17 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, char *user, char *hashst
   return pConn;
 }
 
-SRpcConn *rpcSetConnToServer(SRpcInfo *pRpc, SRpcIpSet ipSet) {
+SRpcConn *rpcSetConnToServer(SRpcReqContext *pContext) {
   SRpcConn   *pConn;
+  SRpcInfo   *pRpc = pContext->pRpc;
+  SRpcIpSet  *pIpSet = &pContext->ipSet;
 
-  pConn = rpcGetConnFromCache(pRpc->pCache, ipSet.ip[ipSet.index], ipSet.port, pRpc->user);
+  pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->ip[pIpSet->index], pIpSet->port, pRpc->user);
   if ( pConn == NULL ) {
     char ipstr[20] = {0};
-    tinet_ntoa(ipstr, ipSet.ip[ipSet.index]);
-    pConn = rpcOpenConn(pRpc, ipstr, ipSet.port);
-    pConn->destIp = ipSet.ip[ipSet.index];
+    tinet_ntoa(ipstr, pIpSet->ip[pIpSet->index]);
+    pConn = rpcOpenConn(pRpc, ipstr, pIpSet->port, pContext->connType);
+    if (pConn) pConn->destIp = pIpSet->ip[pIpSet->index];
   } 
 
   return pConn;
@@ -618,7 +647,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) {
   taosTmrStopA(&pConn->pTimer);
   pConn->retry = 0;
 
-  if (*pHead->content == TSDB_CODE_ACTION_IN_PROGRESS || pHead->tcp) {
+  if (*pHead->content == TSDB_CODE_ACTION_IN_PROGRESS) {
     if (pConn->tretry <= tsRpcMaxRetry) {
       pConn->tretry++;
       tTrace("%s %p, peer is still processing the transaction", pRpc->label, pConn);
@@ -638,13 +667,12 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t rpcProcessHead(SRpcInfo *pRpc, SRpcConn **ppConn, void *data, int dataLen, uint32_t ip) {
-  int32_t    sid, code = 0;
-  SRpcConn * pConn = NULL;
+static SRpcConn *rpcProcessHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
+  int32_t    sid;
+  SRpcConn  *pConn = NULL;
   char       hashstr[40] = {0};
 
-  *ppConn = NULL;
-  SRpcHead *pHead = (SRpcHead *)data;
+  SRpcHead *pHead = (SRpcHead *)pRecv->msg;
 
   sid = htonl(pHead->destId);
   pHead->code = htonl(pHead->code);
@@ -652,50 +680,54 @@ static int32_t rpcProcessHead(SRpcInfo *pRpc, SRpcConn **ppConn, void *data, int
 
   if (pHead->msgType >= TSDB_MSG_TYPE_MAX || pHead->msgType <= 0) {
     tTrace("%s sid:%d, invalid message type:%d", pRpc->label, sid, pHead->msgType);
-    return TSDB_CODE_INVALID_MSG_TYPE;
+    terrno = TSDB_CODE_INVALID_MSG_TYPE; return NULL;
   }
 
-  if (dataLen != pHead->msgLen) {
+  if (pRecv->msgLen != pHead->msgLen) {
     tTrace("%s sid:%d, %s has invalid length, dataLen:%d, msgLen:%d", pRpc->label, sid,
-           taosMsg[pHead->msgType], dataLen, pHead->msgLen);
-    return TSDB_CODE_INVALID_MSG_LEN;
+           taosMsg[pHead->msgType], pRecv->msgLen, pHead->msgLen);
+    terrno = TSDB_CODE_INVALID_MSG_LEN; return NULL;
   }
 
   if (sid < 0 || sid >= pRpc->sessions) {
     tTrace("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid,
            pRpc->sessions, taosMsg[pHead->msgType]);
-    return TSDB_CODE_INVALID_SESSION_ID;
+    terrno = TSDB_CODE_INVALID_SESSION_ID; return NULL;
   }
 
-  if (sid == 0) sprintf(hashstr, "%x:%x:%x", ip, pHead->uid, pHead->sourceId);
+  if (sid == 0) sprintf(hashstr, "%x:%x:%x:%d", pRecv->ip, pHead->uid, pHead->sourceId, pRecv->connType);
   pConn = rpcGetConnObj(pRpc, sid, pHead->user, hashstr);
-  if (pConn == NULL ) return terrno;
+  if (pConn == NULL) return NULL;
 
-  *ppConn = pConn;
   sid = pConn->sid;
 
-  if (pHead->uid) pConn->peerUid = pHead->uid;
-
-  if (pHead->tcp) {
-    tTrace("%s %p, content will be transfered via TCP", pRpc->label, pConn);
-    if (pConn->outType) taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer);
-    return TSDB_CODE_ALREADY_PROCESSED;
+  pConn->chandle = pRecv->chandle;
+  if (pConn->peerIp != pRecv->ip) {
+      pConn->peerIp = pRecv->ip;
+      char ipstr[20] = {0};
+      tinet_ntoa(ipstr, pRecv->ip);
+      strcpy(pConn->peerIpstr, ipstr);
   }
   
-  code = rpcCheckAuthentication(pConn, (char *)pHead, dataLen);
-  if ( code != 0 ) return code;
+  if (pRecv->port) pConn->peerPort = pRecv->port;
+  if (pHead->port)  pConn->peerPort = pHead->port; 
+  if (pHead->uid) pConn->peerUid = pHead->uid;
+
+  terrno = rpcCheckAuthentication(pConn, (char *)pHead, pRecv->msgLen);
+  if (terrno != 0) return pConn;
 
   if (pHead->msgType != TSDB_MSG_TYPE_REG && pHead->encrypt) {
     // decrypt here
   }
 
   if ( rpcIsReq(pHead->msgType) ) {
-    code = rpcProcessReqHead(pConn, pHead);
+    terrno = rpcProcessReqHead(pConn, pHead);
+    pConn->connType = pRecv->connType;
   } else {
-    code = rpcProcessRspHead(pConn, pHead);
+    terrno = rpcProcessRspHead(pConn, pHead);
   }
 
-  return code;
+  return pConn;
 }
 
 static void rpcProcessBrokenLink(SRpcConn *pConn) {
@@ -713,45 +745,31 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
   rpcCloseConn(pConn);
 }
 
-static void *rpcProcessMsgFromPeer(void *msg, int msgLen, uint32_t ip, uint16_t port, void *shandle, void *thandle, void *chandle) {
-  SRpcHead  *pHead = (SRpcHead *)msg;
-  SRpcInfo  *pRpc = (SRpcInfo *)shandle;
-  SRpcConn  *pConn = (SRpcConn *)thandle;
+static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
+  SRpcHead  *pHead = (SRpcHead *)pRecv->msg;
+  SRpcInfo  *pRpc = (SRpcInfo *)pRecv->shandle;
+  SRpcConn  *pConn = (SRpcConn *)pRecv->thandle;
   int32_t    code = 0;
 
-  tDump(msg, msgLen);
+  tDump(pRecv->msg, pRecv->msgLen);
 
-  if (ip==0 && pConn) {
+  // underlying UDP layer does not know it is server or client
+  pRecv->connType = pRecv->connType | pRpc->connType; 
+
+  if (pRecv->ip==0 && pConn) {
     rpcProcessBrokenLink(pConn); 
-    tfree(msg);
+    tfree(pRecv->msg);
     return NULL;
   }
 
   pthread_mutex_lock(&pRpc->mutex);
-
-  code = rpcProcessHead(pRpc, &pConn, msg, msgLen, ip);
-
-  if (pConn) {
-    // update connection info
-    pConn->chandle = chandle;
-    if (pConn->peerIp != ip) {
-      pConn->peerIp = ip;
-      char ipstr[20] = {0};
-      tinet_ntoa(ipstr, ip);
-      strcpy(pConn->peerIpstr, ipstr);
-    }
-  
-    if (port) pConn->peerPort = port;
-    if (pHead->port)  // port maybe changed by the peer
-      pConn->peerPort = pHead->port;
-  }
-
+  pConn = rpcProcessHead(pRpc, pRecv);
   pthread_mutex_unlock(&pRpc->mutex);
 
   if (pHead->msgType < TSDB_MSG_TYPE_HEARTBEAT || (rpcDebugFlag & 16)) {
     tTrace("%s %p, %s received from 0x%x:%hu, parse code:%x len:%d source:0x%08x dest:0x%08x tranId:%d",
-        pRpc->label, pConn, taosMsg[pHead->msgType], ip, port, code, 
-        msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
+        pRpc->label, pConn, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, code, 
+        pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 
   if (pConn && pRpc->idleTime) {
@@ -761,7 +779,7 @@ static void *rpcProcessMsgFromPeer(void *msg, int msgLen, uint32_t ip, uint16_t
   if (code != TSDB_CODE_ALREADY_PROCESSED) {
     if (code != 0) { // parsing error
       if ( rpcIsReq(pHead->msgType) ) {
-        rpcSendErrorMsgToPeer(pRpc, msg, code, ip, port, chandle);
+        rpcSendErrorMsgToPeer(pRecv, code);
         tTrace("%s %p, %s is sent with error code:%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], code);
       } 
     } else { // parsing OK
@@ -769,7 +787,7 @@ static void *rpcProcessMsgFromPeer(void *msg, int msgLen, uint32_t ip, uint16_t
     }
   }
 
-  if ( code != 0 ) free (msg);
+  if ( code != 0 ) free (pRecv->msg);
   return pConn;
 }
 
@@ -816,7 +834,6 @@ static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) {
   pHead->version = 1;
   pHead->msgType = pConn->inType+1;
   pHead->spi = 0;
-  pHead->tcp = 0;
   pHead->encrypt = 0;
   pHead->tranId = pConn->inTranId;
   pHead->sourceId = pConn->ownId;
@@ -828,19 +845,18 @@ static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) {
   rpcSendMsgToPeer(pConn, msg, 0);
 }
 
-static void rpcSendErrorMsgToPeer(SRpcInfo *pRpc, char *pMsg, int32_t code, uint32_t ip, uint16_t port, void *chandle) {
+static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) {
   SRpcHead  *pRecvHead, *pReplyHead;
   char       msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t) ];
   uint32_t   timeStamp;
   int        msgLen;
 
-  pRecvHead = (SRpcHead *)pMsg;
+  pRecvHead = (SRpcHead *)pRecv->msg;
   pReplyHead = (SRpcHead *)msg;
 
   memset(msg, 0, sizeof(SRpcHead));
   pReplyHead->version = pRecvHead->version;
   pReplyHead->msgType = (char)(pRecvHead->msgType + 1);
-  pReplyHead->tcp = 0;
   pReplyHead->spi = 0;
   pReplyHead->encrypt = 0;
   pReplyHead->tranId = pRecvHead->tranId;
@@ -860,7 +876,7 @@ static void rpcSendErrorMsgToPeer(SRpcInfo *pRpc, char *pMsg, int32_t code, uint
   }
 
   pReplyHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
-  (*taosSendData[pRpc->connType])(ip, port, msg, msgLen, chandle);
+  (*taosSendData[pRecv->connType])(pRecv->ip, pRecv->port, msg, msgLen, pRecv->chandle);
 
   return; 
 }
@@ -872,7 +888,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
   char       msgType = pContext->msgType;
 
   pContext->numOfTry++;
-  SRpcConn *pConn = rpcSetConnToServer(pRpc, pContext->ipSet);
+  SRpcConn *pConn = rpcSetConnToServer(pContext);
   if (pConn == NULL) {
     pContext->code = terrno;
     taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl);
@@ -884,7 +900,6 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
   // set the message header  
   pHead->version = 1;
   pHead->msgType = msgType;
-  pHead->tcp = 0;
   pHead->encrypt = 0;
   pConn->tranId++;
   if ( pConn->tranId == 0 ) pConn->tranId++;
@@ -928,7 +943,7 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
           (uint8_t)pHead->content[0], msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
   }
 
-  writtenLen = (*taosSendData[pRpc->connType])(pConn->peerIp, pConn->peerPort, (char *)pHead, msgLen, pConn->chandle);
+  writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle);
 
   if (writtenLen != msgLen) {
     tError("%s %p, failed to send, dataLen:%d writtenLen:%d, reason:%s", pRpc->label, pConn, 

src/rpc/src/rpcServer.c

@@ -46,10 +46,8 @@ typedef struct _thread_obj {
   int             numOfFds;
   int             threadId;
   char            label[12];
-  // char    buffer[128000];  // buffer to receive data
   void           *shandle;  // handle passed by upper layer during server initialization
-  void *(*processData)(char *data, int dataLen, unsigned int ip, uint16_t port, void *shandle, void *thandle,
-                       void *chandle);
+  void           *(*processData)(SRecvInfo *pPacket);
 } SThreadObj;
 
 typedef struct {
@@ -62,59 +60,81 @@ typedef struct {
   pthread_t   thread;
 } SServerObj;
 
-static void taosCleanUpFdObj(SFdObj *pFdObj) {
+static void taosCleanUpFdObj(SFdObj *pFdObj);
+static void taosProcessTcpData(void *param);
+static void taosAcceptTcpConnection(void *arg);
+
+void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
+  int            i;
+  SServerObj    *pServerObj;
+  pthread_attr_t thattr;
   SThreadObj    *pThreadObj;
 
-  if (pFdObj == NULL) return;
-  if (pFdObj->signature != pFdObj) return;
+  pServerObj = (SServerObj *)malloc(sizeof(SServerObj));
+  strcpy(pServerObj->ip, ip);
+  pServerObj->port = port;
+  strcpy(pServerObj->label, label);
+  pServerObj->numOfThreads = numOfThreads;
 
-  pThreadObj = pFdObj->pThreadObj;
-  if (pThreadObj == NULL) {
-    tError("FdObj double clean up!!!");
-    return;
+  pServerObj->pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj) * (size_t)numOfThreads);
+  if (pServerObj->pThreadObj == NULL) {
+    tError("TCP:%s no enough memory", label);
+    return NULL;
   }
+  memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads);
 
-  epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
-  close(pFdObj->fd);
-
-  pthread_mutex_lock(&pThreadObj->threadMutex);
-
-  pThreadObj->numOfFds--;
-
-  if (pThreadObj->numOfFds < 0)
-    tError("%s TCP thread:%d, number of FDs shall never be negative", pThreadObj->label, pThreadObj->threadId);
-
-  // remove from the FdObject list
+  pThreadObj = pServerObj->pThreadObj;
+  for (i = 0; i < numOfThreads; ++i) {
+    pThreadObj->processData = fp;
+    strcpy(pThreadObj->label, label);
+    pThreadObj->shandle = shandle;
 
-  if (pFdObj->prev) {
-    (pFdObj->prev)->next = pFdObj->next;
-  } else {
-    pThreadObj->pHead = pFdObj->next;
+    if (pthread_mutex_init(&(pThreadObj->threadMutex), NULL) < 0) {
+      tError("%s failed to init TCP process data mutex, reason:%s", label, strerror(errno));
+      return NULL;
     }
 
-  if (pFdObj->next) {
-    (pFdObj->next)->prev = pFdObj->prev;
+    if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) {
+      tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno));
+      return NULL;
     }
 
-  pthread_mutex_unlock(&pThreadObj->threadMutex);
-
-  // notify the upper layer, so it will clean the associated context
-  if (pFdObj->thandle) (*(pThreadObj->processData))(NULL, 0, 0, 0, pThreadObj->shandle, pFdObj->thandle, NULL);
-
-  tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId,
-         pFdObj, pThreadObj->numOfFds);
+    pThreadObj->pollFd = epoll_create(10);  // size does not matter
+    if (pThreadObj->pollFd < 0) {
+      tError("%s failed to create TCP epoll", label);
+      return NULL;
+    }
 
-  memset(pFdObj, 0, sizeof(SFdObj));
+    pthread_attr_init(&thattr);
+    pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+    if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) {
+      tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno));
+      return NULL;
+    }
 
-  tfree(pFdObj);
-}
+    pThreadObj->threadId = i;
+    pThreadObj++;
+  }
 
-void taosCloseTcpServerConnection(void *chandle) {
-  SFdObj *pFdObj = (SFdObj *)chandle;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) {
+    tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno));
+    return NULL;
+  }
 
-  if (pFdObj == NULL) return;
+  /*
+    if ( pthread_create(&(pServerObj->thread), &thattr,
+    (void*)taosAcceptUDConnection, (void *)(pServerObj)) != 0 ) {
+      tError("%s failed to create UD accept thread, reason:%s", label,
+    strerror(errno));
+      return NULL;
+    }
+  */
+  pthread_attr_destroy(&thattr);
+  tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
 
-  taosCleanUpFdObj(pFdObj);
+  return (void *)pServerObj;
 }
 
 void taosCleanUpTcpServer(void *handle) {
@@ -148,6 +168,22 @@ void taosCleanUpTcpServer(void *handle) {
   tfree(pServerObj);
 }
 
+void taosCloseTcpServerConnection(void *chandle) {
+  SFdObj *pFdObj = (SFdObj *)chandle;
+
+  if (pFdObj == NULL) return;
+
+  taosCleanUpFdObj(pFdObj);
+}
+
+int taosSendTcpServerData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
+  SFdObj *pFdObj = (SFdObj *)chandle;
+
+  if (chandle == NULL) return -1;
+
+  return (int)send(pFdObj->fd, data, (size_t)len, 0);
+}
+
 #define maxEvents 10
 
 static void taosProcessTcpData(void *param) {
@@ -155,7 +191,7 @@ static void taosProcessTcpData(void *param) {
   int                i, fdNum;
   SFdObj *           pFdObj;
   struct epoll_event events[maxEvents];
-
+  SRecvInfo         recvInfo;
   pThreadObj = (SThreadObj *)param;
 
   while (1) {
@@ -209,15 +245,22 @@ static void taosProcessTcpData(void *param) {
         continue;
       }
 
-      pFdObj->thandle = (*(pThreadObj->processData))(buffer, dataLen, pFdObj->ip, pFdObj->port,
-                                                     pThreadObj->shandle, pFdObj->thandle, pFdObj);
+      recvInfo.msg = buffer;
+      recvInfo.msgLen = dataLen;
+      recvInfo.ip = pFdObj->ip;
+      recvInfo.port = pFdObj->port;
+      recvInfo.shandle = pThreadObj->shandle;
+      recvInfo.thandle = pFdObj->thandle;;
+      recvInfo.chandle = pFdObj;
+      recvInfo.connType = RPC_CONN_TCP;
 
+      pFdObj->thandle = (*(pThreadObj->processData))(&recvInfo);
       if (pFdObj->thandle == NULL) taosCleanUpFdObj(pFdObj);
     }
   }
 }
 
-void taosAcceptTcpConnection(void *arg) {
+static void taosAcceptTcpConnection(void *arg) {
   int                connFd = -1;
   struct sockaddr_in clientAddr;
   int                sockFd;
@@ -280,16 +323,11 @@ void taosAcceptTcpConnection(void *arg) {
 
     // notify the data process, add into the FdObj list
     pthread_mutex_lock(&(pThreadObj->threadMutex));
-
     pFdObj->next = pThreadObj->pHead;
-
     if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj;
-
     pThreadObj->pHead = pFdObj;
-
     pThreadObj->numOfFds++;
     pthread_cond_signal(&pThreadObj->fdReady);
-
     pthread_mutex_unlock(&(pThreadObj->threadMutex));
 
     tTrace("%s TCP thread:%d, a new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, 
@@ -301,7 +339,65 @@ void taosAcceptTcpConnection(void *arg) {
   }
 }
 
-void taosAcceptUDConnection(void *arg) {
+static void taosCleanUpFdObj(SFdObj *pFdObj) {
+  SThreadObj *pThreadObj;
+
+  if (pFdObj == NULL) return;
+  if (pFdObj->signature != pFdObj) return;
+
+  pThreadObj = pFdObj->pThreadObj;
+  if (pThreadObj == NULL) {
+    tError("FdObj double clean up!!!");
+    return;
+  }
+
+  epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
+  close(pFdObj->fd);
+
+  pthread_mutex_lock(&pThreadObj->threadMutex);
+
+  pThreadObj->numOfFds--;
+
+  if (pThreadObj->numOfFds < 0)
+    tError("%s TCP thread:%d, number of FDs shall never be negative", pThreadObj->label, pThreadObj->threadId);
+
+  // remove from the FdObject list
+
+  if (pFdObj->prev) {
+    (pFdObj->prev)->next = pFdObj->next;
+  } else {
+    pThreadObj->pHead = pFdObj->next;
+  }
+
+  if (pFdObj->next) {
+    (pFdObj->next)->prev = pFdObj->prev;
+  }
+
+  pthread_mutex_unlock(&pThreadObj->threadMutex);
+
+  // notify the upper layer, so it will clean the associated context
+  SRecvInfo recvInfo;
+  recvInfo.msg = NULL;
+  recvInfo.msgLen = 0;
+  recvInfo.ip = 0;
+  recvInfo.port = 0;
+  recvInfo.shandle = pThreadObj->shandle;
+  recvInfo.thandle = pFdObj->thandle;;
+  recvInfo.chandle = NULL;
+  recvInfo.connType = RPC_CONN_TCP;
+
+  if (pFdObj->thandle) (*(pThreadObj->processData))(&recvInfo);
+
+  tTrace("%s TCP thread:%d, FD:%p is cleaned up, numOfFds:%d", pThreadObj->label, pThreadObj->threadId,
+         pFdObj, pThreadObj->numOfFds);
+
+  memset(pFdObj, 0, sizeof(SFdObj));
+
+  tfree(pFdObj);
+}
+
+#if 0
+static void taosAcceptUDConnection(void *arg) {
   int                connFd = -1;
   int                sockFd;
   int                threadId = 0;
@@ -353,16 +449,11 @@ void taosAcceptUDConnection(void *arg) {
 
     // notify the data process, add into the FdObj list
     pthread_mutex_lock(&(pThreadObj->threadMutex));
-
     pFdObj->next = pThreadObj->pHead;
-
     if (pThreadObj->pHead) (pThreadObj->pHead)->prev = pFdObj;
-
     pThreadObj->pHead = pFdObj;
-
     pThreadObj->numOfFds++;
     pthread_cond_signal(&pThreadObj->fdReady);
-
     pthread_mutex_unlock(&(pThreadObj->threadMutex));
 
     tTrace("%s UD thread:%d, a new connection, numOfFds:%d", pServerObj->label, pThreadObj->threadId,
@@ -373,79 +464,7 @@ void taosAcceptUDConnection(void *arg) {
     threadId = threadId % pServerObj->numOfThreads;
   }
 }
-
-void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
-  int            i;
-  SServerObj *   pServerObj;
-  pthread_attr_t thattr;
-  SThreadObj *   pThreadObj;
-
-  pServerObj = (SServerObj *)malloc(sizeof(SServerObj));
-  strcpy(pServerObj->ip, ip);
-  pServerObj->port = port;
-  strcpy(pServerObj->label, label);
-  pServerObj->numOfThreads = numOfThreads;
-
-  pServerObj->pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj) * (size_t)numOfThreads);
-  if (pServerObj->pThreadObj == NULL) {
-    tError("TCP:%s no enough memory", label);
-    return NULL;
-  }
-  memset(pServerObj->pThreadObj, 0, sizeof(SThreadObj) * (size_t)numOfThreads);
-
-  pThreadObj = pServerObj->pThreadObj;
-  for (i = 0; i < numOfThreads; ++i) {
-    pThreadObj->processData = fp;
-    strcpy(pThreadObj->label, label);
-    pThreadObj->shandle = shandle;
-
-    if (pthread_mutex_init(&(pThreadObj->threadMutex), NULL) < 0) {
-      tError("%s failed to init TCP process data mutex, reason:%s", label, strerror(errno));
-      return NULL;
-    }
-
-    if (pthread_cond_init(&(pThreadObj->fdReady), NULL) != 0) {
-      tError("%s init TCP condition variable failed, reason:%s\n", label, strerror(errno));
-      return NULL;
-    }
-
-    pThreadObj->pollFd = epoll_create(10);  // size does not matter
-    if (pThreadObj->pollFd < 0) {
-      tError("%s failed to create TCP epoll", label);
-      return NULL;
-    }
-
-    pthread_attr_init(&thattr);
-    pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
-    if (pthread_create(&(pThreadObj->thread), &thattr, (void *)taosProcessTcpData, (void *)(pThreadObj)) != 0) {
-      tError("%s failed to create TCP process data thread, reason:%s", label, strerror(errno));
-      return NULL;
-    }
-
-    pThreadObj->threadId = i;
-    pThreadObj++;
-  }
-
-  pthread_attr_init(&thattr);
-  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
-  if (pthread_create(&(pServerObj->thread), &thattr, (void *)taosAcceptTcpConnection, (void *)(pServerObj)) != 0) {
-    tError("%s failed to create TCP accept thread, reason:%s", label, strerror(errno));
-    return NULL;
-  }
-
-  /*
-    if ( pthread_create(&(pServerObj->thread), &thattr,
-    (void*)taosAcceptUDConnection, (void *)(pServerObj)) != 0 ) {
-      tError("%s failed to create UD accept thread, reason:%s", label,
-    strerror(errno));
-      return NULL;
-    }
-  */
-  pthread_attr_destroy(&thattr);
-  tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads);
-
-  return (void *)pServerObj;
-}
+#endif
 
 #if 0
 void taosListTcpConnection(void *handle, char *buffer) {
@@ -489,10 +508,4 @@ void taosListTcpConnection(void *handle, char *buffer) {
 }
 #endif
 
-int taosSendTcpServerData(uint32_t ip, uint16_t port, char *data, int len, void *chandle) {
-  SFdObj *pFdObj = (SFdObj *)chandle;
 
-  if (chandle == NULL) return -1;
-
-  return (int)send(pFdObj->fd, data, (size_t)len, 0);
-}

src/rpc/test/rclient.c

@@ -85,7 +85,6 @@ int main(int argc, char *argv[]) {
   int      msgSize = 128;
   int      numOfReqs = 0;
   int      appThreads = 1;
-  char     socketType[20] = "udp";
   char     serverIp[40] = "127.0.0.1";
   struct   timeval systemTime;
   int64_t  startTime, endTime;
@@ -113,9 +112,7 @@ int main(int argc, char *argv[]) {
   rpcInit.ckey         = "key";
 
   for (int i=1; i<argc; ++i) { 
-    if ( strcmp(argv[i], "-c")==0 && i < argc-1 ) {
-      strcpy(socketType, argv[++i]); 
-    } else if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
       ipSet.port = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
       ipSet.ip[0] = inet_addr(argv[++i]); 
@@ -138,7 +135,6 @@ int main(int argc, char *argv[]) {
     } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
     } else {
       printf("\nusage: %s [options] \n", argv[0]);
-      printf("  [-c ctype]: connection type:udp or tpc, default is:%s\n", socketType);
       printf("  [-i ip]: first server IP address, default is:%s\n", serverIp);
       printf("  [-p port]: server port number, default is:%d\n", ipSet.port);
       printf("  [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
@@ -154,7 +150,7 @@ int main(int argc, char *argv[]) {
     }
   }
 
-  rpcInit.connType = strcasecmp(socketType, "udp") == 0 ? TAOS_CONN_UDPC : TAOS_CONN_TCPC;
+  rpcInit.connType = TAOS_CONN_CLIENT;
   taosInitLog("client.log", 100000, 10);
 
   void *pRpc = rpcOpen(&rpcInit);

src/rpc/test/rserver.c

@@ -60,7 +60,6 @@ void processRequestMsg(char type, void *pCont, int contLen, void *thandle, int32
 
 int main(int argc, char *argv[]) {
   SRpcInit rpcInit;
-  char     socketType[20] = "udp";
   char     dataName[20] = "server.data";
 
   memset(&rpcInit, 0, sizeof(rpcInit));
@@ -73,9 +72,7 @@ int main(int argc, char *argv[]) {
   rpcInit.idleTime     = 2000;
 
   for (int i=1; i<argc; ++i) {
-    if ( strcmp(argv[i], "-c")==0 && i < argc-1 ) {
-      strcpy(socketType, argv[++i]);
-    } else if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
       rpcInit.localPort = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-i")==0 && i < argc-1) {
       strcpy(rpcInit.localIp, argv[++i]); 
@@ -93,7 +90,6 @@ int main(int argc, char *argv[]) {
       rpcDebugFlag = atoi(argv[++i]);
     } else {
       printf("\nusage: %s [options] \n", argv[0]);
-      printf("  [-c ctype]: connection type:udp or tcp, default is:%s\n", socketType);
       printf("  [-i ip]: server IP address, default is:%s\n", rpcInit.localIp);
       printf("  [-p port]: server port number, default is:%d\n", rpcInit.localPort);
       printf("  [-t threads]: number of threads, default is:%d\n", rpcInit.numOfThreads);
@@ -107,7 +103,7 @@ int main(int argc, char *argv[]) {
     }
   } 
 
-  rpcInit.connType = strcasecmp(socketType, "udp") == 0 ? TAOS_CONN_UDPS : TAOS_CONN_TCPS;
+  rpcInit.connType = TAOS_CONN_SERVER;
 
   taosInitLog("server.log", 100000, 10);