[td-1747] reduce tableMeta size

src/client/inc/tsclient.h

@@ -93,7 +93,7 @@ typedef struct SCMCorVgroupInfo {
   int32_t    version;
   int8_t     inUse;
   int8_t     numOfEps;
-  SEpAddr epAddr[TSDB_MAX_REPLICA];
+  SEpAddr1   epAddr[TSDB_MAX_REPLICA];
 } SCMCorVgroupInfo;
 
 typedef struct STableMeta {

src/client/src/tscSchemaUtil.c

@@ -145,10 +145,11 @@ static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo
   corVgroupInfo->inUse = 0;
   corVgroupInfo->numOfEps = vgroupInfo->numOfEps;
   for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) {
-    strncpy(corVgroupInfo->epAddr[i].fqdn, vgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN);
+    corVgroupInfo->epAddr[i].fqdn = strdup(vgroupInfo->epAddr[i].fqdn);
     corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port;
   }
 }
+
 STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) {
   assert(pTableMetaMsg != NULL);
   
@@ -164,7 +165,17 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
   
   pTableMeta->id.tid = pTableMetaMsg->sid;
   pTableMeta->id.uid = pTableMetaMsg->uid;
-  pTableMeta->vgroupInfo = pTableMetaMsg->vgroup;
+
+  SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo;
+  pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps;
+  pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId;
+
+  for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) {
+    SEpAddrMsg* pEpMsg = &pTableMetaMsg->vgroup.epAddr[i];
+
+    pVgroupInfo->epAddr[i].fqdn = strndup(pEpMsg->fqdn, tListLen(pEpMsg->fqdn));
+    pVgroupInfo->epAddr[i].port = pEpMsg->port;
+  }
 
   tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo);
 

src/client/src/tscServer.c

@@ -1839,22 +1839,28 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) {
   SSqlCmd* pCmd = &parent->cmd;
   for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) {
     STableMetaInfo *pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i);
-    SVgroupsInfo *  pVgroupInfo = (SVgroupsInfo *)pMsg;
-    pVgroupInfo->numOfVgroups = htonl(pVgroupInfo->numOfVgroups);
 
-    size_t size = sizeof(SCMVgroupInfo) * pVgroupInfo->numOfVgroups + sizeof(SVgroupsInfo);
+    SVgroupsMsg *  pVgroupMsg = (SVgroupsMsg *) pMsg;
+    pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups);
+
+    size_t size = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo);
     pInfo->vgroupList = calloc(1, size);
     assert(pInfo->vgroupList != NULL);
 
-    memcpy(pInfo->vgroupList, pVgroupInfo, size);
+    pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups;
     for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) {
       //just init, no need to lock
       SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j];
-      pVgroups->vgId = htonl(pVgroups->vgId);
+
+      SCMVgroupMsg *vgroupMsg = &pVgroupMsg->vgroups[j];
+      pVgroups->vgId = htonl(vgroupMsg->vgId);
+      pVgroups->numOfEps = vgroupMsg->numOfEps;
+
       assert(pVgroups->numOfEps >= 1);
 
       for (int32_t k = 0; k < pVgroups->numOfEps; ++k) {
-        pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port);
+        pVgroups->epAddr[k].port = htons(vgroupMsg->epAddr[k].port);
+        pVgroups->epAddr[k].fqdn = strndup(vgroupMsg->epAddr[k].fqdn, tListLen(vgroupMsg->epAddr[k].fqdn));
       }
     }
 

src/inc/taosmsg.h

@@ -182,10 +182,16 @@ extern char *taosMsg[];
 
 #pragma pack(push, 1)
 
+// null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta
 typedef struct {
   char     fqdn[TSDB_FQDN_LEN];
   uint16_t port;
-} SEpAddr;
+} SEpAddrMsg;
+
+typedef struct {
+  char*    fqdn;
+  uint16_t port;
+} SEpAddr1;
 
 typedef struct {
   int32_t numOfVnodes;
@@ -664,14 +670,25 @@ typedef struct SCMSTableVgroupMsg {
 typedef struct {
   int32_t       vgId;
   int8_t        numOfEps;
-  SEpAddr   epAddr[TSDB_MAX_REPLICA];
+  SEpAddr1      epAddr[TSDB_MAX_REPLICA];
 } SCMVgroupInfo;
 
+typedef struct {
+  int32_t    vgId;
+  int8_t     numOfEps;
+  SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
+} SCMVgroupMsg;
+
 typedef struct {
   int32_t numOfVgroups;
   SCMVgroupInfo vgroups[];
 } SVgroupsInfo;
 
+typedef struct {
+  int32_t numOfVgroups;
+  SCMVgroupMsg vgroups[];
+} SVgroupsMsg;
+
 typedef struct STableMetaMsg {
   int32_t       contLen;
   char          tableId[TSDB_TABLE_FNAME_LEN];   // table id
@@ -684,7 +701,7 @@ typedef struct STableMetaMsg {
   int16_t       tversion;
   int32_t       sid;
   uint64_t      uid;
-  SCMVgroupInfo vgroup;
+  SCMVgroupMsg  vgroup;
   SSchema       schema[];
 } STableMetaMsg;
 

src/mnode/src/mnodeTable.c

@@ -1472,12 +1472,12 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
   int32_t numOfTable = htonl(pInfo->numOfTables);
 
   // reserve space
-  int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo); 
+  int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsMsg); 
   for (int32_t i = 0; i < numOfTable; ++i) {
     char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i;
     SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
     if (pTable != NULL && pTable->vgHash != NULL) {
-      contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo));
+      contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsMsg));
     } 
     mnodeDecTableRef(pTable);
   }
@@ -1506,12 +1506,12 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
       // even this super table has no corresponding table, still return
       pRsp->numOfTables++;
 
-      SVgroupsInfo *pVgroupInfo = (SVgroupsInfo *)msg;
-      pVgroupInfo->numOfVgroups = 0;
+      SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg;
+      pVgroupMsg->numOfVgroups = 0;
       
-      msg += sizeof(SVgroupsInfo);
+      msg += sizeof(SVgroupsMsg);
     } else {
-      SVgroupsInfo *pVgroupInfo = (SVgroupsInfo *)msg;
+      SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg;
 
       SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash);
       int32_t               vgSize = 0;
@@ -1520,15 +1520,15 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
         SVgObj * pVgroup = mnodeGetVgroup(*pVgId);
         if (pVgroup == NULL) continue;
 
-        pVgroupInfo->vgroups[vgSize].vgId = htonl(pVgroup->vgId);
+        pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId);
         for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) {
           SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode;
           if (pDnode == NULL) break;
 
-          tstrncpy(pVgroupInfo->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN);
-          pVgroupInfo->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort);
+          tstrncpy(pVgroupMsg->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN);
+          pVgroupMsg->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort);
 
-          pVgroupInfo->vgroups[vgSize].numOfEps++;
+          pVgroupMsg->vgroups[vgSize].numOfEps++;
         }
 
         vgSize++;
@@ -1538,10 +1538,10 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
       taosHashDestroyIter(pIter);
       mnodeDecTableRef(pTable);
 
-      pVgroupInfo->numOfVgroups = htonl(vgSize);
+      pVgroupMsg->numOfVgroups = htonl(vgSize);
 
       // one table is done, try the next table
-      msg += sizeof(SVgroupsInfo) + vgSize * sizeof(SCMVgroupInfo);
+      msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SCMVgroupMsg);
       pRsp->numOfTables++;
     }
   }