Merge pull request #4358 from taosdata/patch/td-2135

[TD-2135]<patch>: reduce memory usage

src/client/inc/tsclient.h

@@ -282,7 +282,7 @@ typedef struct {
 
   int8_t       dataSourceType;     // load data from file or not
   int8_t       submitSchema; // submit block is built with table schema
-  STagData     tagData;
+  STagData    *pTagData;     // NOTE: pTagData->data is used as a variant length array
   SHashObj    *pTableList;   // referred table involved in sql
   SArray      *pDataBlocks;  // SArray<STableDataBlocks*> submit data blocks after parsing sql
 } SSqlCmd;

src/client/src/tscParseInsert.c

@@ -790,9 +790,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     sql += index;
 
     tscAllocPayload(pCmd, sizeof(STagData));
-    STagData *pTag = &pCmd->tagData;
-
-    memset(pTag, 0, sizeof(STagData));
     
     //the source super table is moved to the secondary position of the pTableMetaInfo list
     if (pQueryInfo->numOfTables < 2) {
@@ -805,7 +802,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return code;
     }
 
+    STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data));
+    if (pTag == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    memset(pTag, 0, offsetof(STagData, data));
     tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
+    pCmd->pTagData = pTag;
+
     code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -934,7 +938,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     tdSortKVRowByColIdx(row);
-    pTag->dataLen = kvRowLen(row);
+
+    pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row));
+    if (pTag == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    pCmd->pTagData = pTag;
+    pTag->dataLen = htonl(kvRowLen(row));
     kvRowCpy(pTag->data, row);
     free(row);
 
@@ -945,8 +955,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z);
     }
 
-    pTag->dataLen = htonl(pTag->dataLen);
-
     if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
       return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr);
     }

src/client/src/tscServer.c

@@ -1565,11 +1565,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);
 
-  size_t len = htonl(pCmd->tagData.dataLen);
-  if (pSql->cmd.autoCreated) {
+  if (pCmd->autoCreated && pCmd->pTagData != NULL) {
+    int len = htonl(pCmd->pTagData->dataLen);
     if (len > 0) {
-      len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen);
-      memcpy(pInfoMsg->tags, &pCmd->tagData, len);
+      len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen);
+      memcpy(pInfoMsg->tags, pCmd->pTagData, len);
       pMsg += len;
     }
   }
@@ -2239,8 +2239,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->signature = pNew;
   pNew->cmd.command = TSDB_SQL_META;
 
-  registerSqlObj(pNew);
-
   tscAddSubqueryInfo(&pNew->cmd);
 
   SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0);
@@ -2248,8 +2246,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->cmd.autoCreated = pSql->cmd.autoCreated;  // create table if not exists
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) {
     tscError("%p malloc failed for payload to get table meta", pSql);
-    free(pNew);
-
+    tscFreeSqlObj(pNew);
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
@@ -2257,12 +2254,25 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
 
   tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
-  memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData));
+
+  if (pSql->cmd.pTagData != NULL) {
+    int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
+    pNew->cmd.pTagData = calloc(1, size);
+    if (pNew->cmd.pTagData == NULL) {
+      tscError("%p malloc failed for new tag data to get table meta", pSql);
+      tscFreeSqlObj(pNew);
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
+  }
+
   tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
 
   pNew->fp = tscTableMetaCallBack;
   pNew->param = pSql;
 
+  registerSqlObj(pNew);
+
   int32_t code = tscProcessSql(pNew);
   if (code == TSDB_CODE_SUCCESS) {
     code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;  // notify upper application that current process need to be terminated

src/client/src/tscUtil.c

@@ -408,7 +408,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) {
   pCmd->pTableList = NULL;
   
   pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
-  
+
   tscFreeQueryInfo(pCmd, removeFromCache);
 }
 
@@ -512,6 +512,8 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   tscFreeSqlResult(pSql);
   tscResetSqlCmdObj(pCmd, false);
 
+  tfree(pCmd->pTagData);
+  
   memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
   tfree(pCmd->payload);
   pCmd->allocSize = 0;
@@ -1909,7 +1911,17 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
   pCmd->command = cmd;
   pCmd->parseFinished = 1;
   pCmd->autoCreated = pSql->cmd.autoCreated;
-  memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData));
+
+  if (pSql->cmd.pTagData != NULL) {
+    int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
+    pNew->cmd.pTagData = calloc(1, size);
+    if (pNew->cmd.pTagData == NULL) {
+      tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0);
+      free(pNew);
+      return NULL;
+    }
+    memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
+  }
 
   if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);