[TD-1870]<enhance>: optimize the memory consumption.

src/client/inc/tscLocalMerge.h

@@ -64,9 +64,8 @@ typedef struct SLocalReducer {
   SColumnModel *         resColModel;
   tExtMemBuffer **       pExtMemBuffer;      // disk-based buffer
   SFillInfo*             pFillInfo;          // interpolation support structure
-  char *                 pFinalRes;          // result data after interpo
-  tFilePage *            discardData;
-  SResultRowCellInfo *          pResInfo;
+  char*                  pFinalRes;          // result data after interpo
+  tFilePage*             discardData;
   bool                   discard;
   int32_t                offset;             // limit offset value
   bool                   orderPrjOnSTable;   // projection query on stable

src/client/src/tscFunctionImpl.c

@@ -99,7 +99,7 @@ typedef struct SSumInfo {
 // the attribute of hasResult is not needed since the num attribute would server as this purpose
 typedef struct SAvgInfo {
   double  sum;
-  int64_t num;  // num servers as the hasResult attribute in other struct
+  int64_t num;
 } SAvgInfo;
 
 typedef struct SStddevInfo {
@@ -167,7 +167,13 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
       functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) {
     *type = (int16_t)dataType;
     *bytes = (int16_t)dataBytes;
-    *interBytes = 0;//*bytes;
+
+    if (functionId == TSDB_FUNC_INTERP) {
+      *interBytes = sizeof(SInterpInfoDetail);
+    } else {
+      *interBytes = 0;
+    }
+
     return TSDB_CODE_SUCCESS;
   }
   
@@ -175,21 +181,21 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct
     *type = TSDB_DATA_TYPE_BINARY;
     *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE);
-    *interBytes = 0;//*bytes;
+    *interBytes = 0;
     return TSDB_CODE_SUCCESS;
   }
   
   if (functionId == TSDB_FUNC_COUNT) {
     *type = TSDB_DATA_TYPE_BIGINT;
     *bytes = sizeof(int64_t);
-    *interBytes = 0;//*bytes;
+    *interBytes = 0;
     return TSDB_CODE_SUCCESS;
   }
   
   if (functionId == TSDB_FUNC_ARITHM) {
     *type = TSDB_DATA_TYPE_DOUBLE;
     *bytes = sizeof(double);
-    *interBytes = 0;//*bytes;
+    *interBytes = 0;
     return TSDB_CODE_SUCCESS;
   }
   
@@ -334,11 +340,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
   return TSDB_CODE_SUCCESS;
 }
 
-//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf) {
-//  assert(GET_ROWCELL_INTERBUF(pResInfo) == NULL);
-//  GET_ROWCELL_INTERBUF(pResInfo) = buf;
-//}
-
 // set the query flag to denote that query is completed
 static void no_next_step(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
@@ -3998,7 +3999,6 @@ static void interp_function(SQLFunctionCtx *pCtx) {
         }
       }
     }
-    
   }
   
   SET_VAL(pCtx, pCtx->size, 1);

src/client/src/tscLocalMerge.c

@@ -99,11 +99,8 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc
       pCtx->param[1].i64Key = pQueryInfo->order.orderColId;
     }
 
-//    SResultRowCellInfo *pResInfo = &pReducer->pResInfo[i];
-    pCtx->interBufBytes   = pExpr->interBytes;
-//    pResInfo->interResultBuf = calloc(1, (size_t) pCtx->interBufBytes);
-
-    pCtx->resultInfo = &pReducer->pResInfo[i];
+    pCtx->interBufBytes = pExpr->interBytes;
+    pCtx->resultInfo = calloc(1, pCtx->interBufBytes + sizeof(SResultRowCellInfo));
     pCtx->stableQuery = true;
   }
 
@@ -345,7 +342,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo);
   
   pReducer->pTempBuffer->num = 0;
-  pReducer->pResInfo = calloc(numOfCols, sizeof(SResultRowCellInfo));
 
   tscCreateResPointerInfo(pRes, pQueryInfo);
   tscInitSqlContext(pCmd, pReducer, pDesc);
@@ -496,6 +492,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
         SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i];
 
         tVariantDestroy(&pCtx->tag);
+        taosTFree(pCtx->resultInfo);
+
         if (pCtx->tagInfo.pTagCtxList != NULL) {
           taosTFree(pCtx->tagInfo.pTagCtxList);
         }
@@ -509,15 +507,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
     taosTFree(pLocalReducer->pTempBuffer);
     taosTFree(pLocalReducer->pResultBuf);
 
-    if (pLocalReducer->pResInfo != NULL) {
-      size_t num = tscSqlExprNumOfExprs(pQueryInfo);
-      for (int32_t i = 0; i < num; ++i) {
-//        taosTFree(pLocalReducer->pResInfo[i].interResultBuf);
-      }
-
-      taosTFree(pLocalReducer->pResInfo);
-    }
-
     if (pLocalReducer->pLoserTree) {
       taosTFree(pLocalReducer->pLoserTree->param);
       taosTFree(pLocalReducer->pLoserTree);

src/query/inc/qExecutor.h

@@ -168,7 +168,7 @@ typedef struct SQuery {
 
 typedef struct SQueryRuntimeEnv {
   jmp_buf              env;
-  SResultRowCellInfo*  resultInfo;       // todo refactor to merge with SWindowResInfo
+  SResultRow*          pResultRow;       // todo refactor to merge with SWindowResInfo
   SQuery*              pQuery;
   SQLFunctionCtx*      pCtx;
   int32_t              numOfRowsPerPage;
@@ -190,7 +190,7 @@ typedef struct SQueryRuntimeEnv {
   SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
   SHashObj*            pResultRowHashTable; // quick locate the window object for each result
   char*                keyBuf;           // window key buffer
-  SResultRowPool*   pool;             // window result object pool
+  SResultRowPool*      pool;             // window result object pool
 
   int32_t*             rowCellInfoOffset;// offset value for each row result cell info
 } SQueryRuntimeEnv;

src/query/inc/qUtil.h

@@ -42,7 +42,7 @@ void    closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
 void    closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
 void    removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order);
 
-static FORCE_INLINE SResultRow *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) {
+static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) {
   assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size);
   return pWindowResInfo->pResult[slot];
 }
@@ -52,7 +52,7 @@ static FORCE_INLINE SResultRow *getWindowResult(SWindowResInfo *pWindowResInfo,
 
 bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
 
-int32_t createQueryResultInfo(SQuery *pQuery, SResultRow *pResultRow);
+int32_t initResultRow(SResultRow *pResultRow);
 
 static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult,
     tFilePage* page) {

src/query/src/qUtil.c

@@ -87,7 +87,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   assert(num >= 0 && num <= numOfClosed);
 
   int16_t  type = pWindowResInfo->type;
-  uint64_t uid  = 0; // uid is always set to be 0.
+  STableId* id  = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0.
   char    *key  = NULL;
   int16_t  bytes = -1;
 
@@ -104,7 +104,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
         bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
       }
 
-      SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
+      SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
       taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
     } else {
       break;
@@ -137,14 +137,14 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
       bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
     }
 
-    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
+    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
     int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
     assert(p != NULL); 
 
     int32_t  v = (*p - num);
     assert(v >= 0 && v <= pWindowResInfo->size);
 
-    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
+    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
     taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t));
   }
   
@@ -217,11 +217,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
 }
 
 bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) {
-  return (getWindowResult(pWindowResInfo, slot)->closed == true);
+  return (getResultRow(pWindowResInfo, slot)->closed == true);
 }
 
 void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) {
-  getWindowResult(pWindowResInfo, slot)->closed = true;
+  getResultRow(pWindowResInfo, slot)->closed = true;
 }
 
 void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) {
@@ -229,18 +229,21 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) {
     return;
   }
 
-  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
+  // the result does not put into the SDiskbasedResultBuf, ignore it.
+  if (pWindowRes->pageId >= 0) {
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
 
-  for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
-    SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i];
-    
-    char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
-    size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
-    memset(s, 0, size);
-    
-    RESET_RESULT_INFO(pResultInfo);
+    for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
+      SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i];
+
+      char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
+      size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
+      memset(s, 0, size);
+
+      RESET_RESULT_INFO(pResultInfo);
+    }
   }
-  
+
   pWindowRes->numOfRows = 0;
   pWindowRes->pageId = -1;
   pWindowRes->rowId = -1;
@@ -327,6 +330,8 @@ SResultRow* getNewResultRow(SResultRowPool* p) {
   }
 
   p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock;
+  initResultRow(ptr);
+
   return ptr;
 }
 

src/util/src/hash.c

@@ -114,10 +114,12 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
  * @param dsize   size of actual data
  * @return        hash node
  */
-static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
+static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
   assert(pNode->keyLen == pNewNode->keyLen);
   if (prev != NULL) {
     prev->next = pNewNode;
+  } else {
+    pe->next = pNewNode;
   }
 
   pNewNode->next = pNode->next;
@@ -242,7 +244,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
   } else {
     // not support the update operation, return error
     if (pHashObj->enableUpdate) {
-      doUpdateHashNode(prev, pNode, pNewNode);
+      doUpdateHashNode(pe, prev, pNode, pNewNode);
+      DO_FREE_HASH_NODE(pNode);
+    } else {
+      DO_FREE_HASH_NODE(pNewNode);
     }
 
     if (pHashObj->type == HASH_ENTRY_LOCK) {
@@ -252,7 +257,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
     // enable resize
     __rd_unlock(&pHashObj->lock, pHashObj->type);
 
-    DO_FREE_HASH_NODE(pNewNode);
     return pHashObj->enableUpdate ? 0 : -1;
   }
 }