merge 3.0

docs/en/14-reference/04-taosadapter.md

@@ -206,8 +206,8 @@ Note: InfluxDB token authorization is not supported at present. Only Basic autho
 You can use any client that supports the http protocol to access the RESTful interface address `http://<fqdn>:6041/<APIEndPoint>` to write data in OpenTSDB compatible format to TDengine.
 
 ```text
-/opentsdb/v1/put/json/:db
-/opentsdb/v1/put/telnet/:db
+/opentsdb/v1/put/json/<db>
+/opentsdb/v1/put/telnet/<db>
 ```
 
 ### collectd

docs/zh/14-reference/04-taosadapter.md

@@ -207,8 +207,8 @@ AllowWebSockets
 您可以使用任何支持 http 协议的客户端访问 Restful 接口地址 `http://<fqdn>:6041/<APIEndPoint>` 来写入 OpenTSDB 兼容格式的数据到 TDengine。EndPoint 如下：
 
 ```text
-/opentsdb/v1/put/json/:db
-/opentsdb/v1/put/telnet/:db
+/opentsdb/v1/put/json/<db>
+/opentsdb/v1/put/telnet/<db>
 ```
 
 ### collectd

include/libs/function/functionMgt.h

@@ -34,6 +34,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_ELAPSED,
   FUNCTION_TYPE_IRATE,
   FUNCTION_TYPE_LAST_ROW,
+  FUNCTION_TYPE_LAST_ROWT, //TODO: removed
   FUNCTION_TYPE_MAX,
   FUNCTION_TYPE_MIN,
   FUNCTION_TYPE_MODE,

include/libs/wal/wal.h

@@ -210,6 +210,8 @@ void     walCloseRef(SWalRef *);
 int32_t  walRefVer(SWalRef *, int64_t ver);
 int32_t  walUnrefVer(SWal *);
 
+bool walLogExist(SWal *, int64_t ver);
+
 // lifecycle check
 bool    walIsEmpty(SWal *);
 int64_t walGetFirstVer(SWal *);

include/os/osMath.h

@@ -60,6 +60,13 @@ extern "C" {
   })
 #endif
 
+#ifndef __COMPAR_FN_T
+#define __COMPAR_FN_T
+typedef int32_t (*__compar_fn_t)(const void *, const void *);
+#endif
+
+void taosSort(void* arr, int64_t sz, int64_t width, __compar_fn_t compar);
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/tmq.c

@@ -1547,7 +1547,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
 }
 
 int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
-  /*printf("call poll\n");*/
+  /*tscDebug("call poll");*/
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
     for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
@@ -1609,8 +1609,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
 
-      char offsetFormatBuf[50];
-      tFormatOffset(offsetFormatBuf, 50, &pVg->currentOffsetNew);
+      char offsetFormatBuf[80];
+      tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew);
       tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %s, reqId %lu", tmq->consumerId,
                pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
@@ -1708,6 +1708,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
 }
 
 TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
+  /*tscDebug("call poll1");*/
   void*   rspObj;
   int64_t startTime = taosGetTimestampMs();
 

source/common/src/tdatablock.c

@@ -892,7 +892,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
         int64_t p0 = taosGetTimestampUs();
 
         __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
-        qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
+        taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
 
         int64_t p1 = taosGetTimestampUs();
         uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows);
@@ -1218,7 +1218,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
     SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
-    if (pSrc->pData == NULL) {
+    if (pSrc->pData == NULL && (!IS_VAR_DATA_TYPE(pSrc->info.type))) {
       continue;
     }
 

source/common/src/tdataformat.c

@@ -931,9 +931,9 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) {
 
   // sort
   if (isJson) {
-    qsort(pArray->pData, nTag, sizeof(STagVal), tTagValJsonCmprFn);
+    taosSort(pArray->pData, nTag, sizeof(STagVal), tTagValJsonCmprFn);
   } else {
-    qsort(pArray->pData, nTag, sizeof(STagVal), tTagValCmprFn);
+    taosSort(pArray->pData, nTag, sizeof(STagVal), tTagValCmprFn);
   }
 
   // get size

source/common/src/tmsg.c

@@ -5402,9 +5402,9 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
   } else if (pVal->type == TMQ_OFFSET__LOG) {
     snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version);
   } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    snprintf(buf, maxLen, "offset(snapshot data) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
+    snprintf(buf, maxLen, "offset(ss data) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
   } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
-    snprintf(buf, maxLen, "offset(snapshot meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
+    snprintf(buf, maxLen, "offset(ss meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
   } else {
     ASSERT(0);
   }

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -113,7 +113,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
   if (!pDataBlocks) {
     terrno = TSDB_CODE_TSMA_INVALID_PTR;
     smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma));
-    return terrno;
+    return TSDB_CODE_FAILED;
   }
 
   if (taosArrayGetSize(pDataBlocks) <= 0) {
@@ -127,9 +127,9 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
     return TSDB_CODE_FAILED;
   }
 
-  SSmaEnv      *pEnv = SMA_TSMA_ENV(pSma);
-  SSmaStat     *pStat = NULL;
-  STSmaStat    *pItem = NULL;
+  SSmaEnv   *pEnv = SMA_TSMA_ENV(pSma);
+  SSmaStat  *pStat = NULL;
+  STSmaStat *pTsmaStat = NULL;
 
   if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) {
     terrno = TSDB_CODE_TSMA_INVALID_STAT;
@@ -137,32 +137,43 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
   }
 
   tdRefSmaStat(pSma, pStat);
-  pItem = &pStat->tsmaStat;
-
-  ASSERT(pItem);
+  pTsmaStat = SMA_TSMA_STAT(pStat);
 
-  if (!pItem->pTSma) {
+  if (!pTsmaStat->pTSma) {
     STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid);
     if (!pTSma) {
-      terrno = TSDB_CODE_TSMA_NO_INDEX_IN_META;
-      smaWarn("vgId:%d, tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno));
-      return TSDB_CODE_FAILED;
+      smaError("vgId:%d, failed to get STSma while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
+               indexUid, tstrerror(terrno));
+      goto _err;
+    }
+    pTsmaStat->pTSma = pTSma;
+    pTsmaStat->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1);
+    if (!pTsmaStat->pTSchema) {
+      smaError("vgId:%d, failed to get STSchema while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
+               indexUid, tstrerror(terrno));
+      goto _err;
     }
-    pItem->pTSma = pTSma;
-    pItem->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1);
-    ASSERT(pItem->pTSchema);  // TODO
   }
 
-  ASSERT(pItem->pTSma->indexUid == indexUid);
-
-  SSubmitReq *pSubmitReq = NULL;
+  if (pTsmaStat->pTSma->indexUid != indexUid) {
+    terrno = TSDB_CODE_VND_APP_ERROR;
+    smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 "(!=%" PRIi64 ") failed since %s", SMA_VID(pSma), indexUid,
+             pTsmaStat->pTSma->indexUid, tstrerror(terrno));
+    goto _err;
+  }
 
-  pSubmitReq = tdBlockToSubmit((const SArray *)msg, pItem->pTSchema, true, pItem->pTSma->dstTbUid,
-                               pItem->pTSma->dstTbName, pItem->pTSma->dstVgId);
+  SSubmitReq *pSubmitReq = tdBlockToSubmit((const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid,
+                                           pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId);
 
-  ASSERT(pSubmitReq);  // TODO
+  if (!pSubmitReq) {
+    smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
+             indexUid, tstrerror(terrno));
+    goto _err;
+  }
 
-  ASSERT(!strncasecmp("td.tsma.rst.tb", pItem->pTSma->dstTbName, 14));
+#if 0
+   ASSERT(!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14));
+#endif
 
   SRpcMsg submitReqMsg = {
       .msgType = TDMT_VND_SUBMIT,
@@ -170,9 +181,15 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
       .contLen = ntohl(pSubmitReq->length),
   };
 
-  ASSERT(tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) == 0);
+  if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) {
+    smaError("vgId:%d, failed to put SubmitReq msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
+             indexUid, tstrerror(terrno));
+    goto _err;
+  }
 
   tdUnRefSmaStat(pSma, pStat);
-
   return TSDB_CODE_SUCCESS;
+_err:
+  tdUnRefSmaStat(pSma, pStat);
+  return TSDB_CODE_FAILED;
 }
\ No newline at end of file

source/dnode/vnode/src/tq/tq.c

@@ -154,10 +154,10 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
   };
   tmsgSendRsp(&rsp);
 
-  char buf1[50];
-  char buf2[50];
-  tFormatOffset(buf1, 50, &pRsp->reqOffset);
-  tFormatOffset(buf2, 50, &pRsp->rspOffset);
+  char buf1[80];
+  char buf2[80];
+  tFormatOffset(buf1, 80, &pRsp->reqOffset);
+  tFormatOffset(buf2, 80, &pRsp->rspOffset);
   tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s",
           TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
 
@@ -238,8 +238,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   STqOffsetVal fetchOffsetNew;
 
   // 1.find handle
-  char buf[50];
-  tFormatOffset(buf, 50, &reqOffset);
+  char buf[80];
+  tFormatOffset(buf, 80, &reqOffset);
   tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch,
           TD_VID(pTq->pVnode), buf);
 
@@ -276,7 +276,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf);
     } else {
       if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
-        if (pReq->useSnapshot) {
+        if (pReq->useSnapshot && pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
           if (!pHandle->fetchMeta) {
             tqOffsetResetToData(&fetchOffsetNew, 0, 0);
           } else {
@@ -360,7 +360,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
         tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
         SMqMetaRsp metaRsp = {0};
         metaRsp.reqOffset = pReq->reqOffset.version;
-        /*tqOffsetResetToLog(&metaR)*/
         metaRsp.rspOffset = fetchVer;
         metaRsp.resMsgType = pHead->msgType;
         metaRsp.metaRspLen = pHead->bodyLen;
@@ -376,22 +375,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
     taosMemoryFree(pHeadWithCkSum);
   } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    // 1. set uid and ts
-    // 2. get data (rebuild reader if needed)
-    // 3. get new uid and ts
-
-    char formatBuf[50];
-    tFormatOffset(formatBuf, 50, &dataRsp.reqOffset);
-    tqInfo("retrieve using snapshot req offset %s", formatBuf);
+    tqInfo("retrieve using snapshot req offset: uid %ld ts %ld", dataRsp.reqOffset.uid, dataRsp.reqOffset.ts);
     if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
       ASSERT(0);
     }
 
     // 4. send rsp
-    if (dataRsp.blockNum != 0) {
-      if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
-        code = -1;
-      }
+    if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+      code = -1;
     }
   } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) {
     ASSERT(0);

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -266,6 +266,8 @@ int vnodeCommit(SVnode *pVnode) {
     ASSERT(0);
     return -1;
   }
+  
+  pVnode->state.committed = info.state.committed;
 
   // apply the commit (TODO)
   vnodeBufPoolReset(pVnode->onCommit);

source/libs/executor/src/executil.c

@@ -118,7 +118,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int
 
   if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) {
     __compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc;
-    qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn);
+    taosSort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn);
   }
 
   pGroupResInfo->index = 0;

source/libs/executor/src/executorimpl.c

@@ -1348,7 +1348,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
       SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i);
       SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
       // it is a reserved column for scalar function, and no data in this column yet.
-      if (pDst->pData == NULL) {
+      if (pDst->pData == NULL || pSrc->pData == NULL) {
         continue;
       }
 
@@ -2837,12 +2837,10 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
 
     if (pInfo->lastStatus.uid != uid || pInfo->lastStatus.ts != ts) {
       tsdbSetTableId(pInfo->dataReader, uid);
-      SQueryTableDataCond tmpCond = pInfo->cond;
-      tmpCond.twindows[0] = (STimeWindow){
-          .skey = ts,
-          .ekey = INT64_MAX,
-      };
-      tsdbResetReadHandle(pInfo->dataReader, &tmpCond, 0);
+      int64_t oldSkey = pInfo->cond.twindows[0].skey;
+      pInfo->cond.twindows[0].skey = ts;
+      tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
+      pInfo->cond.twindows[0].skey = oldSkey;
       pInfo->scanTimes = 0;
       pInfo->curTWinIdx = 0;
     }

source/libs/executor/src/scanoperator.c

@@ -518,7 +518,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
 
   // if scan table by table
   if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
-    // check status
     while (1) {
       SSDataBlock* result = doTableScanGroup(pOperator);
       if (result) {
@@ -530,7 +529,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
         return NULL;
       }
       STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable);
-      /*pTableInfo->uid */
       tsdbSetTableId(pInfo->dataReader, pTableInfo->uid);
       tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
       pInfo->scanTimes = 0;

source/libs/executor/src/timewindowoperator.c

@@ -1246,6 +1246,9 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS
       pCtx[i].fpSet.init(&pCtx[i], pResInfo);
     }
   }
+  SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
+  setBufPageDirty(bufPage, true);
+  releaseBufPage(pResultBuf, bufPage);
 }
 
 bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId,
@@ -3171,6 +3174,10 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup,
         getSessionTimeWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex);
     step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL);
     ASSERT(isInWindow(pCurWin, tsCols[i], gap));
+    if (pCurWin->pos.pageId == -1) {
+      // window has been closed.
+      continue;
+    }
     doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput);
     if (result) {
       taosArrayPush(result, pCurWin);
@@ -3246,12 +3253,12 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
           setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
                              pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
           compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
-          SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
-          setBufPageDirty(bufPage, true);
-          releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
+          SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
+          releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage);
           continue;
+        } else if (!pChWin->isClosed) {
+          break;
         }
-        break;
       }
     }
     SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
@@ -3265,7 +3272,8 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*);
 SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; }
 SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; }
 
-int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn) {
+int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup,
+    SArray* pClosed, __get_win_info_ fn, bool delete) {
   // Todo(liuyao) save window to tdb
   void** pIte = NULL;
   size_t keyLen = 0;
@@ -3279,10 +3287,18 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra
       if (isCloseWindow(&pSeWin->win, pTwSup)) {
         if (!pSeWin->isClosed) {
           pSeWin->isClosed = true;
-          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
+          if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {
             int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed);
+            if (code != TSDB_CODE_SUCCESS) {
+              return code;
+            }
             pSeWin->isOutput = true;
           }
+          if (delete) {
+            taosArrayRemove(pWins, i);
+            i--;
+            size = taosArrayGetSize(pWins);
+          }
         }
         continue;
       }
@@ -3292,6 +3308,16 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra
   return TSDB_CODE_SUCCESS;
 }
 
+static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete) {
+  int32_t size = taosArrayGetSize(pChildren);
+  for (int32_t i = 0; i < size; i++) {
+    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);
+    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;
+    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);
+    closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete);
+  }
+}
+
 int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) {
   void** pIte = NULL;
   while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
@@ -3339,6 +3365,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
     if (pBlock == NULL) {
       break;
     }
+    printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "Final  Session Recv" : "Single Session Recv");
 
     if (pBlock->info.type == STREAM_CLEAR) {
       SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo));
@@ -3385,7 +3412,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
   // restore the value
   pOperator->status = OP_RES_TO_RETURN;
 
-  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession);
+  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
+      getResWinForSession, pInfo->ignoreCloseWindow);
+  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
   copyUpdateResult(pStUpdated, pUpdated);
   taosHashCleanup(pStUpdated);
 
@@ -3437,10 +3466,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
   } else if (pOperator->status == OP_RES_TO_RETURN) {
     doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
     if (pInfo->pDelRes->info.rows > 0) {
+      printDataBlock(pInfo->pDelRes, "Semi  Session");
       return pInfo->pDelRes;
     }
     doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
-    if (pInfo->binfo.pRes->info.rows == 0) {
+    if (pBInfo->pRes->info.rows == 0) {
       pOperator->status = OP_EXEC_DONE;
       if (pInfo->pUpdateRes->info.rows == 0) {
         // semi interval operator clear disk buffer
@@ -3449,9 +3479,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
       }
       // process the rest of the data
       pOperator->status = OP_OPENED;
+      printDataBlock(pInfo->pUpdateRes, "Semi  Session");
       return pInfo->pUpdateRes;
     }
-    return pInfo->binfo.pRes;
+    printDataBlock(pBInfo->pRes, "Semi  Session");
+    return pBInfo->pRes;
   }
 
   _hash_fn_t     hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
@@ -3495,21 +3527,24 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
   finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated,
                         pSup->rowEntryInfoOffset);
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
-  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+  blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
   doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
   if (pInfo->pDelRes->info.rows > 0) {
+    printDataBlock(pInfo->pDelRes, "Semi  Session");
     return pInfo->pDelRes;
   }
-  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
-  if (pInfo->binfo.pRes->info.rows == 0) {
+  doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
+  if (pBInfo->pRes->info.rows == 0) {
     pOperator->status = OP_EXEC_DONE;
     if (pInfo->pUpdateRes->info.rows == 0) {
       return NULL;
     }
     // process the rest of the data
     pOperator->status = OP_OPENED;
+    printDataBlock(pInfo->pUpdateRes, "Semi  Session");
     return pInfo->pUpdateRes;
   }
+  printDataBlock(pBInfo->pRes, "Semi  Session");
   return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
 }
 
@@ -3867,7 +3902,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
   // restore the value
   pOperator->status = OP_RES_TO_RETURN;
 
-  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState);
+  closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
+      getResWinForState, pInfo->ignoreCloseWindow);
+  closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
   copyUpdateResult(pSeUpdated, pUpdated);
   taosHashCleanup(pSeUpdated);
 

source/libs/function/inc/builtinsimpl.h

@@ -117,6 +117,9 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
 int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
 int32_t getFirstLastInfoSize(int32_t resBytes);
 
+int32_t lastRowFunction(SqlFunctionCtx *pCtx);
+int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+
 bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
 bool getTopBotMergeFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
 bool topBotFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);

source/libs/function/src/builtins.c

@@ -1954,15 +1954,12 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "last_row",
     .type = FUNCTION_TYPE_LAST_ROW,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLast,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
-    .processFunc  = lastFunction,
-    .finalizeFunc = firstLastFinalize,
-    .pPartialFunc = "_last_partial",
-    .pMergeFunc   = "_last_merge",
-    .combineFunc  = lastCombine,
+    .processFunc  = lastRowFunction,
+    .finalizeFunc = lastRowFinalize,
   },
   {
     .name = "_cache_last_row",

source/libs/function/src/builtinsimpl.c

@@ -80,6 +80,7 @@ typedef struct STopBotRes {
 
 typedef struct SFirstLastRes {
   bool    hasResult;
+  bool    isNull;  //used for last_row function only
   int32_t bytes;
   char    buf[];
 } SFirstLastRes;
@@ -2764,6 +2765,113 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
+  int32_t numOfElems = 0;
+
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  SFirstLastRes*       pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
+  SInputColumnInfoData* pInput = &pCtx->input;
+  SColumnInfoData*      pInputCol = pInput->pData[0];
+
+  int32_t type = pInputCol->info.type;
+  int32_t bytes = pInputCol->info.bytes;
+  pInfo->bytes = bytes;
+
+  SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL;
+
+  TSKEY startKey = getRowPTs(pInput->pPTS, 0);
+  TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1);
+
+  int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
+
+  if (blockDataOrder == TSDB_ORDER_ASC) {
+    for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
+      char* data = colDataGetData(pInputCol, i);
+      TSKEY cts = getRowPTs(pInput->pPTS, i);
+      if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) {
+        if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
+          pInfo->isNull = true;
+        } else {
+          pInfo->isNull = false;
+          if (IS_VAR_DATA_TYPE(type)) {
+            bytes = varDataTLen(data);
+            pInfo->bytes = bytes;
+          }
+          memcpy(pInfo->buf + sizeof(TSKEY), data, bytes);
+        }
+        *(TSKEY*)(pInfo->buf) = cts;
+        numOfElems++;
+        //handle selectivity
+        if (pCtx->subsidiaries.num > 0) {
+          STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
+          if (!pInfo->hasResult) {
+            saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          } else {
+            copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          }
+        }
+        pInfo->hasResult = true;
+        //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+        pResInfo->numOfRes = 1;
+      }
+      break;
+    }
+  } else {  // descending order
+    for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+      char* data = colDataGetData(pInputCol, i);
+      TSKEY cts = getRowPTs(pInput->pPTS, i);
+      if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) {
+        if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
+          pInfo->isNull = true;
+        } else {
+          pInfo->isNull = false;
+          if (IS_VAR_DATA_TYPE(type)) {
+            bytes = varDataTLen(data);
+            pInfo->bytes = bytes;
+          }
+          memcpy(pInfo->buf + sizeof(TSKEY), data, bytes);
+        }
+        *(TSKEY*)(pInfo->buf) = cts;
+        numOfElems++;
+        //handle selectivity
+        if (pCtx->subsidiaries.num > 0) {
+          STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
+          if (!pInfo->hasResult) {
+            saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          } else {
+            copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
+          }
+        }
+        pInfo->hasResult = true;
+        pResInfo->numOfRes = 1;
+        //DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+      }
+      break;
+    }
+  }
+
+  SET_VAL(pResInfo, numOfElems, 1);
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  int32_t          slotId = pCtx->pExpr->base.resSchema.slotId;
+  SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+
+  SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo);
+  colDataAppend(pCol, pBlock->info.rows, pRes->buf + sizeof(TSKEY), pRes->isNull);
+  //handle selectivity
+  STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
+  setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
+
+  return pResInfo->numOfRes;
+}
+
+
 bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SDiffInfo);
   return true;

source/libs/function/src/taggfunction.c

@@ -1887,10 +1887,10 @@ static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) {
   // user specify the order of output by sort the result according to timestamp
   if (pCtx->param[1].param.i == PRIMARYKEY_TIMESTAMP_COL_ID) {
     __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn;
-    qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
+    taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
   } else /*if (pCtx->param[1].param.i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ {
     __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn;
-    qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
+    taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
   }
   
   GET_TRUE_DATA_TYPE();

source/libs/function/src/tpercentile.c

@@ -44,7 +44,7 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx)
     offset += (int32_t)(pg->num * pMemBucket->bytes);
   }
 
-  qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
+  taosSort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
   return buffer;
 }
 

source/libs/nodes/src/nodesUtilFuncs.c

@@ -1728,7 +1728,6 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) {
     } else {
       pCxt->hasOtherCol = true;
     }
-    return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END;
   }
   return DEAL_RES_CONTINUE;
 }

source/libs/parser/src/parInsert.c

@@ -790,11 +790,11 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
       pColIdx[i].schemaColIdx = pColList->boundColumns[i];
       pColIdx[i].boundIdx = i;
     }
-    qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar);
+    taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar);
     for (col_id_t i = 0; i < pColList->numOfBound; ++i) {
       pColIdx[i].finalIdx = i;
     }
-    qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
+    taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
   }
 
   if (pColList->numOfCols > pColList->numOfBound) {
@@ -2232,11 +2232,11 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS
       pColIdx[i].schemaColIdx = pColList->boundColumns[i];
       pColIdx[i].boundIdx = i;
     }
-    qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar);
+    taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar);
     for (col_id_t i = 0; i < pColList->numOfBound; ++i) {
       pColIdx[i].finalIdx = i;
     }
-    qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
+    taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
   }
 
   if (pColList->numOfCols > pColList->numOfBound) {

source/libs/parser/src/parInsertData.c

@@ -295,7 +295,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) {
     char* pBlockData = pBlocks->data;
 
     // todo. qsort is unstable, if timestamp is same, should get the last one
-    qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar);
+    taosSort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar);
 
     int32_t i = 0;
     int32_t j = 1;
@@ -365,7 +365,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey
     pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
 
     // todo. qsort is unstable, if timestamp is same, should get the last one
-    qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable);
+    taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable);
 
     pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
     int32_t i = 0;

source/libs/parser/src/parTranslater.c

@@ -787,6 +787,13 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
   if (pVal->placeholderNo > 0 || pVal->isNull) {
     return DEAL_RES_CONTINUE;
   }
+  if (TSDB_DATA_TYPE_NULL == pVal->node.resType.type) {
+    // TODO 
+    //pVal->node.resType = targetDt;
+    pVal->translate = true;
+    pVal->isNull = true;
+    return DEAL_RES_CONTINUE;
+  }  
   if (pVal->isDuration) {
     if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, precision) !=
         TSDB_CODE_SUCCESS) {
@@ -5360,7 +5367,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
       if (code != TSDB_CODE_SUCCESS) {
         goto end;
       }
-    } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) {
+    } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) {
       char*   tmpVal = nodesGetValueFromNode(pVal);
       STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
       if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
@@ -5627,8 +5634,8 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  if (DEAL_RES_ERROR ==
-      translateValueImpl(pCxt, pStmt->pVal, schemaToDataType(pTableMeta->tableInfo.precision, pSchema))) {
+  SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
+  if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt)) {
     return pCxt->errCode;
   }
 
@@ -5637,7 +5644,8 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
   }
 
   pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type);
-  if (pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON) {
+  if (targetDt.type == TSDB_DATA_TYPE_JSON) {
+    pReq->isNull = 0;
     if (pStmt->pVal->literal &&
         strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
       return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal);

source/libs/planner/src/planOptimizer.c

@@ -586,11 +586,160 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p
   return code;
 }
 
-static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) {
-  // todo
+typedef struct SPartAggCondContext {
+  SAggLogicNode* pAgg;
+  bool hasAggFunc;
+} SPartAggCondContext;
+
+static EDealRes partAggCondHasAggFuncImpl(SNode* pNode, void* pContext) {
+  SPartAggCondContext* pCxt = pContext;
+  if (QUERY_NODE_COLUMN == nodeType(pNode)) {
+    SNode* pAggFunc = NULL;
+    FOREACH(pAggFunc, pCxt->pAgg->pAggFuncs) {
+      if (strcmp(((SColumnNode*)pNode)->colName, ((SFunctionNode*)pAggFunc)->node.aliasName) == 0) {
+        pCxt->hasAggFunc = true;
+        return DEAL_RES_END;
+      }
+    }
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static int32_t partitionAggCondHasAggFunc(SAggLogicNode* pAgg, SNode* pCond) {
+  SPartAggCondContext cxt = {.pAgg = pAgg, .hasAggFunc = false};
+  nodesWalkExpr(pCond, partAggCondHasAggFuncImpl, &cxt);
+  return cxt.hasAggFunc;
+}
+
+static int32_t partitionAggCondConj(SAggLogicNode* pAgg, SNode** ppAggFuncCond, SNode** ppGroupKeyCond) {
+  SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pAgg->node.pConditions;
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  SNodeList* pAggFuncConds = NULL;
+  SNodeList* pGroupKeyConds = NULL;
+  SNode* pCond = NULL;
+  FOREACH(pCond, pLogicCond->pParameterList) {
+    if (partitionAggCondHasAggFunc(pAgg, pCond)) {
+      code = nodesListMakeAppend(&pAggFuncConds, nodesCloneNode(pCond));
+    } else {
+      code = nodesListMakeAppend(&pGroupKeyConds, nodesCloneNode(pCond));
+    }
+    if (TSDB_CODE_SUCCESS != code) {
+      break;
+    }
+  }
+
+  SNode* pTempAggFuncCond = NULL;
+  SNode* pTempGroupKeyCond = NULL;
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesMergeConds(&pTempAggFuncCond, &pAggFuncConds);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesMergeConds(&pTempGroupKeyCond, &pGroupKeyConds);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *ppAggFuncCond = pTempAggFuncCond;
+    *ppGroupKeyCond = pTempGroupKeyCond;
+  } else {
+    nodesDestroyList(pAggFuncConds);
+    nodesDestroyList(pGroupKeyConds);
+    nodesDestroyNode(pTempAggFuncCond);
+    nodesDestroyNode(pTempGroupKeyCond);
+  }
+  pAgg->node.pConditions = NULL;
+  return code;
+}
+
+static int32_t partitionAggCond(SAggLogicNode* pAgg, SNode** ppAggFunCond, SNode** ppGroupKeyCond) {
+  SNode* pAggNodeCond = pAgg->node.pConditions;
+  if (QUERY_NODE_LOGIC_CONDITION == nodeType(pAggNodeCond) &&
+      LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pAggNodeCond))->condType) {
+    return partitionAggCondConj(pAgg, ppAggFunCond, ppGroupKeyCond);
+  }
+  if (partitionAggCondHasAggFunc(pAgg, pAggNodeCond)) {
+    *ppAggFunCond = pAggNodeCond;
+  } else {
+    *ppGroupKeyCond = pAggNodeCond;
+  }
+  pAgg->node.pConditions = NULL;
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t pushCondToAggCond(SOptimizeContext* pCxt, SAggLogicNode* pAgg, SNode** pAggFuncCond) {
+  pushDownCondOptAppendCond(&pAgg->node.pConditions, pAggFuncCond);
+  return TSDB_CODE_SUCCESS;
+}
+
+typedef struct SRewriteAggGroupKeyCondContext{
+  SAggLogicNode *pAgg;
+  int32_t errCode;
+} SRewriteAggGroupKeyCondContext;
+
+static EDealRes rewriteAggGroupKeyCondForPushDownImpl(SNode** pNode, void* pContext) {
+  SRewriteAggGroupKeyCondContext* pCxt = pContext;
+  SAggLogicNode* pAgg = pCxt->pAgg;
+  if (QUERY_NODE_COLUMN == nodeType(*pNode)) {
+    SNode* pGroupKey = NULL;
+    FOREACH(pGroupKey, pAgg->pGroupKeys) {
+      SNode* pGroup = NULL;
+      FOREACH(pGroup, ((SGroupingSetNode*)pGroupKey)->pParameterList) {
+        if (0 == strcmp(((SExprNode*)pGroup)->aliasName, ((SColumnNode*)(*pNode))->colName)) {
+          SNode* pExpr = nodesCloneNode(pGroup);
+          if (pExpr == NULL) {
+            pCxt->errCode = terrno;
+            return DEAL_RES_ERROR;
+          }
+          nodesDestroyNode(*pNode);
+          *pNode = pExpr;
+        }
+      }
+    }
+    return DEAL_RES_IGNORE_CHILD;
+  }
+  return DEAL_RES_CONTINUE;
+}
+
+static int32_t rewriteAggGroupKeyCondForPushDown(SOptimizeContext* pCxt, SAggLogicNode* pAgg, SNode* pGroupKeyCond) {
+  SRewriteAggGroupKeyCondContext cxt = {.pAgg = pAgg, .errCode = TSDB_CODE_SUCCESS};
+  nodesRewriteExpr(&pGroupKeyCond, rewriteAggGroupKeyCondForPushDownImpl, &cxt);
+  return cxt.errCode;
+}
+
+static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) {
+  if (NULL == pAgg->node.pConditions ||
+      OPTIMIZE_FLAG_TEST_MASK(pAgg->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) {
+    return TSDB_CODE_SUCCESS;
+  }
+  //TODO: remove it after full implementation of pushing down to child
+  if (1 != LIST_LENGTH(pAgg->node.pChildren) ||
+      QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pAgg->node.pChildren, 0))) {
+    return TSDB_CODE_SUCCESS;
+  }
+  
+  SNode* pAggFuncCond = NULL;
+  SNode* pGroupKeyCond = NULL;
+  int32_t code = partitionAggCond(pAgg, &pAggFuncCond, &pGroupKeyCond);
+  if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncCond) {
+    code = pushCondToAggCond(pCxt, pAgg, &pAggFuncCond);
+  }
+  if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeyCond) {
+    code = rewriteAggGroupKeyCondForPushDown(pCxt, pAgg, pGroupKeyCond);
+  }
+  if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeyCond) {
+    SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0);
+    code = pushDownCondOptPushCondToChild(pCxt, pChild, &pGroupKeyCond);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    OPTIMIZE_FLAG_SET_MASK(pAgg->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE);
+    pCxt->optimized = true;
+  } else {
+    nodesDestroyNode(pGroupKeyCond);
+    nodesDestroyNode(pAggFuncCond);
+  }
+  return code;
+}
+
 static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
   int32_t code = TSDB_CODE_SUCCESS;
   switch (nodeType(pLogicNode)) {
@@ -1707,8 +1856,8 @@ static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) {
 
 static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSelfNode) {
   SLogicNode*              pChild = (SLogicNode*)nodesListGetNode(pSelfNode->pChildren, 0);
-  SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS};
 
+  SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS};
   nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt);
   int32_t code = cxt.errCode;
 

source/libs/scalar/src/filter.c

@@ -2059,7 +2059,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
     }
 
     if (colIdxi > 1) {
-      qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
+      taosSort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
     }
 
     for (uint32_t l = 0; l < colIdxi; ++l) {
@@ -2294,7 +2294,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
     return TSDB_CODE_SUCCESS;
   }
 
-  qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx);
+  taosSort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx);
 
   int32_t pEnd = 0, cStart = 0, cEnd = 0;
   uint32_t pColNum = 0, cColNum = 0; 

source/libs/scalar/src/scalar.c

@@ -287,7 +287,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarC
   int32_t code = 0;
   if (NULL == pParamList) {
     if (ctx->pBlockList) {
-      SSDataBlock *pBlock = taosArrayGet(ctx->pBlockList, 0);
+      SSDataBlock *pBlock = taosArrayGetP(ctx->pBlockList, 0);
       *rowNum = pBlock->info.rows;
     } else {
       *rowNum = 1;

source/libs/transport/src/transCli.c

@@ -607,9 +607,10 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
   if (clear) {
     if (!uv_is_closing((uv_handle_t*)conn->stream)) {
       uv_close((uv_handle_t*)conn->stream, cliDestroy);
-    } else {
-      cliDestroy((uv_handle_t*)conn->stream);
     }
+    //} else {
+    //  cliDestroy((uv_handle_t*)conn->stream);
+    //}
   }
 }
 static void cliDestroy(uv_handle_t* handle) {

source/libs/transport/src/transSvr.c

@@ -139,6 +139,7 @@ static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd);
 static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd);
 static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
                                                                     uvHandleRegister, NULL};
+
 static void uvDestroyConn(uv_handle_t* handle);
 
 // server and worker thread
@@ -775,9 +776,10 @@ static void destroyConn(SSvrConn* conn, bool clear) {
     if (!uv_is_closing((uv_handle_t*)conn->pTcp)) {
       tTrace("conn %p to be destroyed", conn);
       uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn);
-    } else {
-      uvDestroyConn((uv_handle_t*)conn->pTcp);
     }
+    //} else {
+    //  uvDestroyConn((uv_handle_t*)conn->pTcp);
+    //}
   }
 }
 static void destroyConnRegArg(SSvrConn* conn) {

source/libs/wal/src/walMeta.c

@@ -19,6 +19,10 @@
 #include "tref.h"
 #include "walInt.h"
 
+bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) {
+  return !walIsEmpty(pWal) && walGetFirstVer(pWal) <= ver && walGetLastVer(pWal) >= ver;
+}
+
 bool FORCE_INLINE walIsEmpty(SWal* pWal) { return pWal->vers.firstVer == -1; }
 
 int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }

source/os/src/osMath.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define ALLOW_FORBID_FUNC
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifdef WINDOWS
+void swapStr(char* j, char* J, int width) {
+  int  i;
+  char tmp;
+  for (i = 0; i < width; i++) {
+    tmp = *j;
+    *j = *J;
+    *J = tmp;
+    j++;
+    J++;
+  }
+}
+#endif
+
+void taosSort(void* arr, int64_t sz, int64_t width, __compar_fn_t compar) {
+#ifdef WINDOWS
+  int64_t i, j;
+  for (i = 0; i < sz - 1; i++) {
+    for (j = 0; j < sz - 1 - i; j++) {
+      if (compar((char*)arr + j * width, (char*)arr + (j + 1) * width) > 0.00) {
+        swapStr((char*)arr + j * width, (char*)arr + (j + 1) * width, width);
+      }
+    }
+  }
+#else
+  qsort(arr, sz, width, compar);
+#endif
+}

source/util/src/tarray.c

@@ -373,7 +373,7 @@ void taosArraySort(SArray* pArray, __compar_fn_t compar) {
   assert(pArray != NULL);
   assert(compar != NULL);
 
-  qsort(pArray->pData, pArray->size, pArray->elemSize, compar);
+  taosSort(pArray->pData, pArray->size, pArray->elemSize, compar);
 }
 
 void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags) {
@@ -390,7 +390,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t
 
 void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
   assert(pArray != NULL);
-  qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn);
+  taosSort(pArray->pData, pArray->size, pArray->elemSize, comparFn);
 }
 
 char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int32_t flags) {

source/util/src/tdigest.c

@@ -124,7 +124,7 @@ void tdigestCompress(TDigest *t) {
     t->num_buffered_pts = 0;
     t->total_weight += unmerged_weight;
 
-    qsort(unmerged_centroids, num_unmerged, sizeof(SCentroid), cmpCentroid);
+    taosSort(unmerged_centroids, num_unmerged, sizeof(SCentroid), cmpCentroid);
     memset(&args, 0, sizeof(SMergeArgs));
     args.centroids = (SCentroid*)taosMemoryMalloc((size_t)(sizeof(SCentroid) * t->size));
     memset(args.centroids, 0, (size_t)(sizeof(SCentroid) * t->size));

source/util/src/terror.c

@@ -614,7 +614,7 @@ static int32_t taosCompareTaosError(const void* a, const void* b) {
 static TdThreadOnce tsErrorInit = PTHREAD_ONCE_INIT;
 
 static void tsSortError(void) {
-  qsort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError);
+  taosSort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError);
 }
 
 const char* tstrerror(int32_t err) {

source/util/src/tlrucache.c

@@ -16,24 +16,24 @@
 #define _DEFAULT_SOURCE
 #include "tlrucache.h"
 #include "os.h"
-#include "tdef.h"
 #include "taoserror.h"
-#include "tlog.h"
 #include "tarray.h"
+#include "tdef.h"
+#include "tlog.h"
 
-typedef struct SLRUEntry SLRUEntry;
+typedef struct SLRUEntry      SLRUEntry;
 typedef struct SLRUEntryTable SLRUEntryTable;
 typedef struct SLRUCacheShard SLRUCacheShard;
-typedef struct SShardedCache SShardedCache;
+typedef struct SShardedCache  SShardedCache;
 
 enum {
-  TAOS_LRU_IN_CACHE = (1 << 0),         // Whether this entry is referenced by the hash table.
+  TAOS_LRU_IN_CACHE = (1 << 0),  // Whether this entry is referenced by the hash table.
 
-  TAOS_LRU_IS_HIGH_PRI = (1 << 1),      // Whether this entry is high priority entry.
+  TAOS_LRU_IS_HIGH_PRI = (1 << 1),  // Whether this entry is high priority entry.
 
-  TAOS_LRU_IN_HIGH_PRI_POOL = (1 << 2), // Whether this entry is in high-pri pool.
+  TAOS_LRU_IN_HIGH_PRI_POOL = (1 << 2),  // Whether this entry is in high-pri pool.
 
-  TAOS_LRU_HAS_HIT = (1 << 3),          // Whether this entry has had any lookups (hits).
+  TAOS_LRU_HAS_HIT = (1 << 3),  // Whether this entry has had any lookups (hits).
 };
 
 struct SLRUEntry {
@@ -50,18 +50,39 @@ struct SLRUEntry {
   char                keyData[1];
 };
 
-#define TAOS_LRU_ENTRY_IN_CACHE(h) ((h)->flags & TAOS_LRU_IN_CACHE)
+#define TAOS_LRU_ENTRY_IN_CACHE(h)     ((h)->flags & TAOS_LRU_IN_CACHE)
 #define TAOS_LRU_ENTRY_IN_HIGH_POOL(h) ((h)->flags & TAOS_LRU_IN_HIGH_PRI_POOL)
-#define TAOS_LRU_ENTRY_IS_HIGH_PRI(h) ((h)->flags & TAOS_LRU_IS_HIGH_PRI)
-#define TAOS_LRU_ENTRY_HAS_HIT(h) ((h)->flags & TAOS_LRU_HAS_HIT)
-
-#define TAOS_LRU_ENTRY_SET_IN_CACHE(h, inCache)  do { if(inCache) {(h)->flags |= TAOS_LRU_IN_CACHE;} else {(h)->flags &= ~TAOS_LRU_IN_CACHE;} } while(0)
-#define TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(h, inHigh)  do { if(inHigh) {(h)->flags |= TAOS_LRU_IN_HIGH_PRI_POOL;} else {(h)->flags &= ~TAOS_LRU_IN_HIGH_PRI_POOL;} } while(0)
-#define TAOS_LRU_ENTRY_SET_PRIORITY(h, priority)  do { if(priority == TAOS_LRU_PRIORITY_HIGH) {(h)->flags |= TAOS_LRU_IS_HIGH_PRI;} else {(h)->flags &= ~TAOS_LRU_IS_HIGH_PRI;} } while(0)
+#define TAOS_LRU_ENTRY_IS_HIGH_PRI(h)  ((h)->flags & TAOS_LRU_IS_HIGH_PRI)
+#define TAOS_LRU_ENTRY_HAS_HIT(h)      ((h)->flags & TAOS_LRU_HAS_HIT)
+
+#define TAOS_LRU_ENTRY_SET_IN_CACHE(h, inCache) \
+  do {                                          \
+    if (inCache) {                              \
+      (h)->flags |= TAOS_LRU_IN_CACHE;          \
+    } else {                                    \
+      (h)->flags &= ~TAOS_LRU_IN_CACHE;         \
+    }                                           \
+  } while (0)
+#define TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(h, inHigh) \
+  do {                                             \
+    if (inHigh) {                                  \
+      (h)->flags |= TAOS_LRU_IN_HIGH_PRI_POOL;     \
+    } else {                                       \
+      (h)->flags &= ~TAOS_LRU_IN_HIGH_PRI_POOL;    \
+    }                                              \
+  } while (0)
+#define TAOS_LRU_ENTRY_SET_PRIORITY(h, priority) \
+  do {                                           \
+    if (priority == TAOS_LRU_PRIORITY_HIGH) {    \
+      (h)->flags |= TAOS_LRU_IS_HIGH_PRI;        \
+    } else {                                     \
+      (h)->flags &= ~TAOS_LRU_IS_HIGH_PRI;       \
+    }                                            \
+  } while (0)
 #define TAOS_LRU_ENTRY_SET_HIT(h) ((h)->flags |= TAOS_LRU_HAS_HIT)
 
 #define TAOS_LRU_ENTRY_HAS_REFS(h) ((h)->refs > 0)
-#define TAOS_LRU_ENTRY_REF(h) (++(h)->refs)
+#define TAOS_LRU_ENTRY_REF(h)      (++(h)->refs)
 
 static bool taosLRUEntryUnref(SLRUEntry *entry) {
   assert(entry->refs > 0);
@@ -90,7 +111,7 @@ struct SLRUEntryTable {
 
 static int taosLRUEntryTableInit(SLRUEntryTable *table, int maxUpperHashBits) {
   table->lengthBits = 4;
-  table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry*));
+  table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry *));
   if (!table->list) {
     return -1;
   }
@@ -125,7 +146,7 @@ static void taosLRUEntryTableCleanup(SLRUEntryTable *table) {
   taosMemoryFree(table->list);
 }
 
-static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) {
+static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable *table, const void *key, size_t keyLen, uint32_t hash) {
   SLRUEntry **entry = &table->list[hash >> (32 - table->lengthBits)];
   while (*entry && ((*entry)->hash != hash || memcmp(key, (*entry)->keyData, keyLen) != 0)) {
     entry = &(*entry)->nextHash;
@@ -134,7 +155,7 @@ static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable * table, const void *
   return entry;
 }
 
-static void taosLRUEntryTableResize(SLRUEntryTable * table) {
+static void taosLRUEntryTableResize(SLRUEntryTable *table) {
   int lengthBits = table->lengthBits;
   if (lengthBits >= table->maxLengthBits) {
     return;
@@ -144,9 +165,9 @@ static void taosLRUEntryTableResize(SLRUEntryTable * table) {
     return;
   }
 
-  uint32_t oldLength = 1 << lengthBits;
-  int newLengthBits = lengthBits + 1;
-  SLRUEntry **newList = taosMemoryCalloc(1 << newLengthBits, sizeof(SLRUEntry*));
+  uint32_t    oldLength = 1 << lengthBits;
+  int         newLengthBits = lengthBits + 1;
+  SLRUEntry **newList = taosMemoryCalloc(1 << newLengthBits, sizeof(SLRUEntry *));
   if (!newList) {
     return;
   }
@@ -154,8 +175,8 @@ static void taosLRUEntryTableResize(SLRUEntryTable * table) {
   for (uint32_t i = 0; i < oldLength; ++i) {
     SLRUEntry *entry = table->list[i];
     while (entry) {
-      SLRUEntry *next = entry->nextHash;
-      uint32_t hash = entry->hash;
+      SLRUEntry  *next = entry->nextHash;
+      uint32_t    hash = entry->hash;
       SLRUEntry **ptr = &newList[hash >> (32 - newLengthBits)];
       entry->nextHash = *ptr;
       *ptr = entry;
@@ -170,13 +191,13 @@ static void taosLRUEntryTableResize(SLRUEntryTable * table) {
   table->lengthBits = newLengthBits;
 }
 
-static SLRUEntry *taosLRUEntryTableLookup(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) {
+static SLRUEntry *taosLRUEntryTableLookup(SLRUEntryTable *table, const void *key, size_t keyLen, uint32_t hash) {
   return *taosLRUEntryTableFindPtr(table, key, keyLen, hash);
 }
 
-static SLRUEntry *taosLRUEntryTableInsert(SLRUEntryTable * table, SLRUEntry *entry) {
+static SLRUEntry *taosLRUEntryTableInsert(SLRUEntryTable *table, SLRUEntry *entry) {
   SLRUEntry **ptr = taosLRUEntryTableFindPtr(table, entry->keyData, entry->keyLength, entry->hash);
-  SLRUEntry *old = *ptr;
+  SLRUEntry  *old = *ptr;
   entry->nextHash = (old == NULL) ? NULL : old->nextHash;
   *ptr = entry;
   if (old == NULL) {
@@ -189,9 +210,9 @@ static SLRUEntry *taosLRUEntryTableInsert(SLRUEntryTable * table, SLRUEntry *ent
   return old;
 }
 
-static SLRUEntry *taosLRUEntryTableRemove(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) {
+static SLRUEntry *taosLRUEntryTableRemove(SLRUEntryTable *table, const void *key, size_t keyLen, uint32_t hash) {
   SLRUEntry **entry = taosLRUEntryTableFindPtr(table, key, keyLen, hash);
-  SLRUEntry *result = *entry;
+  SLRUEntry  *result = *entry;
   if (result) {
     *entry = result->nextHash;
     --table->elems;
@@ -201,17 +222,17 @@ static SLRUEntry *taosLRUEntryTableRemove(SLRUEntryTable * table, const void *ke
 }
 
 struct SLRUCacheShard {
-  size_t          capacity;
-  size_t          highPriPoolUsage;
-  bool            strictCapacity;
-  double          highPriPoolRatio;
-  double          highPriPoolCapacity;
-  SLRUEntry       lru;
-  SLRUEntry      *lruLowPri;
-  SLRUEntryTable  table;
-  size_t          usage; // Memory size for entries residing in the cache.
-  size_t          lruUsage; // Memory size for entries residing only in the LRU list.
-  TdThreadMutex   mutex;
+  size_t         capacity;
+  size_t         highPriPoolUsage;
+  bool           strictCapacity;
+  double         highPriPoolRatio;
+  double         highPriPoolCapacity;
+  SLRUEntry      lru;
+  SLRUEntry     *lruLowPri;
+  SLRUEntryTable table;
+  size_t         usage;     // Memory size for entries residing in the cache.
+  size_t         lruUsage;  // Memory size for entries residing only in the LRU list.
+  TdThreadMutex  mutex;
 };
 
 #define TAOS_LRU_CACHE_SHARD_HASH32(key, len) (MurmurHash3_32((key), (len)))
@@ -231,8 +252,7 @@ static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) {
   assert(e->next == NULL);
   assert(e->prev == NULL);
 
-  if (shard->highPriPoolRatio > 0
-      && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) {
+  if (shard->highPriPoolRatio > 0 && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) {
     e->next = &shard->lru;
     e->prev = shard->lru.prev;
 
@@ -248,7 +268,7 @@ static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) {
 
     e->prev->next = e;
     e->next->prev = e;
-    
+
     TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(e, false);
     shard->lruLowPri = e;
   }
@@ -304,13 +324,13 @@ static void taosLRUCacheShardSetCapacity(SLRUCacheShard *shard, size_t capacity)
 
   for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) {
     SLRUEntry *entry = taosArrayGetP(lastReferenceList, i);
-    taosLRUEntryFree(entry);    
+    taosLRUEntryFree(entry);
   }
   taosArrayDestroy(lastReferenceList);
 }
 
-static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool strict,
-				  double highPriPoolRatio, int maxUpperHashBits) {
+static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool strict, double highPriPoolRatio,
+                                 int maxUpperHashBits) {
   if (taosLRUEntryTableInit(&shard->table, maxUpperHashBits) < 0) {
     return -1;
   }
@@ -341,23 +361,24 @@ static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) {
   taosLRUEntryTableCleanup(&shard->table);
 }
 
-static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, bool freeOnFail) {
+static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle,
+                                              bool freeOnFail) {
   LRUStatus status = TAOS_LRU_STATUS_OK;
-  SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES);
+  SArray   *lastReferenceList = taosArrayInit(16, POINTER_BYTES);
 
   taosThreadMutexLock(&shard->mutex);
 
   taosLRUCacheShardEvictLRU(shard, e->totalCharge, lastReferenceList);
-  
+
   if (shard->usage + e->totalCharge > shard->capacity && (shard->strictCapacity || handle == NULL)) {
     TAOS_LRU_ENTRY_SET_IN_CACHE(e, false);
     if (handle == NULL) {
       taosArrayPush(lastReferenceList, &e);
     } else {
       if (freeOnFail) {
-	taosMemoryFree(e);
+        taosMemoryFree(e);
 
-	*handle = NULL;
+        *handle = NULL;
       }
 
       status = TAOS_LRU_STATUS_INCOMPLETE;
@@ -371,21 +392,21 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *
       assert(TAOS_LRU_ENTRY_IN_CACHE(old));
       TAOS_LRU_ENTRY_SET_IN_CACHE(old, false);
       if (!TAOS_LRU_ENTRY_HAS_REFS(e)) {
-	taosLRUCacheShardLRURemove(shard, old);
-	assert(shard->usage >= old->totalCharge);
-	shard->usage -= old->totalCharge;
+        taosLRUCacheShardLRURemove(shard, old);
+        assert(shard->usage >= old->totalCharge);
+        shard->usage -= old->totalCharge;
 
-	taosArrayPush(lastReferenceList, &old);
+        taosArrayPush(lastReferenceList, &old);
       }
     }
     if (handle == NULL) {
       taosLRUCacheShardLRUInsert(shard, e);
     } else {
       if (!TAOS_LRU_ENTRY_HAS_REFS(e)) {
-	TAOS_LRU_ENTRY_REF(e);
+        TAOS_LRU_ENTRY_REF(e);
       }
 
-      *handle = (LRUHandle*) e;
+      *handle = (LRUHandle *)e;
     }
   }
 
@@ -394,7 +415,7 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *
   for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) {
     SLRUEntry *entry = taosArrayGetP(lastReferenceList, i);
 
-    taosLRUEntryFree(entry);    
+    taosLRUEntryFree(entry);
   }
   taosArrayDestroy(lastReferenceList);
 
@@ -402,8 +423,8 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *
 }
 
 static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash,
-				   void *value, size_t charge, _taos_lru_deleter_t deleter,
-				   LRUHandle **handle, LRUPriority priority) {
+                                         void *value, size_t charge, _taos_lru_deleter_t deleter, LRUHandle **handle,
+                                         LRUPriority priority) {
   SLRUEntry *e = taosMemoryCalloc(1, sizeof(SLRUEntry) - 1 + keyLen);
   if (!e) {
     return TAOS_LRU_STATUS_FAIL;
@@ -442,7 +463,7 @@ static LRUHandle *taosLRUCacheShardLookup(SLRUCacheShard *shard, const void *key
 
   taosThreadMutexUnlock(&shard->mutex);
 
-  return (LRUHandle *) e;
+  return (LRUHandle *)e;
 }
 
 static void taosLRUCacheShardErase(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash) {
@@ -482,7 +503,7 @@ static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) {
     TAOS_LRU_ENTRY_SET_IN_CACHE(old, false);
     assert(shard->usage >= old->totalCharge);
     shard->usage -= old->totalCharge;
-    
+
     taosArrayPush(lastReferenceList, &old);
   }
 
@@ -491,14 +512,14 @@ static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) {
   for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) {
     SLRUEntry *entry = taosArrayGetP(lastReferenceList, i);
 
-    taosLRUEntryFree(entry);    
+    taosLRUEntryFree(entry);
   }
 
   taosArrayDestroy(lastReferenceList);
 }
 
 static bool taosLRUCacheShardRef(SLRUCacheShard *shard, LRUHandle *handle) {
-  SLRUEntry *e = (SLRUEntry *) handle;
+  SLRUEntry *e = (SLRUEntry *)handle;
   taosThreadMutexLock(&shard->mutex);
 
   assert(TAOS_LRU_ENTRY_HAS_REFS(e));
@@ -514,8 +535,8 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b
     return false;
   }
 
-  SLRUEntry *e = (SLRUEntry *) handle;
-  bool lastReference = false;
+  SLRUEntry *e = (SLRUEntry *)handle;
+  bool       lastReference = false;
 
   taosThreadMutexLock(&shard->mutex);
 
@@ -537,7 +558,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b
     assert(shard->usage >= e->totalCharge);
     shard->usage -= e->totalCharge;
   }
-  
+
   taosThreadMutexUnlock(&shard->mutex);
 
   if (lastReference) {
@@ -549,7 +570,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b
 
 static size_t taosLRUCacheShardGetUsage(SLRUCacheShard *shard) {
   size_t usage = 0;
-  
+
   taosThreadMutexLock(&shard->mutex);
   usage = shard->usage;
   taosThreadMutexUnlock(&shard->mutex);
@@ -559,7 +580,7 @@ static size_t taosLRUCacheShardGetUsage(SLRUCacheShard *shard) {
 
 static size_t taosLRUCacheShardGetPinnedUsage(SLRUCacheShard *shard) {
   size_t usage = 0;
-  
+
   taosThreadMutexLock(&shard->mutex);
 
   assert(shard->usage >= shard->lruUsage);
@@ -579,11 +600,11 @@ static void taosLRUCacheShardSetStrictCapacity(SLRUCacheShard *shard, bool stric
 }
 
 struct SShardedCache {
-  uint32_t        shardMask;
-  TdThreadMutex   capacityMutex;
-  size_t          capacity;
-  bool            strictCapacity;
-  uint64_t        lastId; // atomic var for last id
+  uint32_t      shardMask;
+  TdThreadMutex capacityMutex;
+  size_t        capacity;
+  bool          strictCapacity;
+  uint64_t      lastId;  // atomic var for last id
 };
 
 struct SLRUCache {
@@ -593,7 +614,7 @@ struct SLRUCache {
 };
 
 static int getDefaultCacheShardBits(size_t capacity) {
-  int numShardBits = 0;
+  int    numShardBits = 0;
   size_t minShardSize = 512 * 1024;
   size_t numShards = capacity / minShardSize;
   while (numShards >>= 1) {
@@ -621,7 +642,7 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo
     numShardBits = getDefaultCacheShardBits(capacity);
   }
 
-  int numShards =  1 << numShardBits;
+  int numShards = 1 << numShardBits;
   cache->shards = taosMemoryCalloc(numShards, sizeof(SLRUCacheShard));
   if (!cache->shards) {
     taosMemoryFree(cache);
@@ -629,7 +650,7 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo
     return NULL;
   }
 
-  bool strictCapacity = 1;
+  bool   strictCapacity = 1;
   size_t perShard = (capacity + (numShards - 1)) / numShards;
   for (int i = 0; i < numShards; ++i) {
     taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits);
@@ -653,7 +674,7 @@ void taosLRUCacheCleanup(SLRUCache *cache) {
       int numShards = cache->numShards;
       assert(numShards > 0);
       for (int i = 0; i < numShards; ++i) {
-	taosLRUCacheShardCleanup(&cache->shards[i]);
+        taosLRUCacheShardCleanup(&cache->shards[i]);
       }
       taosMemoryFree(cache->shards);
       cache->shards = 0;
@@ -666,11 +687,12 @@ void taosLRUCacheCleanup(SLRUCache *cache) {
 }
 
 LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge,
-		       _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority) {
+                             _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority) {
   uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen);
   uint32_t shardIndex = hash & cache->shardedCache.shardMask;
 
-  return taosLRUCacheShardInsert(&cache->shards[shardIndex], key, keyLen, hash, value, charge, deleter, handle, priority);
+  return taosLRUCacheShardInsert(&cache->shards[shardIndex], key, keyLen, hash, value, charge, deleter, handle,
+                                 priority);
 }
 
 LRUHandle *taosLRUCacheLookup(SLRUCache *cache, const void *key, size_t keyLen) {
@@ -699,7 +721,7 @@ bool taosLRUCacheRef(SLRUCache *cache, LRUHandle *handle) {
     return false;
   }
 
-  uint32_t hash = ((SLRUEntry *) handle)->hash;
+  uint32_t hash = ((SLRUEntry *)handle)->hash;
   uint32_t shardIndex = hash & cache->shardedCache.shardMask;
 
   return taosLRUCacheShardRef(&cache->shards[shardIndex], handle);
@@ -710,15 +732,13 @@ bool taosLRUCacheRelease(SLRUCache *cache, LRUHandle *handle, bool eraseIfLastRe
     return false;
   }
 
-  uint32_t hash = ((SLRUEntry *) handle)->hash;
+  uint32_t hash = ((SLRUEntry *)handle)->hash;
   uint32_t shardIndex = hash & cache->shardedCache.shardMask;
 
   return taosLRUCacheShardRelease(&cache->shards[shardIndex], handle, eraseIfLastRef);
 }
 
-void* taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle) {
-  return ((SLRUEntry*) handle)->value;
-}
+void *taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle) { return ((SLRUEntry *)handle)->value; }
 
 size_t taosLRUCacheGetUsage(SLRUCache *cache) {
   size_t usage = 0;
@@ -742,7 +762,7 @@ size_t taosLRUCacheGetPinnedUsage(SLRUCache *cache) {
 
 void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity) {
   uint32_t numShards = cache->numShards;
-  size_t perShard = (capacity + (numShards = 1)) / numShards;
+  size_t   perShard = (capacity + (numShards - 1)) / numShards;
 
   taosThreadMutexLock(&cache->shardedCache.capacityMutex);
 
@@ -751,7 +771,7 @@ void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity) {
   }
 
   cache->shardedCache.capacity = capacity;
-  
+
   taosThreadMutexUnlock(&cache->shardedCache.capacityMutex);
 }
 
@@ -777,7 +797,7 @@ void taosLRUCacheSetStrictCapacity(SLRUCache *cache, bool strict) {
   }
 
   cache->shardedCache.strictCapacity = strict;
-  
+
   taosThreadMutexUnlock(&cache->shardedCache.capacityMutex);
 }
 

tests/pytest/util/common.py

@@ -17,6 +17,7 @@ import string
 import requests
 import time
 import socket
+import json
 from .boundary import DataBoundary
 import taos
 from util.log import *
@@ -25,7 +26,7 @@ from util.cases import *
 from util.dnodes import *
 from util.common import *
 
-class TDCom: 
+class TDCom:
     def __init__(self):
         self.sml_type = None
         self.env_setting = None
@@ -206,12 +207,12 @@ class TDCom:
         """
             generate long name
             mode could be numbers/letters/letters_mixed/mixed
-        """    
-        if mode == "numbers": 
+        """
+        if mode == "numbers":
             chars = ''.join(random.choice(string.digits) for i in range(len))
-        elif mode == "letters": 
+        elif mode == "letters":
             chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len))
-        elif mode == "letters_mixed": 
+        elif mode == "letters_mixed":
             chars = ''.join(random.choice(string.ascii_letters.upper() + string.ascii_letters.lower()) for i in range(len))
         else:
             chars = ''.join(random.choice(string.ascii_letters.lower() + string.digits) for i in range(len))
@@ -276,7 +277,7 @@ class TDCom:
         vgroups replica precision strict wal fsync comp cachelast single_stable buffer pagesize pages minrows maxrows duration keep retentions
         '''
         sqlString = f'create database if not exists {dbName} '
-        
+
         dbParams = ""
         if len(kwargs) > 0:
             for param, value in kwargs.items():
@@ -306,7 +307,7 @@ class TDCom:
     #     return
 
     # def create_ctables(self,tsql, dbName,stbName,ctbNum,tagDict):
-    #     tsql.execute("use %s" %dbName)        
+    #     tsql.execute("use %s" %dbName)
     #     tagsValues = ''
     #     for i in range(tagDict['int']):
     #         if i > 0:
@@ -323,7 +324,7 @@ class TDCom:
     #             sql = pre_create
     #     if sql != pre_create:
     #         tsql.execute(sql)
-        
+
     #     tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
     #     return
 
@@ -352,7 +353,7 @@ class TDCom:
     #         tsql.execute(sql)
     #     tdLog.debug("insert data ............ [OK]")
     #     return
-        
+
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
@@ -367,7 +368,7 @@ class TDCom:
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
                     break
-        return buildPath        
+        return buildPath
 
     def getClientCfgPath(self):
         buildPath = self.getBuildPath()
@@ -398,7 +399,7 @@ class TDCom:
         return newTdSql
 
     ################################################################################################################
-    # port from the common.py of new test frame 
+    # port from the common.py of new test frame
     ################################################################################################################
     def gen_default_tag_str(self):
         default_tag_str = ""
@@ -527,13 +528,14 @@ class TDCom:
                             tag_value_list.append(self.gen_random_type_value(tag_elm["type"], "", "", "", ""))
                 else:
                     continue
-        return tag_value_list        
+        return tag_value_list
 
-    def gen_column_value_list(self, column_elm_list, ts_value=None):        
+    def gen_column_value_list(self, column_elm_list, ts_value=None):
         if ts_value is None:
             ts_value = self.genTs()[0]
 
         column_value_list = list()
+        column_value_list.append(ts_value)
         if column_elm_list is None:
             column_value_list = list(map(lambda i: self.gen_random_type_value(i, self.default_varchar_length, self.default_varchar_datatype, self.default_nchar_length, self.default_nchar_datatype), self.full_type_list))
         else:
@@ -550,11 +552,11 @@ class TDCom:
                             column_value_list.append(self.gen_random_type_value(column_elm["type"], "", "", "", ""))
                 else:
                     continue
-        column_value_list = [self.ts_value] + self.column_value_list
+        # column_value_list = [self.ts_value] + self.column_value_list
         return column_value_list
 
     def create_stable(self, tsql, dbname=None, stbname="stb", column_elm_list=None, tag_elm_list=None,
-                     count=1, default_stbname_prefix="stb", **kwargs):        
+                     count=1, default_stbname_prefix="stb", **kwargs):
         colname_prefix = 'c'
         tagname_prefix = 't'
         stbname_index_start_num = 1
@@ -589,20 +591,20 @@ class TDCom:
                 tag_value_str += f'"{tag_value}", '
             else:
                 tag_value_str += f'{tag_value}, '
-        tag_value_str = tag_value_str.rstrip()[:-1] 
-            
+        tag_value_str = tag_value_str.rstrip()[:-1]
+
         if int(count) <= 1:
             create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num}  using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};'
             tsql.execute(create_ctable_sql)
         else:
             for _ in range(count):
                 create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num} using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};'
-                ctbname_index_start_num += 1                
+                ctbname_index_start_num += 1
                 tdLog.info("create ctb sql: %s"%create_ctable_sql)
                 tsql.execute(create_ctable_sql)
-    
+
     def create_table(self, tsql, dbname=None, tbname="ntb", column_elm_list=None, count=1, **kwargs):
-        tbname_index_start_num = 1 
+        tbname_index_start_num = 1
         tbname_prefix="ntb"
 
         tb_params = ""
@@ -632,14 +634,23 @@ class TDCom:
                 column_value_str += f'"{column_value}", '
             else:
                 column_value_str += f'{column_value}, '
-        column_value_str = column_value_str.rstrip()[:-1] 
+        column_value_str = column_value_str.rstrip()[:-1]
         if int(count) <= 1:
             insert_sql = f'insert into {self.tb_name} values ({column_value_str});'
             tsql.execute(insert_sql)
         else:
             for num in range(count):
                 column_value_list = self.gen_column_value_list(column_ele_list, f'{start_ts_value}+{num}s')
-                column_value_str = ", ".join(str(v) for v in column_value_list)
+                # column_value_str = ", ".join(str(v) for v in column_value_list)
+                column_value_str = ''
+                idx = 0
+                for column_value in column_value_list:
+                    if isinstance(column_value, str) and idx != 0:
+                        column_value_str += f'"{column_value}", '
+                    else:
+                        column_value_str += f'{column_value}, '
+                        idx += 1
+                column_value_str = column_value_str.rstrip()[:-1]      
                 insert_sql = f'insert into {dbname}.{tbname} values ({column_value_str});'
                 tsql.execute(insert_sql)
     def getOneRow(self, location, containElm):
@@ -651,4 +662,16 @@ class TDCom:
             return res_list
         else:
             tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}")
+
+
+def is_json(msg):
+    if isinstance(msg, str):
+        try:
+            json.loads(msg)
+            return True
+        except:
+            return False
+    else:
+        return False
+
 tdCom = TDCom()

tests/pytest/util/constant.py

@@ -71,6 +71,34 @@ TAOS_KEYWORDS = [
     "COPY",			"IF",			"NOW",			"STABLES",		"WHERE",
 ]
 
+NUM_FUNC = [
+    "ABS", "ACOS", "ASIN", "ATAN", "CEIL", "COS", "FLOOR", "LOG", "POW", "ROUND", "SIN", "SQRT", "TAN",
+]
+
+STR_FUNC = [
+    "CHAR_LENGTH", "CONCAT", "CONCAT_WS", "LENGTH", "LOWER","LTRIM", "RTRIM", "SUBSTR", "UPPER",
+]
+
+CONVER_FUNC = ["CASR", "TO_ISO8601", "TO_JSON", "TP_UNIXTIMESTAMP"]
+
+SELECT_FUNC = [
+    "APERCENTILE", "BOTTOM", "FIRST", "INTERP", "LAST", "MAX", "MIN", "PERCENTILE", "TAIL", "TOP", "UNIQUE",
+]
+
+AGG_FUNC = [
+    "AVG", "COUNT", "ELAPSED", "LEASTSQUARES", "MODE", "SPREAD", "STDDEV", "SUM", "HYPERLOGLOG", "HISTOGRAM",
+]
+
+TS_FUNC = [
+    "CSUM", "DERIVATIVE", "DIFF", "IRATE", "MAVG", "SAMPLE", "STATECOUNT", "STATEDURATION", "TWA"
+]
+
+SYSINFO_FUCN = [
+    "DATABASE", "CLIENT_VERSION", "SERVER_VERSION", "SERVER_STATUS", "CURRENT_USER", "USER"
+]
+
+
+
 # basic data type boundary
 TINYINT_MAX             = 127
 TINYINT_MIN             = -128
@@ -84,16 +112,16 @@ SMALLINT_MIN            = -32768
 SMALLINT_UN_MAX         = 65535
 SMALLINT_UN_MIN         = 0
 
-INT_MAX                 = 2147483647
-INT_MIN                 = -2147483648
+INT_MAX                 = 2_147_483_647
+INT_MIN                 = -2_147_483_648
 
-INT_UN_MAX              = 4294967295
+INT_UN_MAX              = 4_294_967_295
 INT_UN_MIN              = 0
 
-BIGINT_MAX              = 9223372036854775807
-BIGINT_MIN              = -9223372036854775808
+BIGINT_MAX              = 9_223_372_036_854_775_807
+BIGINT_MIN              = -9_223_372_036_854_775_808
 
-BIGINT_UN_MAX           = 18446744073709551615
+BIGINT_UN_MAX           = 18_446_744_073_709_551_615
 BIGINT_UN_MIN           = 0
 
 FLOAT_MAX               = 3.40E+38
@@ -131,13 +159,13 @@ COL_COUNT_MIN           = 2
 TAG_COL_COUNT_MAX       = 4096
 TAG_COL_COUNT_MIN       = 3
 
-MNODE_SHM_SIZE_MAX      = 2147483647
-MNODE_SHM_SIZE_MIN      = 6292480
-MNODE_SHM_SIZE_DEFAULT  = 6292480
+MNODE_SHM_SIZE_MAX      = 2_147_483_647
+MNODE_SHM_SIZE_MIN      = 6_292_480
+MNODE_SHM_SIZE_DEFAULT  = 6_292_480
 
-VNODE_SHM_SIZE_MAX      = 2147483647
-VNODE_SHM_SIZE_MIN      = 6292480
-VNODE_SHM_SIZE_DEFAULT  = 31458304
+VNODE_SHM_SIZE_MAX      = 2_147_483_647
+VNODE_SHM_SIZE_MIN      = 6_292_480
+VNODE_SHM_SIZE_DEFAULT  = 31_458_304
 
 # time_init
 TIME_MS = 1
@@ -160,6 +188,7 @@ INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US
 # streams and related agg-function
 SMA_INDEX_FUNCTIONS = ["MIN", "MAX"]
 ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"]
+BLOCK_FUNCTIONS = ["SUM", "MIN", "MAX"]
 SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"]
 WATERMARK_MAX = 900000
 WATERMARK_MIN = 0

tests/pytest/util/sql.py

@@ -96,6 +96,15 @@ class TDSql:
             return self.queryResult
         return self.queryRows
 
+    def is_err_sql(self, sql):
+        err_flag = True
+        try:
+            self.cursor.execute(sql)
+        except BaseException:
+            err_flag = False
+
+        return False if err_flag else True
+
     def getVariable(self, search_attr):
         '''
             get variable of search_attr access "show variables"
@@ -249,7 +258,6 @@ class TDSql:
             raise Exception(repr(e))
         return self.queryResult
 
-
     def executeTimes(self, sql, times):
         for i in range(times):
             try:
@@ -336,6 +344,38 @@ class TDSql:
         elif precision == "ns":
             return int(times*1000*1000)
 
+    def get_type(self, col):
+        if self.cursor.istype(col, "BOOL"):
+            return "BOOL"
+        if self.cursor.istype(col, "INT"):
+            return "INT"
+        if self.cursor.istype(col, "BIGINT"):
+            return "BIGINT"
+        if self.cursor.istype(col, "TINYINT"):
+            return "TINYINT"
+        if self.cursor.istype(col, "SMALLINT"):
+            return "SMALLINT"
+        if self.cursor.istype(col, "FLOAT"):
+            return "FLOAT"
+        if self.cursor.istype(col, "DOUBLE"):
+            return "DOUBLE"
+        if self.cursor.istype(col, "BINARY"):
+            return "BINARY"
+        if self.cursor.istype(col, "NCHAR"):
+            return "NCHAR"
+        if self.cursor.istype(col, "TIMESTAMP"):
+            return "TIMESTAMP"
+        if self.cursor.istype(col, "JSON"):
+            return "JSON"
+        if self.cursor.istype(col, "TINYINT UNSIGNED"):
+            return "TINYINT UNSIGNED"
+        if self.cursor.istype(col, "SMALLINT UNSIGNED"):
+            return "SMALLINT UNSIGNED"
+        if self.cursor.istype(col, "INT UNSIGNED"):
+            return "INT UNSIGNED"
+        if self.cursor.istype(col, "BIGINT UNSIGNED"):
+            return "BIGINT UNSIGNED"
+
     def taosdStatus(self, state):
         tdLog.sleep(5)
         pstate = 0

tests/script/tsim/tmq/snapshot.sim

@@ -111,7 +111,7 @@ endi
 
 $consumerId    = 0
 $totalMsgOfStb = $ctbNum * $rowsPerCtb
-$expectmsgcnt  = 1
+$expectmsgcnt  = 1000000
 $expectrowcnt  = 100
 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
@@ -131,9 +131,6 @@ endi
 if $data[0][1] != $consumerId then
   return -1
 endi
-if $data[0][2] != $expectmsgcnt then
-  return -1
-endi
 if $data[0][3] != $expectrowcnt then
   return -1
 endi
@@ -183,7 +180,7 @@ endi
 
 $consumerId    = 0
 $totalMsgOfCtb = $rowsPerCtb
-$expectmsgcnt  = 1
+$expectmsgcnt  = 1000000
 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 print == start consumer to pull msgs from ctb
@@ -254,7 +251,7 @@ endi
 
 $consumerId    = 0
 $totalMsgOfNtb = $rowsPerCtb
-$expectmsgcnt  = $totalMsgOfNtb
+$expectmsgcnt  = 1000000
 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 print == start consumer to pull msgs from ntb

tests/script/tsim/tmq/snapshot1.sim

@@ -80,7 +80,7 @@ $topicList = $topicList . '
 $consumerId    = 0
 $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb
 $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum
-$expectmsgcnt  = $totalMsgOfStb
+$expectmsgcnt  = 1000000
 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 
@@ -168,7 +168,7 @@ $consumerId    = 0
 
 $totalMsgOfOneTopic = $rowsPerCtb
 $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum
-$expectmsgcnt  = $totalMsgOfCtb
+$expectmsgcnt  = 1000000
 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 $topicList = ' . topic_ctb_function
@@ -245,7 +245,7 @@ $topicList = $topicList . '
 $consumerId    = 0
 $totalMsgOfOneTopic = $rowsPerCtb
 $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum
-$expectmsgcnt  = $totalMsgOfNtb
+$expectmsgcnt  = 1000000
 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit )
 
 

tests/system-test/0-others/udfTest.py

@@ -32,7 +32,7 @@ class TDTestCase:
                     buildPath = root[:len(root) - len("/build/bin")]
                     break
         return buildPath
-    
+
     def prepare_udf_so(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
@@ -58,7 +58,7 @@ class TDTestCase:
 
 
     def prepare_data(self):
-        
+
         tdSql.execute("drop database if exists db ")
         tdSql.execute("create database if not exists db  duration 300")
         tdSql.execute("use db")
@@ -68,7 +68,7 @@ class TDTestCase:
         tags (t1 int)
         '''
         )
-        
+
         tdSql.execute(
             '''
             create table t1
@@ -150,7 +150,7 @@ class TDTestCase:
             # create aggregate functions
 
             tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
-            
+
             functions = tdSql.getResult("show functions")
             function_nums = len(functions)
             if function_nums == 2:
@@ -175,14 +175,14 @@ class TDTestCase:
         # create aggregate functions
 
         tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
-        
+
         functions = tdSql.getResult("show functions")
         function_nums = len(functions)
         if function_nums == 2:
             tdLog.info("create two udf functions success ")
-       
+
     def basic_udf_query(self):
-        
+
         # scalar functions
 
         tdSql.execute("use db ")
@@ -256,7 +256,7 @@ class TDTestCase:
         tdSql.checkData(0,1,165.247614504)
         tdSql.checkData(0,2,2551.470164435)
         tdSql.checkData(0,3,2.652476145)
-        
+
         # # bug for crash when query sub table
         tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1")
         tdSql.checkData(0,0,378.215547010)
@@ -281,7 +281,7 @@ class TDTestCase:
         tdSql.error("select udf1(num1) , stddev(num1) from tb;")
         tdSql.error("select udf1(num1) , mode(num1) from tb;")
         tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;")
-        # stable 
+        # stable
         tdSql.error("select udf1(c1) , count(c1) from stb1;")
         tdSql.error("select udf1(c1) , avg(c1) from stb1;")
         tdSql.error("select udf1(c1) , twa(c1) from stb1;")
@@ -302,23 +302,25 @@ class TDTestCase:
         tdSql.query("select ceil(num1) , min(num1) from tb;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(num1) , first(num1) from tb;")
-        
+
         tdSql.query("select abs(num1) , first(num1) from tb;")
-        
+
         tdSql.query("select udf1(num1) , last(num1) from tb;")
-        
+
         tdSql.query("select round(num1) , last(num1) from tb;")
-        
+
         tdSql.query("select udf1(num1) , top(num1,1) from tb;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(num1) , bottom(num1,1) from tb;")
         tdSql.checkRows(1)
-        tdSql.error("select udf1(num1) , last_row(num1) from tb;")
-       
-        tdSql.error("select round(num1) , last_row(num1) from tb;")
-   
-        
-        # stable 
+        tdSql.query("select udf1(num1) , last_row(num1) from tb;")
+        tdSql.checkRows(1)
+
+        tdSql.query("select round(num1) , last_row(num1) from tb;")
+        tdSql.checkRows(1)
+
+
+        # stable
         tdSql.query("select udf1(c1) , max(c1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select abs(c1) , max(c1) from stb1;")
@@ -328,9 +330,9 @@ class TDTestCase:
         tdSql.query("select floor(c1) , min(c1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(c1) , first(c1) from stb1;")
-        
+
         tdSql.query("select udf1(c1) , last(c1) from stb1;")
-        
+
         tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select abs(c1) , top(c1 ,1) from stb1;")
@@ -340,9 +342,11 @@ class TDTestCase:
         tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;")
         tdSql.checkRows(1)
 
-        tdSql.error("select udf1(c1) , last_row(c1) from stb1;")
-        tdSql.error("select ceil(c1) , last_row(c1) from stb1;")
-        
+        tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
+        tdSql.checkRows(1)
+        tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
+        tdSql.checkRows(1)
+
         # regular table with compute functions
 
         tdSql.query("select udf1(num1) , abs(num1) from tb;")
@@ -350,7 +354,7 @@ class TDTestCase:
         tdSql.query("select floor(num1) , abs(num1) from tb;")
         tdSql.checkRows(12)
 
-        # # bug need fix 
+        # # bug need fix
 
         #tdSql.query("select udf1(num1) , csum(num1) from tb;")
         #tdSql.checkRows(9)
@@ -382,8 +386,8 @@ class TDTestCase:
         tdSql.checkData(1,0,88)
         tdSql.checkData(1,1,7)
 
-        # bug fix for crash 
-        # order by udf function result 
+        # bug fix for crash
+        # order by udf function result
         for _ in range(50):
             tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)")
             print(tdSql.queryResult)
@@ -401,7 +405,7 @@ class TDTestCase:
         tdSql.checkData(0,1,88)
         tdSql.checkData(0,2,-99.990000000)
         tdSql.checkData(0,3,88)
-        
+
         tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
         tdSql.checkData(0,0,0)
         tdSql.checkData(0,1,0)
@@ -429,7 +433,7 @@ class TDTestCase:
         tdSql.checkData(0,1,168.819430161)
         tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
 
-        # udf functions with  group by 
+        # udf functions with  group by
         tdSql.query("select udf1(c1) from ct1 group by c1")
         tdSql.checkRows(10)
         tdSql.query("select udf1(c1) from stb1 group by c1")
@@ -452,7 +456,7 @@ class TDTestCase:
         tdSql.query("select udf2(c1) from stb1 group by floor(c1)")
         tdSql.checkRows(11)
 
-        # udf mix with order by  
+        # udf mix with order by
         tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)")
         tdSql.checkRows(11)
 
@@ -481,7 +485,7 @@ class TDTestCase:
         tdSql.checkData(0,1,169.661427555)
 
     def try_query_sql(self):
-        udf1_sqls = [ 
+        udf1_sqls = [
         "select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb" ,
         "select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1" ,
         "select udf1(num1) , max(num1) from tb;" ,
@@ -525,7 +529,7 @@ class TDTestCase:
         "select udf2(c1) from stb1 group by udf1(c1)" ,
         "select udf2(c1) from stb1 group by floor(c1)" ,
         "select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)" ,
-    
+
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
@@ -551,7 +555,7 @@ class TDTestCase:
         for aggregate_sql in udf2_sqls:
             tdSql.error(aggregate_sql)
 
-        # create function without aggregate 
+        # create function without aggregate
 
         tdLog.info(" create function with out aggregate ")
         tdSql.query("drop function udf1 ")
@@ -575,8 +579,8 @@ class TDTestCase:
         tdSql.error(" select test(c1) from stb1 ")
         tdSql.error(" select test(c1,c6), test(c6) from stb1 ")
         tdSql.error(" select test(num1,num2), test(num1) from tb ")
-        
-        
+
+
 
     def loop_kill_udfd(self):
 
@@ -585,7 +589,7 @@ class TDTestCase:
             tdLog.exit("taosd not found!")
         else:
             tdLog.info("taosd found in %s" % buildPath)
-        
+
         cfgPath = buildPath + "/../sim/dnode1/cfg"
         udfdPath = buildPath +'/build/bin/udfd'
 
@@ -596,19 +600,19 @@ class TDTestCase:
             tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
             tdSql.checkData(0,0,169.661427555)
             tdSql.checkData(0,1,169.661427555)
-            # stop udfd cmds 
+            # stop udfd cmds
             get_processID = "ps -ef | grep -w udfd | grep -v grep| grep -v defunct | awk '{print $2}'"
             processID = subprocess.check_output(get_processID, shell=True).decode("utf-8")
             stop_udfd = " kill -9 %s" % processID
             os.system(stop_udfd)
 
             time.sleep(2)
-            
+
             tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
             tdSql.checkData(0,0,169.661427555)
             tdSql.checkData(0,1,169.661427555)
 
-            # # start udfd  cmds 
+            # # start udfd  cmds
             # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &"
             # tdLog.info("start udfd : %s " % start_udfd)
 
@@ -643,11 +647,11 @@ class TDTestCase:
             tdDnodes.stop(1)
             tdDnodes.start(1)
             time.sleep(2)
-            
-            
+
+
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
-        
-        print(" env is ok for all ") 
+
+        print(" env is ok for all ")
         self.prepare_udf_so()
         self.prepare_data()
         self.create_udf_function()
@@ -659,7 +663,7 @@ class TDTestCase:
         time.sleep(2)
         self.basic_udf_query()
         self.test_function_name()
-        
+
 
     def stop(self):
         tdSql.close()

tests/system-test/0-others/udf_create.py

@@ -34,7 +34,7 @@ class TDTestCase:
                     buildPath = root[:len(root) - len("/build/bin")]
                     break
         return buildPath
-    
+
     def prepare_udf_so(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
@@ -60,7 +60,7 @@ class TDTestCase:
 
 
     def prepare_data(self):
-        
+
         tdSql.execute("drop database if exists db ")
         tdSql.execute("create database if not exists db  duration 300")
         tdSql.execute("use db")
@@ -70,7 +70,7 @@ class TDTestCase:
         tags (t1 int)
         '''
         )
-        
+
         tdSql.execute(
             '''
             create table t1
@@ -152,7 +152,7 @@ class TDTestCase:
             # create aggregate functions
 
             tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
-            
+
             functions = tdSql.getResult("show functions")
             function_nums = len(functions)
             if function_nums == 2:
@@ -177,14 +177,14 @@ class TDTestCase:
         # create aggregate functions
 
         tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
-        
+
         functions = tdSql.getResult("show functions")
         function_nums = len(functions)
         if function_nums == 2:
             tdLog.info("create two udf functions success ")
-       
+
     def basic_udf_query(self):
-        
+
         # scalar functions
 
         tdSql.execute("use db ")
@@ -258,7 +258,7 @@ class TDTestCase:
         tdSql.checkData(0,1,165.247614504)
         tdSql.checkData(0,2,2551.470164435)
         tdSql.checkData(0,3,2.652476145)
-        
+
         # # bug for crash when query sub table
         tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1")
         tdSql.checkData(0,0,378.215547010)
@@ -283,7 +283,7 @@ class TDTestCase:
         tdSql.error("select udf1(num1) , stddev(num1) from tb;")
         tdSql.error("select udf1(num1) , mode(num1) from tb;")
         tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;")
-        # stable 
+        # stable
         tdSql.error("select udf1(c1) , count(c1) from stb1;")
         tdSql.error("select udf1(c1) , avg(c1) from stb1;")
         tdSql.error("select udf1(c1) , twa(c1) from stb1;")
@@ -304,23 +304,25 @@ class TDTestCase:
         tdSql.query("select ceil(num1) , min(num1) from tb;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(num1) , first(num1) from tb;")
-        
+
         tdSql.query("select abs(num1) , first(num1) from tb;")
-        
+
         tdSql.query("select udf1(num1) , last(num1) from tb;")
-        
+
         tdSql.query("select round(num1) , last(num1) from tb;")
-        
+
         tdSql.query("select udf1(num1) , top(num1,1) from tb;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(num1) , bottom(num1,1) from tb;")
         tdSql.checkRows(1)
-        tdSql.error("select udf1(num1) , last_row(num1) from tb;")
-       
-        tdSql.error("select round(num1) , last_row(num1) from tb;")
-   
-        
-        # stable 
+        tdSql.query("select udf1(num1) , last_row(num1) from tb;")
+        tdSql.checkRows(1)
+
+        tdSql.query("select round(num1) , last_row(num1) from tb;")
+        tdSql.checkRows(1)
+
+
+        # stable
         tdSql.query("select udf1(c1) , max(c1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select abs(c1) , max(c1) from stb1;")
@@ -330,9 +332,9 @@ class TDTestCase:
         tdSql.query("select floor(c1) , min(c1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(c1) , first(c1) from stb1;")
-        
+
         tdSql.query("select udf1(c1) , last(c1) from stb1;")
-        
+
         tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select abs(c1) , top(c1 ,1) from stb1;")
@@ -342,9 +344,11 @@ class TDTestCase:
         tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;")
         tdSql.checkRows(1)
 
-        tdSql.error("select udf1(c1) , last_row(c1) from stb1;")
-        tdSql.error("select ceil(c1) , last_row(c1) from stb1;")
-        
+        tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
+        tdSql.checkRows(1)
+        tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
+        tdSql.checkRows(1)
+
         # regular table with compute functions
 
         tdSql.query("select udf1(num1) , abs(num1) from tb;")
@@ -352,7 +356,7 @@ class TDTestCase:
         tdSql.query("select floor(num1) , abs(num1) from tb;")
         tdSql.checkRows(12)
 
-        # # bug need fix 
+        # # bug need fix
 
         #tdSql.query("select udf1(num1) , csum(num1) from tb;")
         #tdSql.checkRows(9)
@@ -384,8 +388,8 @@ class TDTestCase:
         tdSql.checkData(1,0,88)
         tdSql.checkData(1,1,7)
 
-        # bug fix for crash 
-        # order by udf function result 
+        # bug fix for crash
+        # order by udf function result
         for _ in range(50):
             tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)")
             print(tdSql.queryResult)
@@ -403,7 +407,7 @@ class TDTestCase:
         tdSql.checkData(0,1,88)
         tdSql.checkData(0,2,-99.990000000)
         tdSql.checkData(0,3,88)
-        
+
         tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
         tdSql.checkData(0,0,0)
         tdSql.checkData(0,1,0)
@@ -431,7 +435,7 @@ class TDTestCase:
         tdSql.checkData(0,1,168.819430161)
         tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
 
-        # udf functions with  group by 
+        # udf functions with  group by
         tdSql.query("select udf1(c1) from ct1 group by c1")
         tdSql.checkRows(10)
         tdSql.query("select udf1(c1) from stb1 group by c1")
@@ -454,7 +458,7 @@ class TDTestCase:
         tdSql.query("select udf2(c1) from stb1 group by floor(c1)")
         tdSql.checkRows(11)
 
-        # udf mix with order by  
+        # udf mix with order by
         tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)")
         tdSql.checkRows(11)
 
@@ -483,7 +487,7 @@ class TDTestCase:
         tdSql.checkData(0,1,169.661427555)
 
     def try_query_sql(self):
-        udf1_sqls = [ 
+        udf1_sqls = [
         "select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb" ,
         "select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1" ,
         "select udf1(num1) , max(num1) from tb;" ,
@@ -527,7 +531,7 @@ class TDTestCase:
         "select udf2(c1) from stb1 group by udf1(c1)" ,
         "select udf2(c1) from stb1 group by floor(c1)" ,
         "select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)" ,
-    
+
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
@@ -563,7 +567,7 @@ class TDTestCase:
         for aggregate_sql in udf2_sqls:
             tdSql.error(aggregate_sql)
 
-        # create function without aggregate 
+        # create function without aggregate
 
         tdLog.info(" create function with out aggregate ")
         tdSql.query("drop function udf1 ")
@@ -587,8 +591,8 @@ class TDTestCase:
         tdSql.error(" select test(c1) from stb1 ")
         tdSql.error(" select test(c1,c6), test(c6) from stb1 ")
         tdSql.error(" select test(num1,num2), test(num1) from tb ")
-        
-        
+
+
 
     def loop_kill_udfd(self):
 
@@ -597,7 +601,7 @@ class TDTestCase:
             tdLog.exit("taosd not found!")
         else:
             tdLog.info("taosd found in %s" % buildPath)
-        
+
         cfgPath = buildPath + "/../sim/dnode1/cfg"
         udfdPath = buildPath +'/build/bin/udfd'
 
@@ -608,19 +612,19 @@ class TDTestCase:
             tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
             tdSql.checkData(0,0,169.661427555)
             tdSql.checkData(0,1,169.661427555)
-            # stop udfd cmds 
+            # stop udfd cmds
             get_processID = "ps -ef | grep -w udfd | grep -v grep| grep -v defunct | awk '{print $2}'"
             processID = subprocess.check_output(get_processID, shell=True).decode("utf-8")
             stop_udfd = " kill -9 %s" % processID
             os.system(stop_udfd)
 
             time.sleep(2)
-            
+
             tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
             tdSql.checkData(0,0,169.661427555)
             tdSql.checkData(0,1,169.661427555)
 
-            # # start udfd  cmds 
+            # # start udfd  cmds
             # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &"
             # tdLog.info("start udfd : %s " % start_udfd)
 
@@ -655,17 +659,17 @@ class TDTestCase:
             tdDnodes.stop(1)
             tdDnodes.start(1)
             time.sleep(2)
-            
-            
+
+
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
-        
-        print(" env is ok for all ") 
+
+        print(" env is ok for all ")
         self.prepare_udf_so()
         self.prepare_data()
         self.create_udf_function()
         self.basic_udf_query()
         self.unexpected_create()
-        
+
 
     def stop(self):
         tdSql.close()

tests/system-test/0-others/udf_restart_taosd.py

@@ -31,7 +31,7 @@ class TDTestCase:
                     buildPath = root[:len(root) - len("/build/bin")]
                     break
         return buildPath
-    
+
     def prepare_udf_so(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
@@ -57,7 +57,7 @@ class TDTestCase:
 
 
     def prepare_data(self):
-        
+
         tdSql.execute("drop database if exists db ")
         tdSql.execute("create database if not exists db  duration 300")
         tdSql.execute("use db")
@@ -67,7 +67,7 @@ class TDTestCase:
         tags (t1 int)
         '''
         )
-        
+
         tdSql.execute(
             '''
             create table t1
@@ -149,7 +149,7 @@ class TDTestCase:
             # create aggregate functions
 
             tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
-            
+
             functions = tdSql.getResult("show functions")
             function_nums = len(functions)
             if function_nums == 2:
@@ -174,14 +174,14 @@ class TDTestCase:
         # create aggregate functions
 
         tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2)
-        
+
         functions = tdSql.getResult("show functions")
         function_nums = len(functions)
         if function_nums == 2:
             tdLog.info("create two udf functions success ")
-       
+
     def basic_udf_query(self):
-        
+
         # scalar functions
 
         tdSql.execute("use db ")
@@ -255,7 +255,7 @@ class TDTestCase:
         tdSql.checkData(0,1,165.247614504)
         tdSql.checkData(0,2,2551.470164435)
         tdSql.checkData(0,3,2.652476145)
-        
+
         # # bug for crash when query sub table
         tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1")
         tdSql.checkData(0,0,378.215547010)
@@ -280,7 +280,7 @@ class TDTestCase:
         tdSql.error("select udf1(num1) , stddev(num1) from tb;")
         tdSql.error("select udf1(num1) , mode(num1) from tb;")
         tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;")
-        # stable 
+        # stable
         tdSql.error("select udf1(c1) , count(c1) from stb1;")
         tdSql.error("select udf1(c1) , avg(c1) from stb1;")
         tdSql.error("select udf1(c1) , twa(c1) from stb1;")
@@ -301,23 +301,25 @@ class TDTestCase:
         tdSql.query("select ceil(num1) , min(num1) from tb;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(num1) , first(num1) from tb;")
-        
+
         tdSql.query("select abs(num1) , first(num1) from tb;")
-        
+
         tdSql.query("select udf1(num1) , last(num1) from tb;")
-        
+
         tdSql.query("select round(num1) , last(num1) from tb;")
-        
+
         tdSql.query("select udf1(num1) , top(num1,1) from tb;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(num1) , bottom(num1,1) from tb;")
         tdSql.checkRows(1)
-        tdSql.error("select udf1(num1) , last_row(num1) from tb;")
-       
-        tdSql.error("select round(num1) , last_row(num1) from tb;")
-   
-        
-        # stable 
+        tdSql.query("select udf1(num1) , last_row(num1) from tb;")
+        tdSql.checkRows(1)
+
+        tdSql.query("select round(num1) , last_row(num1) from tb;")
+        tdSql.checkRows(1)
+
+
+        # stable
         tdSql.query("select udf1(c1) , max(c1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select abs(c1) , max(c1) from stb1;")
@@ -327,9 +329,9 @@ class TDTestCase:
         tdSql.query("select floor(c1) , min(c1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select udf1(c1) , first(c1) from stb1;")
-        
+
         tdSql.query("select udf1(c1) , last(c1) from stb1;")
-        
+
         tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;")
         tdSql.checkRows(1)
         tdSql.query("select abs(c1) , top(c1 ,1) from stb1;")
@@ -339,9 +341,11 @@ class TDTestCase:
         tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;")
         tdSql.checkRows(1)
 
-        tdSql.error("select udf1(c1) , last_row(c1) from stb1;")
-        tdSql.error("select ceil(c1) , last_row(c1) from stb1;")
-        
+        tdSql.query("select udf1(c1) , last_row(c1) from stb1;")
+        tdSql.checkRows(1)
+        tdSql.query("select ceil(c1) , last_row(c1) from stb1;")
+        tdSql.checkRows(1)
+
         # regular table with compute functions
 
         tdSql.query("select udf1(num1) , abs(num1) from tb;")
@@ -349,7 +353,7 @@ class TDTestCase:
         tdSql.query("select floor(num1) , abs(num1) from tb;")
         tdSql.checkRows(12)
 
-        # # bug need fix 
+        # # bug need fix
 
         #tdSql.query("select udf1(num1) , csum(num1) from tb;")
         #tdSql.checkRows(9)
@@ -381,8 +385,8 @@ class TDTestCase:
         tdSql.checkData(1,0,88)
         tdSql.checkData(1,1,7)
 
-        # bug fix for crash 
-        # order by udf function result 
+        # bug fix for crash
+        # order by udf function result
         for _ in range(50):
             tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)")
             print(tdSql.queryResult)
@@ -400,7 +404,7 @@ class TDTestCase:
         tdSql.checkData(0,1,88)
         tdSql.checkData(0,2,-99.990000000)
         tdSql.checkData(0,3,88)
-        
+
         tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
         tdSql.checkData(0,0,0)
         tdSql.checkData(0,1,0)
@@ -428,7 +432,7 @@ class TDTestCase:
         tdSql.checkData(0,1,168.819430161)
         tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
 
-        # udf functions with  group by 
+        # udf functions with  group by
         tdSql.query("select udf1(c1) from ct1 group by c1")
         tdSql.checkRows(10)
         tdSql.query("select udf1(c1) from stb1 group by c1")
@@ -451,7 +455,7 @@ class TDTestCase:
         tdSql.query("select udf2(c1) from stb1 group by floor(c1)")
         tdSql.checkRows(11)
 
-        # udf mix with order by  
+        # udf mix with order by
         tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)")
         tdSql.checkRows(11)
 
@@ -480,7 +484,7 @@ class TDTestCase:
         tdSql.checkData(0,1,169.661427555)
 
     def try_query_sql(self):
-        udf1_sqls = [ 
+        udf1_sqls = [
         "select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb" ,
         "select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1" ,
         "select udf1(num1) , max(num1) from tb;" ,
@@ -507,7 +511,7 @@ class TDTestCase:
         "select c1,c2, udf1(c1,c2) from stb1 group by c1,c2" ,
         "select num1,num2,num3,udf1(num1,num2,num3) from tb" ,
         "select c1,c6,udf1(c1,c6) from stb1 order by ts" ,
-        "select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null  order by ts;" 
+        "select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null  order by ts;"
         ]
         udf2_sqls = ["select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
         "select udf2(c1) from stb1 group by 1-udf1(c1)" ,
@@ -524,7 +528,7 @@ class TDTestCase:
         "select udf2(c1) from stb1 group by udf1(c1)" ,
         "select udf2(c1) from stb1 group by floor(c1)" ,
         "select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)" ,
-    
+
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
         "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" ,
@@ -550,7 +554,7 @@ class TDTestCase:
         for aggregate_sql in udf2_sqls:
             tdSql.error(aggregate_sql)
 
-        # create function without aggregate 
+        # create function without aggregate
 
         tdLog.info(" create function with out aggregate ")
         tdSql.query("drop function udf1 ")
@@ -574,8 +578,8 @@ class TDTestCase:
         tdSql.error(" select test(c1) from stb1 ")
         tdSql.error(" select test(c1,c6), test(c6) from stb1 ")
         tdSql.error(" select test(num1,num2), test(num1) from tb ")
-        
-        
+
+
 
     def loop_kill_udfd(self):
 
@@ -584,7 +588,7 @@ class TDTestCase:
             tdLog.exit("taosd not found!")
         else:
             tdLog.info("taosd found in %s" % buildPath)
-        
+
         cfgPath = buildPath + "/../sim/dnode1/cfg"
         udfdPath = buildPath +'/build/bin/udfd'
 
@@ -595,19 +599,19 @@ class TDTestCase:
             tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
             tdSql.checkData(0,0,169.661427555)
             tdSql.checkData(0,1,169.661427555)
-            # stop udfd cmds 
+            # stop udfd cmds
             get_processID = "ps -ef | grep -w udfd | grep -v grep| grep -v defunct | awk '{print $2}'"
             processID = subprocess.check_output(get_processID, shell=True).decode("utf-8")
             stop_udfd = " kill -9 %s" % processID
             os.system(stop_udfd)
 
             time.sleep(2)
-            
+
             tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null")
             tdSql.checkData(0,0,169.661427555)
             tdSql.checkData(0,1,169.661427555)
 
-            # # start udfd  cmds 
+            # # start udfd  cmds
             # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &"
             # tdLog.info("start udfd : %s " % start_udfd)
 
@@ -640,19 +644,19 @@ class TDTestCase:
             tdDnodes.stop(1)
             tdDnodes.start(1)
             time.sleep(2)
-            
-            
+
+
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
-        
-        print(" env is ok for all ") 
+
+        print(" env is ok for all ")
         self.prepare_udf_so()
         self.prepare_data()
         self.create_udf_function()
         self.basic_udf_query()
         self.multi_cols_udf()
         self.restart_taosd_query_udf()
-       
-        
+
+
 
     def stop(self):
         tdSql.close()

tests/system-test/2-query/explain.py

@@ -163,38 +163,6 @@ class TDTestCase:
         # return filter(None, sqls)
         return list(filter(None, sqls))
 
-    def __get_type(self, col):
-        if tdSql.cursor.istype(col, "BOOL"):
-            return "BOOL"
-        if tdSql.cursor.istype(col, "INT"):
-            return "INT"
-        if tdSql.cursor.istype(col, "BIGINT"):
-            return "BIGINT"
-        if tdSql.cursor.istype(col, "TINYINT"):
-            return "TINYINT"
-        if tdSql.cursor.istype(col, "SMALLINT"):
-            return "SMALLINT"
-        if tdSql.cursor.istype(col, "FLOAT"):
-            return "FLOAT"
-        if tdSql.cursor.istype(col, "DOUBLE"):
-            return "DOUBLE"
-        if tdSql.cursor.istype(col, "BINARY"):
-            return "BINARY"
-        if tdSql.cursor.istype(col, "NCHAR"):
-            return "NCHAR"
-        if tdSql.cursor.istype(col, "TIMESTAMP"):
-            return "TIMESTAMP"
-        if tdSql.cursor.istype(col, "JSON"):
-            return "JSON"
-        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
-            return "TINYINT UNSIGNED"
-        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
-            return "SMALLINT UNSIGNED"
-        if tdSql.cursor.istype(col, "INT UNSIGNED"):
-            return "INT UNSIGNED"
-        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
-            return "BIGINT UNSIGNED"
-
     def explain_check(self):
         sqls = self.sql_list()
         tdLog.printNoPrefix("===step 1: curent case, must return query OK")

tests/system-test/2-query/hyperloglog.py

@@ -116,37 +116,6 @@ class TDTestCase:
         # return filter(None, sqls)
         return list(filter(None, sqls))
 
-    def __get_type(self, col):
-        if tdSql.cursor.istype(col, "BOOL"):
-            return "BOOL"
-        if tdSql.cursor.istype(col, "INT"):
-            return "INT"
-        if tdSql.cursor.istype(col, "BIGINT"):
-            return "BIGINT"
-        if tdSql.cursor.istype(col, "TINYINT"):
-            return "TINYINT"
-        if tdSql.cursor.istype(col, "SMALLINT"):
-            return "SMALLINT"
-        if tdSql.cursor.istype(col, "FLOAT"):
-            return "FLOAT"
-        if tdSql.cursor.istype(col, "DOUBLE"):
-            return "DOUBLE"
-        if tdSql.cursor.istype(col, "BINARY"):
-            return "BINARY"
-        if tdSql.cursor.istype(col, "NCHAR"):
-            return "NCHAR"
-        if tdSql.cursor.istype(col, "TIMESTAMP"):
-            return "TIMESTAMP"
-        if tdSql.cursor.istype(col, "JSON"):
-            return "JSON"
-        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
-            return "TINYINT UNSIGNED"
-        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
-            return "SMALLINT UNSIGNED"
-        if tdSql.cursor.istype(col, "INT UNSIGNED"):
-            return "INT UNSIGNED"
-        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
-            return "BIGINT UNSIGNED"
 
     def hyperloglog_check(self):
         sqls = self.sql_list()
@@ -228,10 +197,10 @@ class TDTestCase:
             tag_sql += f"{k} {v},"
         tdSql.execute(f'create table if not exists {stbname} ({column_sql[:-1]}) tags({tag_sql[:-1]})')
 
-    def __insert_data(self): 
+    def __insert_data(self):
 
         pass
-    
+
     def __hyperloglog_check_distribute(self):
         dbname = "dbtest"
         stbname = "stb"
@@ -286,7 +255,7 @@ class TDTestCase:
             tdSql.query(f"select hyperloglog({k}) from {stbname} group by {k}")
 
         tdSql.execute(f'drop database {dbname}')
-        
+
 
     def __insert_data(self, rows):
         now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)

tests/system-test/2-query/leastsquares.py

@@ -195,38 +195,6 @@ class TDTestCase:
         # return filter(None, sqls)
         return list(filter(None, current_sqls)), list(filter(None, err_sqls))
 
-    def __get_type(self, col):
-        if tdSql.cursor.istype(col, "BOOL"):
-            return "BOOL"
-        if tdSql.cursor.istype(col, "INT"):
-            return "INT"
-        if tdSql.cursor.istype(col, "BIGINT"):
-            return "BIGINT"
-        if tdSql.cursor.istype(col, "TINYINT"):
-            return "TINYINT"
-        if tdSql.cursor.istype(col, "SMALLINT"):
-            return "SMALLINT"
-        if tdSql.cursor.istype(col, "FLOAT"):
-            return "FLOAT"
-        if tdSql.cursor.istype(col, "DOUBLE"):
-            return "DOUBLE"
-        if tdSql.cursor.istype(col, "BINARY"):
-            return "BINARY"
-        if tdSql.cursor.istype(col, "NCHAR"):
-            return "NCHAR"
-        if tdSql.cursor.istype(col, "TIMESTAMP"):
-            return "TIMESTAMP"
-        if tdSql.cursor.istype(col, "JSON"):
-            return "JSON"
-        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
-            return "TINYINT UNSIGNED"
-        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
-            return "SMALLINT UNSIGNED"
-        if tdSql.cursor.istype(col, "INT UNSIGNED"):
-            return "INT UNSIGNED"
-        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
-            return "BIGINT UNSIGNED"
-
     def leastsquares_check(self):
         current_sqls, err_sqls = self.sql_list()
         for i in range(len(err_sqls)):

tests/system-test/2-query/spread.py

@@ -159,38 +159,6 @@ class TDTestCase:
         # return filter(None, sqls)
         return list(filter(None, sqls))
 
-    def __get_type(self, col):
-        if tdSql.cursor.istype(col, "BOOL"):
-            return "BOOL"
-        if tdSql.cursor.istype(col, "INT"):
-            return "INT"
-        if tdSql.cursor.istype(col, "BIGINT"):
-            return "BIGINT"
-        if tdSql.cursor.istype(col, "TINYINT"):
-            return "TINYINT"
-        if tdSql.cursor.istype(col, "SMALLINT"):
-            return "SMALLINT"
-        if tdSql.cursor.istype(col, "FLOAT"):
-            return "FLOAT"
-        if tdSql.cursor.istype(col, "DOUBLE"):
-            return "DOUBLE"
-        if tdSql.cursor.istype(col, "BINARY"):
-            return "BINARY"
-        if tdSql.cursor.istype(col, "NCHAR"):
-            return "NCHAR"
-        if tdSql.cursor.istype(col, "TIMESTAMP"):
-            return "TIMESTAMP"
-        if tdSql.cursor.istype(col, "JSON"):
-            return "JSON"
-        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
-            return "TINYINT UNSIGNED"
-        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
-            return "SMALLINT UNSIGNED"
-        if tdSql.cursor.istype(col, "INT UNSIGNED"):
-            return "INT UNSIGNED"
-        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
-            return "BIGINT UNSIGNED"
-
     def spread_check(self):
         sqls = self.sql_list()
         tdLog.printNoPrefix("===step 1: curent case, must return query OK")

tests/system-test/7-tmq/tmqAlterSchema.py

+
+from ntpath import join
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+from util.cluster import *
+
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.dnodes = 5
+        self.mnodes = 3
+        self.idIndex = 0
+        self.roleIndex = 2
+        self.mnodeStatusIndex = 3
+        self.mnodeEpIndex = 1
+        self.dnodeStatusIndex = 4
+        self.mnodeCheckCnt    = 10
+        self.host = socket.gethostname()
+        self.startPort = 6030
+        self.portStep = 100
+        self.dnodeOfLeader = 0
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+        #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+        
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: topic: select * from stb, while consume, add column int-A/bianry-B/float-C, and then modify B, drop C")
+        tdLog.printNoPrefix("add tag int-A/bianry-B/float-C, and then rename A, modify B, drop C, set t2")
+        paraDict = {'dbName':     'db1',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}, {'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     1,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1}
+
+        topicNameList = ['topic1']
+        expectRowsList = []
+        queryStringList = []   
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+        tdLog.info("create stb")
+        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tdLog.info("create ctb")
+        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+        # tdLog.info("async insert data")
+        # pThread = tmqCom.asyncInsertData(paraDict)
+        tmqCom.insert_data_2(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"])
+        
+        tdLog.info("create topics from stb with filter")
+        queryStringList.append("select * from %s.%s" %(paraDict['dbName'], paraDict['stbName']))
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryStringList[0])
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryStringList[0])        
+        expectRowsList.append(tdSql.getRows())        
+                
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        dstFile = tmqCom.getResultFileByTaosShell(consumerId, queryStringList[0])
+        
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+        tdLog.info("wait the notify info of start consume, then alter schema")
+        tmqCom.getStartConsumeNotifyFromTmqsim()
+                
+        # add column double-A/bianry-B/double-C, and then modify B, drop C    
+        sqlString = "alter table %s.%s add column newc1 double"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s add column newc2 binary(16)"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s add column newc3 double"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s modify column newc2 binary(32)"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s drop column newc3"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)        
+        # add tag double-A/bianry-B/double-C, and then rename A, modify B, drop C, set t1    
+        sqlString = "alter table %s.%s add tag newt1 double"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s add tag newt2 binary(16)"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s add tag newt3 double"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s rename tag newt1 newt1n"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s modify tag newt2 binary(32)"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s drop tag newt3"%(paraDict["dbName"],paraDict['stbName'])
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s0 set tag newt2='new tag'"%(paraDict["dbName"],paraDict['ctbPrefix'])
+        tdSql.execute(sqlString)
+
+        tdLog.info("check the consume result") 
+        tdSql.query(queryStringList[0])        
+        expectRowsList.append(tdSql.getRows())
+
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        tdLog.info("expect consume rows: %d"%(expectRowsList[0]))
+        tdLog.info("act consume rows: %d"%(resultList[0]))
+        
+        if expectRowsList[0] != resultList[0]:
+            tdLog.exit("0 tmq consume rows error!")
+
+        tmqCom.checkTmqConsumeFileContent(consumerId, dstFile)
+        
+        time.sleep(10)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+        
+    def tmqCase2(self):
+        tdLog.printNoPrefix("======== test case 2: topic: select * from ntb, while consume, add column int-A/bianry-B/float-C, and then rename A, modify B, drop C")
+        paraDict = {'dbName':     'db1',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':2}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     1,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1}
+        
+        ntbName = 'ntb'        
+
+        topicNameList = ['topic1']
+        expectRowsList = []
+        queryStringList = []   
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+        tdLog.info("create stb")
+        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tdLog.info("create ntb")
+        tdCom.create_table(tdSql, dbname=paraDict["dbName"],tbname=ntbName,column_elm_list=paraDict['colSchema'],count=1)
+        tdLog.info("start insert data ....")
+        # pThread = tmqCom.asyncInsertData(paraDict)
+        tdCom.insert_rows(tdSql, dbname=paraDict["dbName"], tbname=ntbName, column_ele_list=paraDict['colSchema'], start_ts_value=paraDict["startTs"], count=paraDict["rowsPerTbl"])
+        tdLog.info("insert data end")
+        
+        tdLog.info("create topics from ntb with filter")
+        queryStringList.append("select * from %s.%s" %(paraDict['dbName'], ntbName))
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryStringList[0])
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryStringList[0])        
+        expectRowsList.append(tdSql.getRows())        
+                
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        dstFile = tmqCom.getResultFileByTaosShell(consumerId, queryStringList[0])
+        
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+        tdLog.info("wait the notify info of start consume, then alter schema")
+        tmqCom.getStartConsumeNotifyFromTmqsim()
+                
+        # add column double-A/bianry-B/double-C, and then rename A, modify B, drop C
+        sqlString = "alter table %s.%s add column newc1 double"%(paraDict["dbName"],ntbName)
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s add column newc2 binary(16)"%(paraDict["dbName"],ntbName)
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s add column newc3 double"%(paraDict["dbName"],ntbName)
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s rename column newc1 newc1n"%(paraDict["dbName"],ntbName)
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s modify column newc2 binary(32)"%(paraDict["dbName"],ntbName)
+        tdSql.execute(sqlString)
+        sqlString = "alter table %s.%s drop column newc3"%(paraDict["dbName"],ntbName)
+        tdSql.execute(sqlString)
+
+        tdLog.info("check the consume result") 
+        tdSql.query(queryStringList[0])        
+        expectRowsList.append(tdSql.getRows())
+
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        tdLog.info("expect consume rows: %d"%(expectRowsList[0]))
+        tdLog.info("act consume rows: %d"%(resultList[0]))
+        
+        if expectRowsList[0] != resultList[0]:
+            tdLog.exit("0 tmq consume rows error!")
+
+        tmqCom.checkTmqConsumeFileContent(consumerId, dstFile)
+        
+        time.sleep(10)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 2 end ...... ")
+
+    def run(self):
+        self.tmqCase1()
+        self.tmqCase2()
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqCommon.py

@@ -93,7 +93,7 @@ class TMQCom:
         
         return resultList
 
-    def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0,alias=0):
+    def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0,alias=0,snapshot=0):
         buildPath = tdCom.getBuildPath()
         cfgPath = tdCom.getClientCfgPath()
         if valgrind == 1:
@@ -109,7 +109,7 @@ class TMQCom:
                 os.system(shellCmd)
                 processorName = processorNameNew
             shellCmd = 'mintty -h never ' + processorName + ' -c ' + cfgPath
-            shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) 
+            shellCmd += " -y %d -d %s -g %d -r %d -w %s -e %d "%(pollDelay, dbName, showMsg, showRow, cdbName, snapshot) 
             shellCmd += "> nul 2>&1 &"   
         else:
             processorName = buildPath + '/build/bin/tmq_sim'
@@ -119,7 +119,7 @@ class TMQCom:
                 os.system(shellCmd)
                 processorName = processorNameNew
             shellCmd = 'nohup ' + processorName + ' -c ' + cfgPath
-            shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) 
+            shellCmd += " -y %d -d %s -g %d -r %d -w %s -e %d "%(pollDelay, dbName, showMsg, showRow, cdbName, snapshot) 
             shellCmd += "> /dev/null 2>&1 &"
         tdLog.info(shellCmd)
         os.system(shellCmd) 
@@ -383,6 +383,69 @@ class TMQCom:
         pThread.start()
         return pThread
 
+    def checkFileContent(self, consumerId, queryString):
+        buildPath = tdCom.getBuildPath()
+        cfgPath = tdCom.getClientCfgPath()
+        dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
+        cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+        
+        consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
+        tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
+
+        consumeFile = open(consumeRowsFile, mode='r')
+        queryFile = open(dstFile, mode='r')
+        
+        # skip first line for it is schema
+        queryFile.readline()
+        lines = 0
+        while True:
+            dst = queryFile.readline()
+            src = consumeFile.readline()
+            lines += 1
+            if dst:
+                if dst != src:
+                    tdLog.info("src row: %s"%src)
+                    tdLog.info("dst row: %s"%dst)
+                    tdLog.exit("consumerId %d consume rows[%d] is not match the rows by direct query"%(consumerId, lines))
+            else:
+                break
+        return 
+
+    def getResultFileByTaosShell(self, consumerId, queryString):
+        buildPath = tdCom.getBuildPath()
+        cfgPath = tdCom.getClientCfgPath()
+        dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId)
+        cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile)
+        tdLog.info(cmdStr)
+        os.system(cmdStr)
+        return dstFile
+    
+    def checkTmqConsumeFileContent(self, consumerId, dstFile):   
+        cfgPath = tdCom.getClientCfgPath()     
+        consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId)
+        tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile))
+
+        consumeFile = open(consumeRowsFile, mode='r')
+        queryFile = open(dstFile, mode='r')
+        
+        # skip first line for it is schema
+        queryFile.readline()
+        lines = 0
+        while True:
+            dst = queryFile.readline()
+            src = consumeFile.readline()
+            lines += 1
+            if dst:
+                if dst != src:
+                    tdLog.info("src row: %s"%src)
+                    tdLog.info("dst row: %s"%dst)
+                    tdLog.exit("consumerId %d consume rows[%d] is not match the rows by direct query"%(consumerId, lines))
+            else:
+                break
+        return 
+
     def close(self):
         self.cursor.close()
 

tests/system-test/7-tmq/tmqConsFromTsdb.py

+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db1',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     1,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   1}
+
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+        tdLog.info("create stb")
+        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tdLog.info("create ctb")
+        tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
+        tdLog.info("insert data")
+        tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+        
+        tdDnodes.stop(1)
+        time.sleep(2)
+        tdDnodes.start(1)
+        
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        # sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result") 
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        if expectRowsList[0] != resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+            tdLog.exit("0 tmq consume rows error!")
+
+        tmqCom.checkFileContent(consumerId, queryString)     
+
+
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def run(self):
+        tdSql.prepare()
+        self.tmqCase1()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/fulltest.sh

@@ -153,3 +153,4 @@ python3 ./test.py -f 7-tmq/tmqUdf.py
 #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5
 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py
 python3 ./test.py -f 7-tmq/tmqShow.py
+python3 ./test.py -f 7-tmq/tmqAlterSchema.py

tests/test/c/tmqSim.c

@@ -36,7 +36,11 @@
 #define MAX_CONSUMER_THREAD_CNT (16)
 #define MAX_VGROUP_CNT          (32)
 
-typedef enum { NOTIFY_CMD_START_CONSUM, NOTIFY_CMD_START_COMMIT, NOTIFY_CMD_ID_BUTT } NOTIFY_CMD_ID;
+typedef enum {
+  NOTIFY_CMD_START_CONSUM,
+  NOTIFY_CMD_START_COMMIT,
+  NOTIFY_CMD_ID_BUTT,
+} NOTIFY_CMD_ID;
 
 typedef struct {
   TdThread thread;
@@ -633,8 +637,9 @@ void loop_consume(SThreadInfo* pInfo) {
     }
   }
 
-  uint64_t lastPrintTime = taosGetTimestampMs();
-  uint64_t startTs = taosGetTimestampMs();
+  int64_t    lastTotalMsgs = 0;
+  uint64_t   lastPrintTime = taosGetTimestampMs();
+  uint64_t   startTs = taosGetTimestampMs();
 
   int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000);
   while (running) {
@@ -647,20 +652,22 @@ void loop_consume(SThreadInfo* pInfo) {
       taos_free_result(tmqMsg);
 
       totalMsgs++;
-
-      int64_t currentPrintTime = taosGetTimestampMs();
-      if (currentPrintTime - lastPrintTime > 10 * 1000) {
-        taosFprintfFile(g_fp, "consumer id %d has currently poll total msgs: %" PRId64 "\n", pInfo->consumerId,
-                        totalMsgs);
-        lastPrintTime = currentPrintTime;
-      }
-
+	  
+	  int64_t currentPrintTime = taosGetTimestampMs();
+	  if (currentPrintTime - lastPrintTime > 10 * 1000) {
+		  taosFprintfFile(g_fp,	
+		  	              "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n", 
+		  	              pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0/(currentPrintTime - lastPrintTime));
+		  lastPrintTime = currentPrintTime;
+		  lastTotalMsgs = totalMsgs;
+	  }
+	  
       if (0 == once_flag) {
         once_flag = 1;
         notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM);
       }
 
-      if (totalRows >= pInfo->expectMsgCnt) {
+      if ((totalRows >= pInfo->expectMsgCnt) || (totalMsgs >= pInfo->expectMsgCnt)) {
         char tmpString[128];
         taosFprintfFile(g_fp, "%s over than expect rows, so break consume\n", getCurrentTimeString(tmpString));
         break;
@@ -671,7 +678,7 @@ void loop_consume(SThreadInfo* pInfo) {
       break;
     }
   }
-
+  
   if (0 == running) {
     taosFprintfFile(g_fp, "receive stop signal and not continue consume\n");
   }
@@ -881,11 +888,11 @@ int main(int32_t argc, char* argv[]) {
 
   int64_t t = end - start;
   if (0 == t) t = 1;
-
+  
   double tInMs = (double)t / 1000000.0;
   taosFprintfFile(g_fp,
-                  "Spent %.4f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.2f msgs/second\n\n",
-                  tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs));
+				"Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/second\n\n",
+				tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs));
 
   taosFprintfFile(g_fp, "==== close tmqlog ====\n");
   taosCloseFile(&g_fp);