Merge remote-tracking branch 'origin/3.0' into feature/check

example/src/tmq.c

@@ -167,7 +167,7 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
   /*tmq_conf_set(conf, "td.connect.db", "abc1");*/
   tmq_conf_set(conf, "msg.with.table.name", "true");
-  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
+  tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
   tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
   assert(tmq);
   return tmq;
@@ -176,6 +176,7 @@ tmq_t* build_consumer() {
 tmq_list_t* build_topic_list() {
   tmq_list_t* topic_list = tmq_list_new();
   tmq_list_append(topic_list, "topic_ctb_column");
+  /*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/
   return topic_list;
 }
 
@@ -190,7 +191,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   int32_t cnt = 0;
   /*clock_t startTime = clock();*/
   while (running) {
-    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0);
     if (tmqmessage) {
       cnt++;
       /*printf("get data\n");*/

include/client/taos.h

@@ -233,6 +233,8 @@ DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
 DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
+DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
+DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
 DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
@@ -251,7 +253,7 @@ typedef enum tmq_conf_res_t tmq_conf_res_t;
 DLL_EXPORT tmq_conf_t    *tmq_conf_new();
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
-DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
+DLL_EXPORT void           tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
 
 /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
 

include/common/tmsg.h

@@ -403,6 +403,19 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWra
   return 0;
 }
 
+static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaWrapper* pSW) {
+  if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
+
+  pSW->pSchema = (SSchema*)tDecoderMalloc(pDecoder, pSW->nCols * sizeof(SSchema));
+  if (pSW->pSchema == NULL) return -1;
+  for (int32_t i = 0; i < pSW->nCols; i++) {
+    if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1;
+  }
+
+  return 0;
+}
+
 STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
 
 typedef struct {

include/util/taoserror.h

@@ -668,6 +668,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_UDF_INVALID_STATE             TAOS_DEF_ERROR_CODE(0, 0x2906)
 #define TSDB_CODE_UDF_INVALID_INPUT             TAOS_DEF_ERROR_CODE(0, 0x2907)
 #define TSDB_CODE_UDF_NO_FUNC_HANDLE            TAOS_DEF_ERROR_CODE(0, 0x2908)
+#define TSDB_CODE_UDF_INVALID_BUFSIZE           TAOS_DEF_ERROR_CODE(0, 0x2909)
+#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE       TAOS_DEF_ERROR_CODE(0, 0x290A)
 
 #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE     TAOS_DEF_ERROR_CODE(0, 0x3000)
 #define TSDB_CODE_SML_INVALID_PRECISION_TYPE    TAOS_DEF_ERROR_CODE(0, 0x3001)

source/client/src/tmq.c

@@ -182,10 +182,14 @@ typedef struct {
 
 typedef struct {
   tmq_t*         tmq;
-  int32_t        async;
+  int8_t         async;
+  int8_t         automatic;
+  int8_t         freeOffsets;
+  tmq_commit_cb* userCb;
   tsem_t         rspSem;
   tmq_resp_err_t rspErr;
   SArray*        offsets;
+  void*          userParam;
 } SMqCommitCbParam;
 
 tmq_conf_t* tmq_conf_new() {
@@ -314,6 +318,142 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
   return sprintf(dst, "%s:%d", topicName, vg);
 }
 
+int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
+  SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
+  pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
+  if (pParam->async) {
+    if (pParam->automatic && pParam->tmq->commitCb) {
+      pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets,
+                            pParam->tmq->commitCbUserParam);
+    } else if (!pParam->automatic && pParam->userCb) {
+      pParam->userCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets, pParam->userParam);
+    }
+
+    if (pParam->freeOffsets) {
+      taosArrayDestroy(pParam->offsets);
+    }
+
+    taosMemoryFree(pParam);
+  } else {
+    tsem_post(&pParam->rspSem);
+  }
+  return 0;
+}
+
+int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_t automatic, int8_t async,
+                       tmq_commit_cb* userCb, void* userParam) {
+  SMqCMCommitOffsetReq req;
+  SArray*              pOffsets = NULL;
+  void*                buf = NULL;
+  SMqCommitCbParam*    pParam = NULL;
+  SMsgSendInfo*        sendInfo = NULL;
+  int8_t               freeOffsets;
+  int32_t              code = -1;
+
+  if (offsets == NULL) {
+    freeOffsets = 1;
+    pOffsets = taosArrayInit(0, sizeof(SMqOffset));
+    for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
+      SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
+      for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
+        SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
+        SMqOffset    offset;
+        tstrncpy(offset.topicName, pTopic->topicName, TSDB_TOPIC_FNAME_LEN);
+        tstrncpy(offset.cgroup, tmq->groupId, TSDB_CGROUP_LEN);
+        offset.vgId = pVg->vgId;
+        offset.offset = pVg->currentOffset;
+        taosArrayPush(pOffsets, &offset);
+      }
+    }
+  } else {
+    freeOffsets = 0;
+    pOffsets = (SArray*)&offsets->container;
+  }
+
+  req.num = (int32_t)pOffsets->size;
+  req.offsets = pOffsets->pData;
+
+  SEncoder encoder;
+
+  tEncoderInit(&encoder, NULL, 0);
+  code = tEncodeSMqCMCommitOffsetReq(&encoder, &req);
+  if (code < 0) {
+    goto END;
+  }
+  int32_t tlen = encoder.pos;
+  buf = taosMemoryMalloc(tlen);
+  if (buf == NULL) {
+    tEncoderClear(&encoder);
+    goto END;
+  }
+  tEncoderClear(&encoder);
+
+  tEncoderInit(&encoder, buf, tlen);
+  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
+  tEncoderClear(&encoder);
+
+  pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam));
+  if (pParam == NULL) {
+    goto END;
+  }
+  pParam->tmq = tmq;
+  pParam->automatic = automatic;
+  pParam->async = async;
+  pParam->offsets = pOffsets;
+  pParam->freeOffsets = freeOffsets;
+  pParam->userCb = userCb;
+  pParam->userParam = userParam;
+  if (!async) tsem_init(&pParam->rspSem, 0, 0);
+
+  sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
+  if (sendInfo == NULL) goto END;
+  sendInfo->msgInfo = (SDataBuf){
+      .pData = buf,
+      .len = tlen,
+      .handle = NULL,
+  };
+
+  sendInfo->requestId = generateRequestId();
+  sendInfo->requestObjRefId = 0;
+  sendInfo->param = pParam;
+  sendInfo->fp = tmqCommitCb;
+  sendInfo->msgType = TDMT_MND_MQ_COMMIT_OFFSET;
+
+  SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
+
+  int64_t transporterId = 0;
+  asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
+
+  if (!async) {
+    tsem_wait(&pParam->rspSem);
+    code = pParam->rspErr;
+    tsem_destroy(&pParam->rspSem);
+    taosMemoryFree(pParam);
+  }
+
+  // avoid double free if msg is sent
+  buf = NULL;
+
+  code = 0;
+END:
+  if (buf) taosMemoryFree(buf);
+  /*if (pParam) taosMemoryFree(pParam);*/
+  /*if (sendInfo) taosMemoryFree(sendInfo);*/
+
+  if (code != 0 && async) {
+    if (automatic) {
+      tmq->commitCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
+    } else {
+      userCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
+    }
+  }
+
+  if (!async && freeOffsets) {
+    taosArrayDestroy(pOffsets);
+  }
+  return code;
+}
+
 void tmqAssignDelayedHbTask(void* param, void* tmrId) {
   tmq_t*  tmq = (tmq_t*)param;
   int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM);
@@ -350,7 +490,8 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
       tmqAskEp(tmq, true);
       taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
-      tmq_commit(tmq, NULL, true);
+      /*tmq_commit(tmq, NULL, true);*/
+      tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam);
       taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer);
     } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
     } else {
@@ -385,32 +526,11 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
 int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
   pParam->rspErr = code;
-  tmq_t* tmq = pParam->tmq;
+  /*tmq_t* tmq = pParam->tmq;*/
   tsem_post(&pParam->rspSem);
   return 0;
 }
 
-int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
-  SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
-  pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
-  if (pParam->tmq->commitCb) {
-    pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, NULL, pParam->tmq->commitCbUserParam);
-  }
-  if (!pParam->async)
-    tsem_post(&pParam->rspSem);
-  else {
-    if (pParam->offsets) {
-      taosArrayDestroy(pParam->offsets);
-    }
-    tsem_destroy(&pParam->rspSem);
-    /*if (pParam->pArray) {*/
-    /*taosArrayDestroy(pParam->pArray);*/
-    /*}*/
-    taosMemoryFree(pParam);
-  }
-  return 0;
-}
-
 tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
   if (*topics == NULL) {
     *topics = tmq_list_new();
@@ -541,6 +661,8 @@ FAIL:
 }
 
 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
+  return tmqCommitInner(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam);
+#if 0
   // TODO: add read write lock
   SRequestObj*   pRequest = NULL;
   tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS;
@@ -627,6 +749,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
   }
 
   return resp;
+#endif
 }
 
 tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
@@ -723,7 +846,7 @@ FAIL:
   return code;
 }
 
-void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) {
+void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) {
   //
   conf->commitCb = cb;
   conf->commitCbUserParam = param;
@@ -1384,3 +1507,10 @@ const char* tmq_get_table_name(TAOS_RES* res) {
   }
   return NULL;
 }
+DLL_EXPORT void tmq_commit_async(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, tmq_commit_cb* cb, void* param) {
+  tmqCommitInner(tmq, offsets, 0, 1, cb, param);
+}
+
+DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
+  return tmqCommitInner(tmq, offsets, 0, 0, NULL, NULL);
+}

source/dnode/mnode/impl/src/mndOffset.c

@@ -196,6 +196,8 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
     }
   }
 
+  tDecoderClear(&decoder);
+
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);

source/dnode/vnode/src/inc/sma.h

@@ -70,7 +70,7 @@ struct SSmaStatItem {
    * N.B. only applicable to tsma
    */
   int8_t    state;           // ETsdbSmaStat
-  SHashObj *expiredWindows;  // key: skey of time window, value: N/A
+  SHashObj *expiredWindows;  // key: skey of time window, value: version
   STSma    *pTSma;           // cache schema
 };
 

source/dnode/vnode/src/inc/tsdb.h

@@ -60,7 +60,7 @@ typedef struct {
   TSKEY minKey;
 } SRtn;
 
-#define TSDB_DATA_DIR_LEN 6
+#define TSDB_DATA_DIR_LEN 6  // adapt accordingly
 struct STsdb {
   char          *path;
   SVnode        *pVnode;

source/dnode/vnode/src/meta/metaEntry.c

@@ -56,8 +56,8 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
   if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
 
   if (pME->type == TSDB_SUPER_TABLE) {
-    if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1;
-    if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
+    if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schema) < 0) return -1;
+    if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
   } else if (pME->type == TSDB_CHILD_TABLE) {
     if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
@@ -67,9 +67,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
     if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
     if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1;
-    if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1;
+    if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schema) < 0) return -1;
   } else if (pME->type == TSDB_TSMA_TABLE) {
-    pME->smaEntry.tsma = taosMemoryCalloc(1, sizeof(STSma));
+    pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma));
     if(!pME->smaEntry.tsma) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;

source/dnode/vnode/src/meta/metaQuery.c

@@ -394,11 +394,6 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
     goto _err;
   }
 
-  SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
-  if (pCur == NULL) {
-    goto _err;
-  }
-
   SMetaReader mr = {0};
   metaReaderInit(&mr, pMeta, 0);
   int64_t smaId;
@@ -442,12 +437,10 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) {
 
   metaReaderClear(&mr);
   taosArrayDestroy(pSmaIds);
-  metaCloseSmaCursor(pCur);
   return pSW;
 _err:
   metaReaderClear(&mr);
   taosArrayDestroy(pSmaIds);
-  metaCloseSmaCursor(pCur);
   tdFreeTSmaWrapper(pSW, deepCopy);
   return NULL;
 }

source/dnode/vnode/src/sma/smaTDBImpl.c

@@ -55,12 +55,13 @@ static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int
 }
 
 static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) {
-  int           ret;
   tdb_cmpr_fn_t compFunc;
 
   // Create a database
   compFunc = tdSmaKeyCmpr;
-  ret = tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB);
+  if(tdbDbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) {
+    return -1;
+  }
 
   return 0;
 }
@@ -76,7 +77,7 @@ int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) {
 
   // Open DBF
   if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) {
-    terrno = TSDB_CODE_TDB_INIT_FAILED;
+    smaError("failed to open DBF: %s", pDBF->path);
     smaCloseDBDb(pDBF->pDB);
     return -1;
   }
@@ -97,9 +98,10 @@ int32_t smaCloseDBF(SDBFile *pDBF) {
 int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
   int32_t ret;
 
-  ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
+  printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn);
+  ret = tdbDbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
   if (ret < 0) {
-    smaError("failed to create insert sma data into db, ret = %d", ret);
+    smaError("failed to upsert tsma data into db, ret = %d", ret);
     return -1;
   }
 
@@ -113,7 +115,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_
   ret = tdbDbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
 
   if (ret < 0) {
-    smaError("failed to get sma data from db, ret = %d", ret);
+    smaError("failed to get tsma data from db, ret = %d", ret);
     return NULL;
   }
 

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -16,21 +16,22 @@
 #include "sma.h"
 #include "tsdb.h"
 
+typedef STsdbCfg STSmaKeepCfg;
+
 #undef _TEST_SMA_PRINT_DEBUG_LOG_
-#define SMA_STORAGE_TSDB_DAYS   30
-#define SMA_STORAGE_TSDB_TIMES  10
-#define SMA_STORAGE_SPLIT_HOURS 24
-#define SMA_KEY_LEN             16  // TSKEY+groupId 8+8
-#define SMA_DROP_EXPIRED_TIME   10  // default is 10 seconds
+#define SMA_STORAGE_TSDB_MINUTES 86400
+#define SMA_STORAGE_TSDB_TIMES   10
+#define SMA_STORAGE_SPLIT_FACTOR 144  // least records in tsma file
+#define SMA_KEY_LEN              16   // TSKEY+groupId 8+8
+#define SMA_DROP_EXPIRED_TIME    10   // default is 10 seconds
 
 #define SMA_STATE_ITEM_HASH_SLOT 32
 
-
 typedef struct {
   SSma         *pSma;
   SDBFile       dFile;
   const SArray *pDataBlocks;  // sma data
-  int32_t       interval;     // interval with the precision of DB
+  int64_t       interval;     // interval with the precision of DB
 } STSmaWriteH;
 
 typedef struct {
@@ -42,10 +43,10 @@ typedef struct {
   STsdb    *pTsdb;
   SSma     *pSma;
   SDBFile   dFile;
-  int32_t   interval;   // interval with the precision of DB
+  int64_t   interval;   // interval with the precision of DB
   int32_t   blockSize;  // size of SMA block item
+  int32_t   days;
   int8_t    storageLevel;
-  int8_t    days;
   SmaFsIter smaFsIter;
 } STSmaReadH;
 
@@ -58,9 +59,9 @@ typedef enum {
 // static func
 
 static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted);
-static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit);
+static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval);
 static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
-                                  int8_t intervalUnit);
+                                int8_t intervalUnit);
 static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit);
 static void    tdDestroyTSmaWriteH(STSmaWriteH *pSmaH);
 static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel);
@@ -92,9 +93,10 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
  * @return int32_t
  */
 static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) {
+  STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma);
   pSmaH->pSma = pSma;
   pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true);
-  pSmaH->storageLevel = tdGetSmaStorageLevel(interval, intervalUnit);
+  pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval);
   pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel);
   return TSDB_CODE_SUCCESS;
 }
@@ -275,11 +277,13 @@ static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t f
  */
 static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) {
   STsdbCfg *pCfg = SMA_TSDB_CFG(pSma);
-  int32_t   daysPerFile = pCfg->days;
+  int32_t   daysPerFile = pCfg->days;  // unit is minute
 
   if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
-    int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
-    daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS;
+    int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
+    if (minutes > SMA_STORAGE_TSDB_MINUTES) {
+      daysPerFile = SMA_STORAGE_TSDB_MINUTES;
+    }
   }
 
   return daysPerFile;
@@ -288,45 +292,14 @@ static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel)
 /**
  * @brief Judge the tSma storage level
  *
+ * @param pCfg
  * @param interval
- * @param intervalUnit
  * @return int32_t
  */
-static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
-  // TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
-  switch (intervalUnit) {
-    case TIME_UNIT_HOUR:
-      if (interval < SMA_STORAGE_SPLIT_HOURS) {
-        return SMA_STORAGE_LEVEL_DFILESET;
-      }
-      break;
-    case TIME_UNIT_MINUTE:
-      if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) {
-        return SMA_STORAGE_LEVEL_DFILESET;
-      }
-      break;
-    case TIME_UNIT_SECOND:
-      if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) {
-        return SMA_STORAGE_LEVEL_DFILESET;
-      }
-      break;
-    case TIME_UNIT_MILLISECOND:
-      if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) {
-        return SMA_STORAGE_LEVEL_DFILESET;
-      }
-      break;
-    case TIME_UNIT_MICROSECOND:
-      if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) {
-        return SMA_STORAGE_LEVEL_DFILESET;
-      }
-      break;
-    case TIME_UNIT_NANOSECOND:
-      if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) {
-        return SMA_STORAGE_LEVEL_DFILESET;
-      }
-      break;
-    default:
-      break;
+static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) {
+  int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE);
+  if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) {
+    return SMA_STORAGE_LEVEL_DFILESET;
   }
   return SMA_STORAGE_LEVEL_TSDB;
 }
@@ -346,6 +319,7 @@ static int32_t tdGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
 int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
   STsdbCfg     *pCfg = SMA_TSDB_CFG(pSma);
   const SArray *pDataBlocks = (const SArray *)msg;
+  int64_t       testSkey = TSKEY_INITIAL_VAL;
 
   // TODO: destroy SSDataBlocks(msg)
 
@@ -403,8 +377,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
   }
 
   // Step 1: Judge the storage level and days
-  int32_t storageLevel = tdGetSmaStorageLevel(pTSma->interval, pTSma->intervalUnit);
-  int32_t daysPerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel);
+  int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval);
+  int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel);
 
   char    smaKey[SMA_KEY_LEN] = {0};  // key: skey + groupId
   char    dataBuf[512] = {0};         // val: aggr data // TODO: handle 512 buffer?
@@ -432,6 +406,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
             if (!isStartKey) {
               isStartKey = true;
               skey = *(TSKEY *)var;
+              testSkey = skey;
               printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId);
               tdEncodeTSmaKey(groupId, skey, &pSmaKey);
             } else {
@@ -503,9 +478,10 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
             break;
         }
       }
+      printf("\n");
       // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) {
       if (tlen > 0) {
-        int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision));
+        int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision));
 
         // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index
         // file
@@ -517,6 +493,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
             smaCloseDBF(&tSmaH.dFile);
           }
           tdSetTSmaDataFile(&tSmaH, indexUid, fid);
+          smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 " queryKey:%" PRIi64,
+                   SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey);
           if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) {
             smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma),
                      tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
@@ -528,16 +506,17 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
         }
 
         if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) {
-          smaWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
-                   " since %s",
-                   SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno));
+          smaWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
+                  " since %s",
+                  SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno));
           tdSmaEndCommit(pEnv);
           tdDestroyTSmaWriteH(&tSmaH);
           tdUnRefSmaStat(pSma, pStat);
           return TSDB_CODE_FAILED;
         }
-        smaDebug("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
-                  SMA_VID(pSma), indexUid, skey, groupId);
+
+        smaDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
+                 SMA_VID(pSma), indexUid, skey, groupId);
         // TODO:tsdbEndTSmaCommit();
 
         // Step 3: reset the SSmaStat
@@ -547,7 +526,6 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
                  SMA_VID(pSma), skey, tlen, indexUid);
       }
 
-      printf("\n");
     }
   }
   tdSmaEndCommit(pEnv);  // TODO: not commit for every insert
@@ -579,14 +557,14 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL
                                   TXN *txn) {
   SDBFile *pDBFile = &pSmaH->dFile;
 
-  // TODO: insert sma data blocks into B+Tree(TDB)
+  // TODO: insert tsma data blocks into B+Tree(TDB)
   if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
-    smaWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
-           SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
+    smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
+            SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
     return TSDB_CODE_FAILED;
   }
-  smaDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
-          SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
+  smaDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
+           SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
 
 #ifdef _TEST_SMA_PRINT_DEBUG_LOG_
   uint32_t valueSize = 0;
@@ -776,6 +754,8 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
   tdUnRefSmaStat(pSma, pStat);
 
   tdInitTSmaFile(&tReadH, indexUid, querySKey);
+  smaDebug("### vgId:%d read from DBF %s  days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64,
+          SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey);
   if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) {
     smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno));
     return TSDB_CODE_FAILED;
@@ -783,7 +763,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
 
   char    smaKey[SMA_KEY_LEN] = {0};
   void   *pSmaKey = &smaKey;
-  int64_t queryGroupId = 1;
+  int64_t queryGroupId = 0;
   tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey);
 
   smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma),
@@ -915,8 +895,8 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde
       terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
       taosHashCleanup(pItem->expiredWindows);
       taosMemoryFree(pItem);
-      smaWarn("vgId:%d update expired window failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid,
-               tstrerror(terrno));
+      smaWarn("vgId:%d set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
+              indexUid, tstrerror(terrno));
       return TSDB_CODE_FAILED;
     }
     pItem->pTSma = pTSma;
@@ -1021,7 +1001,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
         pSW = tdFreeTSmaWrapper(pSW, false);
         break;
       }
-      if (!pSW || (pTSma->tableUid != msgIter.suid)) {
+      if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) {
         if (pSW) {
           pSW = tdFreeTSmaWrapper(pSW, false);
         }
@@ -1043,6 +1023,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
         interval.slidingUnit = pTSma->slidingUnit;
       }
 
+      // TODO: process multiple tsma for one table uid
       TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision);
 
       if (lastWinSKey != winSKey) {

source/dnode/vnode/src/tsdb/tsdbOpen.c

@@ -42,7 +42,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
   int    slen = 0;
 
   *ppTsdb = NULL;
-  slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + TSDB_DATA_DIR_LEN + 3;
+  slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(dir) + 3;
 
   // create handle
   pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
@@ -73,7 +73,8 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
     goto _err;
   }
 
-  tsdbDebug("vgId:%d tsdb is opened for %s", TD_VID(pVnode), pTsdb->path);
+  tsdbDebug("vgId:%d tsdb is opened for %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days,
+            pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2);
 
   *ppTsdb = pTsdb;
   return 0;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -372,13 +372,13 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle,
     }
 
     if (level == TSDB_RETENTION_L0) {
-      tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L0);
+      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L0);
       return VND_RSMA0(pVnode);
     } else if (level == TSDB_RETENTION_L1) {
-      tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L1);
+      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L1);
       return VND_RSMA1(pVnode);
     } else {
-      tsdbDebug("%p rsma level %d is selected to query", pReadHandle, TSDB_RETENTION_L2);
+      tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L2);
       return VND_RSMA2(pVnode);
     }
   }

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -876,13 +876,13 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t ke
                                     TXN *txn) {
   SDBFile *pDBFile = &pSmaH->dFile;
 
-  // TODO: insert sma data blocks into B+Tree(TDB)
+  // TODO: insert tsma data blocks into B+Tree(TDB)
   if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
-    tsdbWarn("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
+    tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
              REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
     return TSDB_CODE_FAILED;
   }
-  tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
+  tsdbDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
             REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
 
 #ifdef _TEST_SMA_PRINT_DEBUG_LOG_
@@ -1245,7 +1245,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
         }
 
         if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) {
-          tsdbWarn("vgId:%d insert tSma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
+          tsdbWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
                    " since %s",
                    REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno));
           tsdbSmaEndCommit(pEnv);
@@ -1253,7 +1253,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
           tsdbUnRefSmaStat(pTsdb, pStat);
           return TSDB_CODE_FAILED;
         }
-        tsdbDebug("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
+        tsdbDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
                   REPO_ID(pTsdb), indexUid, skey, groupId);
         // TODO:tsdbEndTSmaCommit();
 

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -103,7 +103,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
 
   // open sma
   if (smaOpen(pVnode)) {
-    vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
+    vError("vgId:%d failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
 

source/libs/function/inc/builtinsimpl.h

@@ -85,6 +85,11 @@ bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo)
 int32_t spreadFunction(SqlFunctionCtx* pCtx);
 int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
+bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+int32_t elapsedFunction(SqlFunctionCtx* pCtx);
+int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+
 bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t histogramFunction(SqlFunctionCtx* pCtx);

source/libs/function/src/builtins.c

@@ -226,6 +226,27 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  int32_t paraNum = LIST_LENGTH(pFunc->pParameterList);
+  if (1 != paraNum && 2 != paraNum) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
+  if (QUERY_NODE_COLUMN != nodeType(pPara)) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "The input parameter of ELAPSED function can only be column");
+  }
+
+  uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  if (TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateLeastSQR(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
   if (3 != numOfParams) {
@@ -794,6 +815,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = spreadFunction,
     .finalizeFunc = spreadFinalize
   },
+  {
+    .name = "elapsed",
+    .type = FUNCTION_TYPE_ELAPSED,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .dataRequiredFunc = statisDataRequired,
+    .translateFunc = translateElapsed,
+    .getEnvFunc   = getElapsedFuncEnv,
+    .initFunc     = elapsedFunctionSetup,
+    .processFunc  = elapsedFunction,
+    .finalizeFunc = elapsedFinalize
+  },
   {
     .name = "last_row",
     .type = FUNCTION_TYPE_LAST_ROW,

source/libs/function/src/builtinsimpl.c

@@ -112,6 +112,13 @@ typedef struct SSpreadInfo {
   double max;
 } SSpreadInfo;
 
+typedef struct SElapsedInfo {
+  double  result;
+  TSKEY   min;
+  TSKEY   max;
+  int64_t timeUnit;
+} SElapsedInfo;
+
 typedef struct SHistoFuncBin {
   double lower;
   double upper;
@@ -2494,6 +2501,116 @@ int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return functionFinalize(pCtx, pBlock);
 }
 
+bool getElapsedFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
+  pEnv->calcMemSize = sizeof(SElapsedInfo);
+  return true;
+}
+
+bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
+  if (!functionSetup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
+  pInfo->result = 0;
+  pInfo->min = MAX_TS_KEY;
+  pInfo->max = 0;
+
+  if (pCtx->numOfParams == 3) {
+    pInfo->timeUnit = pCtx->param[1].param.i;
+  } else {
+    pInfo->timeUnit = 1;
+  }
+
+  return true;
+}
+
+int32_t elapsedFunction(SqlFunctionCtx *pCtx) {
+  int32_t numOfElems = 0;
+
+  // Only the pre-computing information loaded and actual data does not loaded
+  SInputColumnInfoData* pInput = &pCtx->input;
+  SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0];
+
+  SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  numOfElems = pInput->numOfRows; //since this is the primary timestamp, no need to exclude NULL values
+  if (numOfElems == 0) {
+    goto _elapsed_over;
+  }
+
+  if (pInput->colDataAggIsSet) {
+
+    if (pInfo->min == MAX_TS_KEY) {
+      pInfo->min = GET_INT64_VAL(&pAgg->min);
+      pInfo->max = GET_INT64_VAL(&pAgg->max);
+    } else {
+      if (pCtx->order == TSDB_ORDER_ASC) {
+        pInfo->max = GET_INT64_VAL(&pAgg->max);
+      } else {
+        pInfo->min = GET_INT64_VAL(&pAgg->min);
+      }
+    }
+  } else {  // computing based on the true data block
+    if (0 == pCtx->size) {
+      if (pCtx->order == TSDB_ORDER_DESC) {
+        if (pCtx->end.key != INT64_MIN) {
+          pInfo->min = pCtx->end.key;
+        }
+      } else {
+        if (pCtx->end.key != INT64_MIN) {
+          pInfo->max = pCtx->end.key + 1;
+        }
+      }
+      goto _elapsed_over;
+    }
+
+    SColumnInfoData* pCol = pInput->pData[0];
+
+    int32_t start     = pInput->startRowIndex;
+    TSKEY* ptsList = (int64_t*)colDataGetData(pCol, start);
+    if (pCtx->order == TSDB_ORDER_DESC) {
+      if (pCtx->start.key == INT64_MIN) {
+        pInfo->max = (pInfo->max < ptsList[pCtx->size - 1]) ? ptsList[pCtx->size - 1] : pInfo->max;
+      } else {
+        pInfo->max = pCtx->start.key + 1;
+      }
+
+      if (pCtx->end.key != INT64_MIN) {
+        pInfo->min = pCtx->end.key;
+      } else {
+        pInfo->min = ptsList[0];
+      }
+    } else {
+      if (pCtx->start.key == INT64_MIN) {
+        pInfo->min = (pInfo->min > ptsList[0]) ? ptsList[0] : pInfo->min;
+      } else {
+        pInfo->min = pCtx->start.key;
+      }
+
+      if (pCtx->end.key != INT64_MIN) {
+        pInfo->max = pCtx->end.key + 1;
+      } else {
+        pInfo->max = ptsList[pCtx->size - 1];
+      }
+    }
+  }
+
+_elapsed_over:
+  // data in the check operation are all null, not output
+  SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
+  SElapsedInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  double result = (double)pInfo->max - (double)pInfo->min;
+  result = (result >= 0) ? result : -result;
+  pInfo->result = result / pInfo->timeUnit;
+  return functionFinalize(pCtx, pBlock);
+}
+
 bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SHistoFuncInfo) + HISTOGRAM_MAX_BINS_NUM * sizeof(SHistoFuncBin);
   return true;

source/libs/function/src/tudf.c

@@ -1605,7 +1605,12 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
     return false;
   }
   udfRes->interResNum = buf.numOfResult;
-  memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
+  if (buf.bufLen <= session->bufSize) {
+    memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
+  } else {
+    fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize);
+    return false;
+  }
   freeUdfInterBuf(&buf);
   return true;
 }
@@ -1655,7 +1660,12 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
     newState.numOfResult = 0;
   } else {
     udfRes->interResNum = newState.numOfResult;
-    memcpy(udfRes->interResBuf, newState.buf, newState.bufLen);
+    if (newState.bufLen <= session->bufSize) {
+      memcpy(udfRes->interResBuf, newState.buf, newState.bufLen);
+    } else {
+      fnError("udfc inter buf size %d is greater than function bufSize %d", newState.bufLen, session->bufSize);
+      udfCode = TSDB_CODE_UDF_INVALID_BUFSIZE;
+    }
   }
   if (newState.numOfResult == 1 || state.numOfResult == 1) {
     GET_RES_INFO(pCtx)->numOfRes = 1;
@@ -1688,9 +1698,15 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
     fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode);
     GET_RES_INFO(pCtx)->numOfRes = 0;
   } else {
-    memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen);
-    udfRes->finalResNum = resultBuf.numOfResult;
-    GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
+    if (resultBuf.bufLen <= session->outputLen) {
+      memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen);
+      udfRes->finalResNum = resultBuf.numOfResult;
+      GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum;
+    } else {
+      fnError("udfc inter buf size %d is greater than function output size %d", resultBuf.bufLen, session->outputLen);
+      GET_RES_INFO(pCtx)->numOfRes = 0;
+      udfCallCode = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE;
+    }
   }
 
   freeUdfInterBuf(&resultBuf);

source/libs/function/src/udfd.c

@@ -616,9 +616,8 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
 }
 
 void udfdOnNewConnection(uv_stream_t *server, int status) {
-  fnDebug("new connection");
   if (status < 0) {
-    // TODO
+    fnError("udfd new connection error. code: %s", uv_strerror(status));
     return;
   }
 

source/libs/index/src/indexComm.c

@@ -20,12 +20,13 @@
 #include "tcompare.h"
 #include "tdataformat.h"
 #include "ttypes.h"
+#include "tvariant.h"
 
 char JSON_COLUMN[] = "JSON";
 char JSON_VALUE_DELIM = '&';
 
 char* indexInt2str(int64_t val, char* dst, int radix) {
-  char     buffer[65];
+  char     buffer[65] = {0};
   char*    p;
   int64_t  new_val;
   uint64_t uval = (uint64_t)val;
@@ -74,28 +75,70 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
   return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
 }
 TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
-  if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR) {
+  if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) {
     return tDoCompare(func, cmptype, a, b);
   }
 #if 1
-  int8_t bytes = tDataTypes[dtype].bytes;
-  if (bytes == 1) {
+  if (dtype == TSDB_DATA_TYPE_TIMESTAMP) {
+    int64_t va = taosStr2int64(a);
+    int64_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_BOOL || dtype == TSDB_DATA_TYPE_UTINYINT) {
+    uint8_t va = taosStr2int64(a);
+    uint8_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_TINYINT) {
     int8_t va = taosStr2int64(a);
     int8_t vb = taosStr2int64(b);
     return tDoCompare(func, cmptype, &va, &vb);
-  } else if (bytes == 2) {
+  } else if (dtype == TSDB_DATA_TYPE_SMALLINT) {
     int16_t va = taosStr2int64(a);
     int16_t vb = taosStr2int64(b);
     return tDoCompare(func, cmptype, &va, &vb);
-  } else if (bytes == 4) {
+  } else if (dtype == TSDB_DATA_TYPE_USMALLINT) {
+    uint16_t va = taosStr2int64(a);
+    uint16_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_INT) {
     int32_t va = taosStr2int64(a);
     int32_t vb = taosStr2int64(b);
     return tDoCompare(func, cmptype, &va, &vb);
-  } else {
+  } else if (dtype == TSDB_DATA_TYPE_UINT) {
+    uint32_t va = taosStr2int64(a);
+    uint32_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_BIGINT) {
     int64_t va = taosStr2int64(a);
     int64_t vb = taosStr2int64(b);
     return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_UBIGINT) {
+    uint64_t va, vb;
+    if (0 != toUInteger(a, strlen(a), 10, &va) || 0 != toUInteger(b, strlen(b), 10, &vb)) {
+      return CONTINUE;
+    }
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_FLOAT) {
+    float va = strtod(a, NULL);
+    if (errno == ERANGE && va == -1) {
+      return CONTINUE;
+    }
+    float vb = strtod(b, NULL);
+    if (errno == ERANGE && va == -1) {
+      return CONTINUE;
+    }
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_DOUBLE) {
+    double va = strtod(a, NULL);
+    if (errno == ERANGE && va == -1) {
+      return CONTINUE;
+    }
+    double vb = strtod(b, NULL);
+    if (errno == ERANGE && va == -1) {
+      return CONTINUE;
+    }
+    return tDoCompare(func, cmptype, &va, &vb);
   }
+  assert(0);
 #endif
 }
 TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
@@ -248,20 +291,16 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-#if 1
       tlen = taosEncodeBinary(NULL, src, strlen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, src, strlen(src));
       break;
-#endif
     }
     case TSDB_DATA_TYPE_VARBINARY:
-#if 1
       tlen = taosEncodeBinary(NULL, src, strlen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, src, strlen(src));
       break;
-#endif
     default:
       TASSERT(0);
       break;
@@ -271,87 +310,73 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
   return tlen;
 }
 int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
-  int tlen = tDataTypes[type].bytes;
-
+  int     tlen = tDataTypes[type].bytes;
+  int32_t bufSize = 64;
   switch (type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(int64_t*)src, *dst, -1);
       break;
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_UTINYINT:
-      // tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src);
-      //*dst = taosMemoryCalloc(1, tlen + 1);
-      // tlen = taosEncodeFixedU8(dst, *(uint8_t*)src);
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(uint8_t*)src, *dst, 1);
       break;
     case TSDB_DATA_TYPE_TINYINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(int8_t*)src, *dst, 1);
       break;
     case TSDB_DATA_TYPE_SMALLINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(int16_t*)src, *dst, -1);
       break;
     case TSDB_DATA_TYPE_USMALLINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(uint16_t*)src, *dst, -1);
       break;
     case TSDB_DATA_TYPE_INT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(int32_t*)src, *dst, -1);
       break;
-    case TSDB_DATA_TYPE_FLOAT:
-      tlen = taosEncodeBinary(NULL, src, sizeof(float));
-      *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeBinary(dst, src, sizeof(float));
-      *dst = (char*) * dst - tlen;
-      break;
     case TSDB_DATA_TYPE_UINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(uint32_t*)src, *dst, 1);
       break;
     case TSDB_DATA_TYPE_BIGINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
-      indexInt2str(*(int64_t*)src, *dst, 1);
-      break;
-    case TSDB_DATA_TYPE_DOUBLE:
-      tlen = taosEncodeBinary(NULL, src, sizeof(double));
-      *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeBinary(dst, src, sizeof(double));
-      *dst = (char*) * dst - tlen;
+      *dst = taosMemoryCalloc(1, bufSize + 1);
+      sprintf(*dst, "%" PRIu64, *(uint64_t*)src);
       break;
     case TSDB_DATA_TYPE_UBIGINT:
-      assert(0);
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(uint64_t*)src, *dst, 1);
+    case TSDB_DATA_TYPE_FLOAT:
+      *dst = taosMemoryCalloc(1, bufSize + 1);
+      sprintf(*dst, "%.9lf", *(float*)src);
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      *dst = taosMemoryCalloc(1, bufSize + 1);
+      sprintf(*dst, "%.9lf", *(double*)src);
       break;
     case TSDB_DATA_TYPE_NCHAR: {
       tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
-      *dst = (char*) * dst - tlen;
-
+      *dst = *dst - tlen;
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-#if 1
       tlen = taosEncodeBinary(NULL, src, strlen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, src, strlen(src));
-      *dst = (char*) * dst - tlen;
+      *dst = (char*)*dst - tlen;
       break;
-#endif
     }
     case TSDB_DATA_TYPE_VARBINARY:
-#if 1
       tlen = taosEncodeBinary(NULL, src, strlen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, src, strlen(src));
-      *dst = (char*) * dst - tlen;
+      *dst = (char*)*dst - tlen;
       break;
-#endif
     default:
       TASSERT(0);
       break;

source/libs/index/src/indexTfile.c

@@ -20,6 +20,7 @@ p *
 #include "indexFstCountingWriter.h"
 #include "indexUtil.h"
 #include "taosdef.h"
+#include "taoserror.h"
 #include "tcoding.h"
 #include "tcompare.h"
 
@@ -472,16 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
 
     int32_t sz = 0;
     char*   ch = (char*)fstSliceData(s, &sz);
-    char*   tmp = taosMemoryCalloc(1, sz + 1);
-    memcpy(tmp, ch, sz);
+    // char*   tmp = taosMemoryCalloc(1, sz + 1);
+    // memcpy(tmp, ch, sz);
 
-    if (0 != strncmp(tmp, p, skip)) {
+    if (0 != strncmp(ch, p, skip)) {
       swsResultDestroy(rt);
-      taosMemoryFree(tmp);
+      // taosMemoryFree(tmp);
       break;
     }
 
-    TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
+    TExeCond cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
 
     if (MATCH == cond) {
       tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
@@ -490,7 +491,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
       swsResultDestroy(rt);
       break;
     }
-    taosMemoryFree(tmp);
+    // taosMemoryFree(tmp);
     swsResultDestroy(rt);
   }
   streamWithStateDestroy(st);
@@ -533,10 +534,12 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
   tfileGenFileFullName(fullname, path, suid, colName, version);
 
   WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
-  indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
   if (wc == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    indexError("failed to open readonly file: %s, reason: %s", fullname, terrstr());
     return NULL;
   }
+  indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
 
   TFileReader* reader = tfileReaderCreate(wc);
   return reader;
@@ -613,9 +616,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
     if (tfileWriteData(tw, v) != 0) {
       indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset,
                  (int)taosArrayGetSize(v->tableId));
-      // printf("write faile\n");
     } else {
-      // printf("write sucee\n");
       // indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset,
       //          (int)taosArrayGetSize(v->tableId));
 

source/libs/index/test/jsonUT.cc

@@ -553,7 +553,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
     float       val = 2.0;
     std::string colName("test1");
     for (int i = 0; i < 1000; i++) {
-      WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i);
+      WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i + 1000);
     }
   }
   {

source/libs/tdb/src/db/tdbPCache.c

@@ -15,10 +15,10 @@
 #include "tdbInt.h"
 
 struct SPCache {
-  int         pageSize;
-  int         cacheSize;
+  int         szPage;
+  int         nPages;
+  SPage     **aPage;
   tdb_mutex_t mutex;
-  SPage      *pList;
   int         nFree;
   SPage      *pFree;
   int         nPage;
@@ -52,13 +52,14 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
   void    *pPtr;
   SPage   *pPgHdr;
 
-  pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache));
+  pCache = (SPCache *)tdbOsCalloc(1, sizeof(*pCache) + sizeof(SPage *) * cacheSize);
   if (pCache == NULL) {
     return -1;
   }
 
-  pCache->pageSize = pageSize;
-  pCache->cacheSize = cacheSize;
+  pCache->szPage = pageSize;
+  pCache->nPages = cacheSize;
+  pCache->aPage = (SPage **)&pCache[1];
 
   if (tdbPCacheOpenImpl(pCache) < 0) {
     tdbOsFree(pCache);
@@ -84,7 +85,7 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
 
   pPage = tdbPCacheFetchImpl(pCache, pPgid, pTxn);
   if (pPage) {
-    TDB_REF_PAGE(pPage);
+    tdbRefPage(pPage);
   }
 
   tdbPCacheUnlock(pCache);
@@ -97,7 +98,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
 
   ASSERT(pTxn);
 
-  nRef = TDB_UNREF_PAGE(pPage);
+  nRef = tdbUnrefPage(pPage);
   ASSERT(nRef >= 0);
 
   if (nRef == 0) {
@@ -105,7 +106,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
 
     // test the nRef again to make sure
     // it is safe th handle the page
-    nRef = TDB_GET_PAGE_REF(pPage);
+    nRef = tdbGetPageRef(pPage);
     if (nRef == 0) {
       if (pPage->isLocal) {
         tdbPCacheUnpinPage(pCache, pPage);
@@ -123,7 +124,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
   }
 }
 
-int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
+int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->szPage; }
 
 static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
   int    ret = 0;
@@ -168,7 +169,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
 
   // 4. Try a create new page
   if (!pPage) {
-    ret = tdbPageCreate(pCache->pageSize, &pPage, pTxn->xMalloc, pTxn->xArg);
+    ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg);
     if (ret < 0) {
       // TODO
       ASSERT(0);
@@ -178,7 +179,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
     // init the page fields
     pPage->isAnchor = 0;
     pPage->isLocal = 0;
-    TDB_INIT_PAGE_REF(pPage);
+    pPage->nRef = 0;
+    pPage->id = -1;
   }
 
   // 5. Page here are just created from a free list
@@ -212,20 +214,25 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
 }
 
 static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) {
-  if (!PAGE_IS_PINNED(pPage)) {
+  if (pPage->pLruNext != NULL) {
+    ASSERT(tdbGetPageRef(pPage) == 0);
+
     pPage->pLruPrev->pLruNext = pPage->pLruNext;
     pPage->pLruNext->pLruPrev = pPage->pLruPrev;
     pPage->pLruNext = NULL;
 
     pCache->nRecyclable--;
+
+    tdbTrace("pin page %d", pPage->id);
   }
 }
 
 static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
   i32 nRef;
 
+  ASSERT(pPage->isLocal);
   ASSERT(!pPage->isDirty);
-  ASSERT(TDB_GET_PAGE_REF(pPage) == 0);
+  ASSERT(tdbGetPageRef(pPage) == 0);
 
   ASSERT(pPage->pLruNext == NULL);
 
@@ -235,6 +242,8 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
   pCache->lru.pLruNext = pPage;
 
   pCache->nRecyclable++;
+
+  tdbTrace("unpin page %d", pPage->id);
 }
 
 static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
@@ -248,6 +257,8 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
   *ppPage = pPage->pHashNext;
 
   pCache->nPage--;
+
+  tdbTrace("remove page %d to hash", pPage->id);
 }
 
 static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) {
@@ -259,6 +270,8 @@ static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) {
   pCache->pgHash[h] = pPage;
 
   pCache->nPage++;
+
+  tdbTrace("add page %d to hash", pPage->id);
 }
 
 static int tdbPCacheOpenImpl(SPCache *pCache) {
@@ -272,8 +285,8 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
   // Open the free list
   pCache->nFree = 0;
   pCache->pFree = NULL;
-  for (int i = 0; i < pCache->cacheSize; i++) {
-    ret = tdbPageCreate(pCache->pageSize, &pPage, tdbDefaultMalloc, NULL);
+  for (int i = 0; i < pCache->nPages; i++) {
+    ret = tdbPageCreate(pCache->szPage, &pPage, tdbDefaultMalloc, NULL);
     if (ret < 0) {
       // TODO: handle error
       return -1;
@@ -282,7 +295,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
     // pPage->pgid = 0;
     pPage->isAnchor = 0;
     pPage->isLocal = 1;
-    TDB_INIT_PAGE_REF(pPage);
+    pPage->nRef = 0;
     pPage->pHashNext = NULL;
     pPage->pLruNext = NULL;
     pPage->pLruPrev = NULL;
@@ -294,13 +307,13 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
     pCache->nFree++;
 
     // add to local list
-    pPage->pCacheNext = pCache->pList;
-    pCache->pList = pPage;
+    pPage->id = i;
+    pCache->aPage[i] = pPage;
   }
 
   // Open the hash table
   pCache->nPage = 0;
-  pCache->nHash = pCache->cacheSize < 8 ? 8 : pCache->cacheSize;
+  pCache->nHash = pCache->nPages < 8 ? 8 : pCache->nPages;
   pCache->pgHash = (SPage **)tdbOsCalloc(pCache->nHash, sizeof(SPage *));
   if (pCache->pgHash == NULL) {
     // TODO
@@ -317,11 +330,11 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
 }
 
 static int tdbPCacheCloseImpl(SPCache *pCache) {
-  SPage *pPage;
-
-  for (pPage = pCache->pList; pPage; pPage = pCache->pList) {
-    pCache->pList = pPage->pCacheNext;
-    tdbPageDestroy(pPage, tdbDefaultFree, NULL);
+  for (i32 iPage = 0; iPage < pCache->nPages; iPage++) {
+    if (pCache->aPage[iPage]) {
+      tdbPageDestroy(pCache->aPage[iPage], tdbDefaultFree, NULL);
+      pCache->aPage[iPage] = NULL;
+    }
   }
 
   tdbOsFree(pCache->pgHash);

source/libs/tdb/src/db/tdbPager.c

@@ -149,7 +149,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
   if (pPage->isDirty) return 0;
 
   // ref page one more time so the page will not be release
-  TDB_REF_PAGE(pPage);
+  tdbRefPage(pPage);
 
   // Set page as dirty
   pPage->isDirty = 1;

source/libs/tdb/src/inc/tdbInt.h

@@ -18,10 +18,23 @@
 
 #include "tdb.h"
 
+#include "tlog.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+// clang-format off
+extern int32_t tdbDebugFlag;
+
+#define tdbFatal(...) do { if (tdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
+#define tdbError(...) do { if (tdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
+#define tdbWarn(...)  do { if (tdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
+#define tdbInfo(...)  do { if (tdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
+#define tdbDebug(...) do { if (tdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tdbTrace(...) do { if (tdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tdbDebugFlag, __VA_ARGS__); }} while(0)
+// clang-format on
+
 typedef int8_t   i8;
 typedef int16_t  i16;
 typedef int32_t  i32;
@@ -161,25 +174,21 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
 int  tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
 
 // tdbPCache.c ====================================
-#define TDB_PCACHE_PAGE \
-  u8      isAnchor;     \
-  u8      isLocal;      \
-  u8      isDirty;      \
-  i32     nRef;         \
-  SPage  *pCacheNext;   \
-  SPage  *pFreeNext;    \
-  SPage  *pHashNext;    \
-  SPage  *pLruNext;     \
-  SPage  *pLruPrev;     \
-  SPage  *pDirtyNext;   \
-  SPager *pPager;       \
-  SPgid   pgid;
+#define TDB_PCACHE_PAGE    \
+  u8           isAnchor;   \
+  u8           isLocal;    \
+  u8           isDirty;    \
+  volatile i32 nRef;       \
+  i32          id;         \
+  SPage       *pFreeNext;  \
+  SPage       *pHashNext;  \
+  SPage       *pLruNext;   \
+  SPage       *pLruPrev;   \
+  SPage       *pDirtyNext; \
+  SPager      *pPager;     \
+  SPgid        pgid;
 
 // For page ref
-#define TDB_INIT_PAGE_REF(pPage) ((pPage)->nRef = 0)
-#define TDB_REF_PAGE(pPage)      atomic_add_fetch_32(&((pPage)->nRef), 1)
-#define TDB_UNREF_PAGE(pPage)    atomic_sub_fetch_32(&((pPage)->nRef), 1)
-#define TDB_GET_PAGE_REF(pPage)  atomic_load_32(&((pPage)->nRef))
 
 int    tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
 int    tdbPCacheClose(SPCache *pCache);
@@ -246,6 +255,20 @@ struct SPage {
   TDB_PCACHE_PAGE
 };
 
+static inline i32 tdbRefPage(SPage *pPage) {
+  i32 nRef = atomic_add_fetch_32(&((pPage)->nRef), 1);
+  tdbTrace("ref page %d, nRef %d", pPage->id, nRef);
+  return nRef;
+}
+
+static inline i32 tdbUnrefPage(SPage *pPage) {
+  i32 nRef = atomic_sub_fetch_32(&((pPage)->nRef), 1);
+  tdbTrace("unref page %d, nRef %d", pPage->id, nRef);
+  return nRef;
+}
+
+#define tdbGetPageRef(pPage) atomic_load_32(&((pPage)->nRef))
+
 // For page lock
 #define P_LOCK_SUCC 0
 #define P_LOCK_BUSY 1

source/util/src/terror.c

@@ -470,6 +470,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE,         "udf load failure")
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE,            "udf invalid state")
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT,            "udf invalid function input")
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE,           "udf no function handle")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE,          "udf invalid bufsize")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE,      "udf invalid output type")
+
 //schemaless
 TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE,    "Invalid line protocol type")
 TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE,   "Invalid timestamp precision type")

source/util/src/tlog.c

@@ -14,8 +14,8 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "os.h"
 #include "tlog.h"
+#include "os.h"
 #include "tutil.h"
 
 #define LOG_MAX_LINE_SIZE             (1024)
@@ -90,6 +90,7 @@ int32_t qDebugFlag = 131;
 int32_t wDebugFlag = 135;
 int32_t sDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
+int32_t tdbDebugFlag = 131;
 int32_t tqDebugFlag = 135;
 int32_t fsDebugFlag = 135;
 int32_t metaDebugFlag = 135;

tests/system-test/2-query/union.py

@@ -35,53 +35,26 @@ class TDTestCase:
         for char_col in CHAR_COL:
             query_condition.extend(
                 (
-                    f"{tbname}.{char_col}",
-                    f"upper( {tbname}.{char_col} )",
-                    f"char_length( {tbname}.{char_col} )",
-                    f"concat( {tbname}.{char_col}, {tbname}.{char_col} )",
-                    f"concat_ws( '_', {tbname}.{char_col}, {tbname}.{char_col} )",
-                    f"length( {tbname}.{char_col} )",
-                    f"lower( {tbname}.{char_col} )",
-                    f"ltrim( {tbname}.{char_col} )",
                     f"rtrim( {tbname}.{char_col} )",
                     f"substr( {tbname}.{char_col}, 1 )",
                     f"count( {tbname}.{char_col} )",
                     f"cast( {tbname}.{char_col} as nchar(3) )",
-                    f"cast( {tbname}.{char_col} as nchar(8) )",
                 )
             )
-            query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL)
-            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
-            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL )
 
         for num_col in NUM_COL:
             query_condition.extend(
                 (
                     f"{tbname}.{num_col}",
-                    f"ceil( {tbname}.{num_col} )",
-                    f"abs( {tbname}.{num_col} )",
-                    f"acos( {tbname}.{num_col} )",
-                    f"asin( {tbname}.{num_col} )",
-                    f"atan( {tbname}.{num_col} )",
-                    f"cos( {tbname}.{num_col} )",
                     f"floor( {tbname}.{num_col} )",
                     f"log( {tbname}.{num_col},  {tbname}.{num_col})",
                     f"sin( {tbname}.{num_col} )",
                     f"sqrt( {tbname}.{num_col} )",
-                    f"tan( {tbname}.{num_col} )",
-                    f"round( {tbname}.{num_col} )",
-                    f"max( {tbname}.{num_col} )",
-                    f"sum( {tbname}.{num_col} )",
-                    f"count( {tbname}.{num_col} )",
-                    f"min( {tbname}.{num_col} )",
                 )
             )
-            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_2}" for num_col_2 in NUM_COL )
-            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{char_col} " for char_col in CHAR_COL )
 
         query_condition.extend(
             (
-                ''' "test1234!@#$%^&*():'><?/.,][}{" ''',
                 ''' "test12" ''',
                 # 1010,
             )
@@ -208,8 +181,8 @@ class TDTestCase:
                         )
                     )
 
-        return filter(None, sqls)
-        # return list(filter(None, sqls))
+        # return filter(None, sqls)
+        return list(filter(None, sqls))
 
     def __get_type(self, col):
         if tdSql.cursor.istype(col, "BOOL"):
@@ -245,34 +218,55 @@ class TDTestCase:
 
     def union_check(self):
         sqls = self.sql_list()
-        for sql1 in sqls:
-            tdSql.query(sql1)
+        for i in range(len(sqls)):
+            tdSql.query(sqls[i])
             res1_type = self.__get_type(0)
-            for sql2 in sqls:
-                tdSql.query(sql2)
-                union_type = False
+            for j in range(len(sqls[i:])):
+                tdSql.query(sqls[j+i])
+                order_union_type = False
+                rev_order_type = False
+                all_union_type = False
                 res2_type =  self.__get_type(0)
 
-                if res1_type in  ( "BIGINT" , "NCHAR" ):
-                    union_type = True
-                elif res2_type == res1_type:
-                    union_type = True
+                if res2_type == res1_type:
+                    all_union_type = True
+                elif res1_type in ( "BIGINT" , "NCHAR" ) and res2_type in ("BIGINT" , "NCHAR"):
+                    all_union_type = True
+                elif res1_type in ("BIGINT", "NCHAR"):
+                    order_union_type = True
+                elif res2_type in ("BIGINT", "NCHAR"):
+                    rev_order_type = True
                 elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
-                    union_type = True
+                    order_union_type = True
+                elif res2_type == "TIMESAMP" and res1_type not in ("BINARY", "NCHAR"):
+                    rev_order_type = True
                 elif res1_type == "BINARY" and res2_type != "NCHAR":
-                    union_type = True
+                    order_union_type = True
+                elif res2_type == "BINARY" and res1_type != "NCHAR":
+                    rev_order_type = True
 
-                if union_type:
-                    tdSql.query(f"{sql1} union {sql2}")
+                if all_union_type:
+                    tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
+                    tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
                     tdSql.checkCols(1)
-                    tdSql.query(f"{sql1} union all {sql2}")
+                    tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
+                    tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
+                    tdSql.checkCols(1)
+                elif order_union_type:
+                    tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
+                    tdSql.checkCols(1)
+                elif rev_order_type:
+                    tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
                     tdSql.checkCols(1)
                 else:
-                    tdSql.error(f"{sql1} union {sql2}")
+                    tdSql.error(f"{sqls[i]} union {sqls[j+i]}")
 
     def __test_error(self):
 
-
         tdSql.error( "show tables union show tables" )
         tdSql.error( "create table errtb1 union all create table errtb2" )
         tdSql.error( "drop table ct1 union all drop table ct3" )

tests/system-test/2-query/union2.py

+import datetime
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+PRIMARY_COL = "ts"
+
+INT_COL     = "c1"
+BINT_COL    = "c2"
+SINT_COL    = "c3"
+TINT_COL    = "c4"
+FLOAT_COL   = "c5"
+DOUBLE_COL  = "c6"
+BOOL_COL    = "c7"
+
+BINARY_COL  = "c8"
+NCHAR_COL   = "c9"
+TS_COL      = "c10"
+
+NUM_COL     = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
+CHAR_COL    = [ BINARY_COL, NCHAR_COL, ]
+BOOLEAN_COL = [ BOOL_COL, ]
+TS_TYPE_COL = [ TS_COL, ]
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+
+    def __query_condition(self,tbname):
+        query_condition = []
+        for char_col in CHAR_COL:
+            query_condition.extend(
+                (
+                    f"{tbname}.{char_col}",
+                    f"upper( {tbname}.{char_col} )",
+                    f"char_length( {tbname}.{char_col} )",
+                    f"concat( {tbname}.{char_col}, {tbname}.{char_col} )",
+                )
+            )
+
+        for num_col in NUM_COL:
+            query_condition.extend(
+                (
+                    f"{tbname}.{num_col}",
+                    f"ceil( {tbname}.{num_col} )",
+                    f"abs( {tbname}.{num_col} )",
+                    f"acos( {tbname}.{num_col} )",
+                    f"max( {tbname}.{num_col} )",
+                )
+            )
+
+        query_condition.extend(
+            (
+                # 1010,
+            )
+        )
+
+        return query_condition
+
+    def __join_condition(self, tb_list, filter=PRIMARY_COL, INNER=False):
+        table_reference = tb_list[0]
+        join_condition = table_reference
+        join = "inner join" if INNER else "join"
+        for i in range(len(tb_list[1:])):
+            join_condition += f" {join} {tb_list[i+1]} on {table_reference}.{filter}={tb_list[i+1]}.{filter}"
+
+        return join_condition
+
+    def __where_condition(self, col=None, tbname=None, query_conditon=None):
+        if query_conditon and isinstance(query_conditon, str):
+            if query_conditon.startswith("count"):
+                query_conditon = query_conditon[6:-1]
+            elif query_conditon.startswith("max"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("sum"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("min"):
+                query_conditon = query_conditon[4:-1]
+
+
+        if query_conditon:
+            return f" where {query_conditon} is not null"
+        if col in NUM_COL:
+            return f" where abs( {tbname}.{col} ) >= 0"
+        if col in CHAR_COL:
+            return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
+        if col in BOOLEAN_COL:
+            return f" where {tbname}.{col} in (false, true)  "
+        if col in TS_TYPE_COL or col in PRIMARY_COL:
+            return f" where cast( {tbname}.{col} as binary(16) ) is not null "
+
+        return ""
+
+
+    def __group_condition(self, col, having = None):
+        if isinstance(col, str):
+            if col.startswith("count"):
+                col = col[6:-1]
+            elif col.startswith("max"):
+                col = col[4:-1]
+            elif col.startswith("sum"):
+                col = col[4:-1]
+            elif col.startswith("min"):
+                col = col[4:-1]
+        return f" group by {col} having {having}" if having else f" group by {col} "
+
+    def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""):
+        if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]:
+            return
+        return f"select {select_clause} from {from_clause} {where_condition} {group_condition}"
+
+
+    @property
+    def __join_tblist(self):
+        return [
+            ["ct1", "ct2"],
+            ["ct1", "ct4"],
+            ["ct1", "t1"],
+            ["ct2", "ct4"],
+            ["ct2", "t1"],
+            ["ct4", "t1"],
+            # ["ct1", "ct2", "ct4"],
+            # ["ct1", "ct2", "t1"],
+            # ["ct1", "ct4", "t1"],
+            # ["ct2", "ct4", "t1"],
+            # ["ct1", "ct2", "ct4", "t1"],
+        ]
+
+    @property
+    def __tb_liast(self):
+        return [
+            "ct1",
+            "ct2",
+            "ct4",
+            "t1",
+        ]
+
+    def sql_list(self):
+        sqls = []
+        __join_tblist = self.__join_tblist
+        for join_tblist in __join_tblist:
+            for join_tb in join_tblist:
+                select_claus_list = self.__query_condition(join_tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition( col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb),
+
+                        )
+                    )
+        __no_join_tblist = self.__tb_liast
+        for tb in __no_join_tblist:
+                select_claus_list = self.__query_condition(tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition(col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb),
+                        )
+                    )
+
+        # return filter(None, sqls)
+        return list(filter(None, sqls))
+
+    def __get_type(self, col):
+        if tdSql.cursor.istype(col, "BOOL"):
+            return "BOOL"
+        if tdSql.cursor.istype(col, "INT"):
+            return "INT"
+        if tdSql.cursor.istype(col, "BIGINT"):
+            return "BIGINT"
+        if tdSql.cursor.istype(col, "TINYINT"):
+            return "TINYINT"
+        if tdSql.cursor.istype(col, "SMALLINT"):
+            return "SMALLINT"
+        if tdSql.cursor.istype(col, "FLOAT"):
+            return "FLOAT"
+        if tdSql.cursor.istype(col, "DOUBLE"):
+            return "DOUBLE"
+        if tdSql.cursor.istype(col, "BINARY"):
+            return "BINARY"
+        if tdSql.cursor.istype(col, "NCHAR"):
+            return "NCHAR"
+        if tdSql.cursor.istype(col, "TIMESTAMP"):
+            return "TIMESTAMP"
+        if tdSql.cursor.istype(col, "JSON"):
+            return "JSON"
+        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
+            return "TINYINT UNSIGNED"
+        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
+            return "SMALLINT UNSIGNED"
+        if tdSql.cursor.istype(col, "INT UNSIGNED"):
+            return "INT UNSIGNED"
+        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
+            return "BIGINT UNSIGNED"
+
+    def union_check(self):
+        sqls = self.sql_list()
+        for i in range(len(sqls)):
+            tdSql.query(sqls[i])
+            res1_type = self.__get_type(0)
+            for j in range(len(sqls[i:])):
+                tdSql.query(sqls[j+i])
+                order_union_type = False
+                rev_order_type = False
+                all_union_type = False
+                res2_type =  self.__get_type(0)
+
+                if res2_type == res1_type:
+                    all_union_type = True
+                elif res1_type in ( "BIGINT" , "NCHAR" ) and res2_type in ("BIGINT" , "NCHAR"):
+                    all_union_type = True
+                elif res1_type in ("BIGINT", "NCHAR"):
+                    order_union_type = True
+                elif res2_type in ("BIGINT", "NCHAR"):
+                    rev_order_type = True
+                elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
+                    order_union_type = True
+                elif res2_type == "TIMESAMP" and res1_type not in ("BINARY", "NCHAR"):
+                    rev_order_type = True
+                elif res1_type == "BINARY" and res2_type != "NCHAR":
+                    order_union_type = True
+                elif res2_type == "BINARY" and res1_type != "NCHAR":
+                    rev_order_type = True
+
+                if all_union_type:
+                    tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
+                    tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
+                    tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
+                    tdSql.checkCols(1)
+                elif order_union_type:
+                    tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
+                    tdSql.checkCols(1)
+                elif rev_order_type:
+                    tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
+                    tdSql.checkCols(1)
+                else:
+                    tdSql.error(f"{sqls[i]} union {sqls[j+i]}")
+
+    def __test_error(self):
+
+        tdSql.error( "show tables union show tables" )
+        tdSql.error( "create table errtb1 union all create table errtb2" )
+        tdSql.error( "drop table ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all '' " )
+        tdSql.error( " '' union all select c1 from ct1 " )
+        tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
+
+    def all_test(self):
+        self.__test_error()
+        self.union_check()
+
+
+    def __create_tb(self):
+
+        tdLog.printNoPrefix("==========step1:create table")
+        create_stb_sql  =  f'''create table stb1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            ) tags (t1 int)
+            '''
+        create_ntb_sql = f'''create table t1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            )
+            '''
+        tdSql.execute(create_stb_sql)
+        tdSql.execute(create_ntb_sql)
+
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+            { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}
+
+    def __insert_data(self, rows):
+        now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
+        for i in range(rows):
+            tdSql.execute(
+                f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i},  {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+        tdSql.execute(
+            f'''insert into ct1 values
+            ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
+            ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct4 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time +  7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
+                )
+            (
+                { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
+                )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct2 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
+                { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
+                { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+        for i in range(rows):
+            insert_data = f'''insert into t1 values
+                ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
+                "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
+                '''
+            tdSql.execute(insert_data)
+        tdSql.execute(
+            f'''insert into t1 values
+            ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
+                "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
+                "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        self.__create_tb()
+
+        tdLog.printNoPrefix("==========step2:insert data")
+        self.rows = 10
+        self.__insert_data(self.rows)
+
+        tdLog.printNoPrefix("==========step3:all check")
+        self.all_test()
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.execute("use db")
+
+        tdLog.printNoPrefix("==========step4:after wal, all check again ")
+        self.all_test()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/2-query/union3.py

+import datetime
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+PRIMARY_COL = "ts"
+
+INT_COL     = "c1"
+BINT_COL    = "c2"
+SINT_COL    = "c3"
+TINT_COL    = "c4"
+FLOAT_COL   = "c5"
+DOUBLE_COL  = "c6"
+BOOL_COL    = "c7"
+
+BINARY_COL  = "c8"
+NCHAR_COL   = "c9"
+TS_COL      = "c10"
+
+NUM_COL     = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
+CHAR_COL    = [ BINARY_COL, NCHAR_COL, ]
+BOOLEAN_COL = [ BOOL_COL, ]
+TS_TYPE_COL = [ TS_COL, ]
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+
+    def __query_condition(self,tbname):
+        query_condition = []
+        for char_col in CHAR_COL:
+            query_condition.extend(
+                (
+                    f"concat_ws( '_', {tbname}.{char_col}, {tbname}.{char_col} )",
+                    f"length( {tbname}.{char_col} )",
+                    f"lower( {tbname}.{char_col} )",
+                    f"ltrim( {tbname}.{char_col} )",
+                )
+            )
+
+        for num_col in NUM_COL:
+            query_condition.extend(
+                (
+                    f"asin( {tbname}.{num_col} )",
+                    f"atan( {tbname}.{num_col} )",
+                    f"cos( {tbname}.{num_col} )",
+                    f"sum( {tbname}.{num_col} )",
+                )
+            )
+            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_2}" for num_col_2 in NUM_COL )
+
+        query_condition.extend(
+            (
+                ''' "test1234!@#$%^&*():'><?/.,][}{" ''',
+                # 1010,
+            )
+        )
+
+        return query_condition
+
+    def __join_condition(self, tb_list, filter=PRIMARY_COL, INNER=False):
+        table_reference = tb_list[0]
+        join_condition = table_reference
+        join = "inner join" if INNER else "join"
+        for i in range(len(tb_list[1:])):
+            join_condition += f" {join} {tb_list[i+1]} on {table_reference}.{filter}={tb_list[i+1]}.{filter}"
+
+        return join_condition
+
+    def __where_condition(self, col=None, tbname=None, query_conditon=None):
+        if query_conditon and isinstance(query_conditon, str):
+            if query_conditon.startswith("count"):
+                query_conditon = query_conditon[6:-1]
+            elif query_conditon.startswith("max"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("sum"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("min"):
+                query_conditon = query_conditon[4:-1]
+
+
+        if query_conditon:
+            return f" where {query_conditon} is not null"
+        if col in NUM_COL:
+            return f" where abs( {tbname}.{col} ) >= 0"
+        if col in CHAR_COL:
+            return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
+        if col in BOOLEAN_COL:
+            return f" where {tbname}.{col} in (false, true)  "
+        if col in TS_TYPE_COL or col in PRIMARY_COL:
+            return f" where cast( {tbname}.{col} as binary(16) ) is not null "
+
+        return ""
+
+
+    def __group_condition(self, col, having = None):
+        if isinstance(col, str):
+            if col.startswith("count"):
+                col = col[6:-1]
+            elif col.startswith("max"):
+                col = col[4:-1]
+            elif col.startswith("sum"):
+                col = col[4:-1]
+            elif col.startswith("min"):
+                col = col[4:-1]
+        return f" group by {col} having {having}" if having else f" group by {col} "
+
+    def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""):
+        if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]:
+            return
+        return f"select {select_clause} from {from_clause} {where_condition} {group_condition}"
+
+
+    @property
+    def __join_tblist(self):
+        return [
+            ["ct1", "ct2"],
+            ["ct1", "ct4"],
+            ["ct1", "t1"],
+            ["ct2", "ct4"],
+            ["ct2", "t1"],
+            ["ct4", "t1"],
+            # ["ct1", "ct2", "ct4"],
+            # ["ct1", "ct2", "t1"],
+            # ["ct1", "ct4", "t1"],
+            # ["ct2", "ct4", "t1"],
+            # ["ct1", "ct2", "ct4", "t1"],
+        ]
+
+    @property
+    def __tb_liast(self):
+        return [
+            "ct1",
+            "ct2",
+            "ct4",
+            "t1",
+        ]
+
+    def sql_list(self):
+        sqls = []
+        __join_tblist = self.__join_tblist
+        for join_tblist in __join_tblist:
+            for join_tb in join_tblist:
+                select_claus_list = self.__query_condition(join_tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition( col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb),
+
+                        )
+                    )
+        __no_join_tblist = self.__tb_liast
+        for tb in __no_join_tblist:
+                select_claus_list = self.__query_condition(tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition(col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb),
+                        )
+                    )
+
+        # return filter(None, sqls)
+        return list(filter(None, sqls))
+
+    def __get_type(self, col):
+        if tdSql.cursor.istype(col, "BOOL"):
+            return "BOOL"
+        if tdSql.cursor.istype(col, "INT"):
+            return "INT"
+        if tdSql.cursor.istype(col, "BIGINT"):
+            return "BIGINT"
+        if tdSql.cursor.istype(col, "TINYINT"):
+            return "TINYINT"
+        if tdSql.cursor.istype(col, "SMALLINT"):
+            return "SMALLINT"
+        if tdSql.cursor.istype(col, "FLOAT"):
+            return "FLOAT"
+        if tdSql.cursor.istype(col, "DOUBLE"):
+            return "DOUBLE"
+        if tdSql.cursor.istype(col, "BINARY"):
+            return "BINARY"
+        if tdSql.cursor.istype(col, "NCHAR"):
+            return "NCHAR"
+        if tdSql.cursor.istype(col, "TIMESTAMP"):
+            return "TIMESTAMP"
+        if tdSql.cursor.istype(col, "JSON"):
+            return "JSON"
+        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
+            return "TINYINT UNSIGNED"
+        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
+            return "SMALLINT UNSIGNED"
+        if tdSql.cursor.istype(col, "INT UNSIGNED"):
+            return "INT UNSIGNED"
+        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
+            return "BIGINT UNSIGNED"
+
+    def union_check(self):
+        sqls = self.sql_list()
+        for i in range(len(sqls)):
+            tdSql.query(sqls[i])
+            res1_type = self.__get_type(0)
+            for j in range(len(sqls[i:])):
+                tdSql.query(sqls[j+i])
+                order_union_type = False
+                rev_order_type = False
+                all_union_type = False
+                res2_type =  self.__get_type(0)
+
+                if res2_type == res1_type:
+                    all_union_type = True
+                elif res1_type in ( "BIGINT" , "NCHAR" ) and res2_type in ("BIGINT" , "NCHAR"):
+                    all_union_type = True
+                elif res1_type in ("BIGINT", "NCHAR"):
+                    order_union_type = True
+                elif res2_type in ("BIGINT", "NCHAR"):
+                    rev_order_type = True
+                elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
+                    order_union_type = True
+                elif res2_type == "TIMESAMP" and res1_type not in ("BINARY", "NCHAR"):
+                    rev_order_type = True
+                elif res1_type == "BINARY" and res2_type != "NCHAR":
+                    order_union_type = True
+                elif res2_type == "BINARY" and res1_type != "NCHAR":
+                    rev_order_type = True
+
+                if all_union_type:
+                    tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
+                    tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
+                    tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
+                    tdSql.checkCols(1)
+                elif order_union_type:
+                    tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
+                    tdSql.checkCols(1)
+                elif rev_order_type:
+                    tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
+                    tdSql.checkCols(1)
+                else:
+                    tdSql.error(f"{sqls[i]} union {sqls[j+i]}")
+
+    def __test_error(self):
+
+        tdSql.error( "show tables union show tables" )
+        tdSql.error( "create table errtb1 union all create table errtb2" )
+        tdSql.error( "drop table ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all '' " )
+        tdSql.error( " '' union all select c1 from ct1 " )
+        tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
+
+    def all_test(self):
+        self.__test_error()
+        self.union_check()
+
+
+    def __create_tb(self):
+
+        tdLog.printNoPrefix("==========step1:create table")
+        create_stb_sql  =  f'''create table stb1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            ) tags (t1 int)
+            '''
+        create_ntb_sql = f'''create table t1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            )
+            '''
+        tdSql.execute(create_stb_sql)
+        tdSql.execute(create_ntb_sql)
+
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+            { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}
+
+    def __insert_data(self, rows):
+        now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
+        for i in range(rows):
+            tdSql.execute(
+                f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i},  {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+        tdSql.execute(
+            f'''insert into ct1 values
+            ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
+            ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct4 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time +  7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
+                )
+            (
+                { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
+                )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct2 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
+                { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
+                { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+        for i in range(rows):
+            insert_data = f'''insert into t1 values
+                ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
+                "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
+                '''
+            tdSql.execute(insert_data)
+        tdSql.execute(
+            f'''insert into t1 values
+            ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
+                "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
+                "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        self.__create_tb()
+
+        tdLog.printNoPrefix("==========step2:insert data")
+        self.rows = 10
+        self.__insert_data(self.rows)
+
+        tdLog.printNoPrefix("==========step3:all check")
+        self.all_test()
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.execute("use db")
+
+        tdLog.printNoPrefix("==========step4:after wal, all check again ")
+        self.all_test()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/2-query/union4.py

+import datetime
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+PRIMARY_COL = "ts"
+
+INT_COL     = "c1"
+BINT_COL    = "c2"
+SINT_COL    = "c3"
+TINT_COL    = "c4"
+FLOAT_COL   = "c5"
+DOUBLE_COL  = "c6"
+BOOL_COL    = "c7"
+
+BINARY_COL  = "c8"
+NCHAR_COL   = "c9"
+TS_COL      = "c10"
+
+NUM_COL     = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
+CHAR_COL    = [ BINARY_COL, NCHAR_COL, ]
+BOOLEAN_COL = [ BOOL_COL, ]
+TS_TYPE_COL = [ TS_COL, ]
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+
+    def __query_condition(self,tbname):
+        query_condition = []
+        for char_col in CHAR_COL:
+            query_condition.extend(
+                (
+                    f"cast( {tbname}.{char_col} as nchar(8) )",
+                )
+            )
+            query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL)
+            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL )
+            query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL )
+
+        for num_col in NUM_COL:
+            query_condition.extend(
+                (
+                    f"tan( {tbname}.{num_col} )",
+                    f"round( {tbname}.{num_col} )",
+                    f"count( {tbname}.{num_col} )",
+                    f"min( {tbname}.{num_col} )",
+                )
+            )
+            query_condition.extend( f"{tbname}.{num_col} + {tbname}.{char_col} " for char_col in CHAR_COL )
+
+        query_condition.extend(
+            (
+                ''' "test12" ''',
+                # 1010,
+            )
+        )
+
+        return query_condition
+
+    def __join_condition(self, tb_list, filter=PRIMARY_COL, INNER=False):
+        table_reference = tb_list[0]
+        join_condition = table_reference
+        join = "inner join" if INNER else "join"
+        for i in range(len(tb_list[1:])):
+            join_condition += f" {join} {tb_list[i+1]} on {table_reference}.{filter}={tb_list[i+1]}.{filter}"
+
+        return join_condition
+
+    def __where_condition(self, col=None, tbname=None, query_conditon=None):
+        if query_conditon and isinstance(query_conditon, str):
+            if query_conditon.startswith("count"):
+                query_conditon = query_conditon[6:-1]
+            elif query_conditon.startswith("max"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("sum"):
+                query_conditon = query_conditon[4:-1]
+            elif query_conditon.startswith("min"):
+                query_conditon = query_conditon[4:-1]
+
+
+        if query_conditon:
+            return f" where {query_conditon} is not null"
+        if col in NUM_COL:
+            return f" where abs( {tbname}.{col} ) >= 0"
+        if col in CHAR_COL:
+            return f" where lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' "
+        if col in BOOLEAN_COL:
+            return f" where {tbname}.{col} in (false, true)  "
+        if col in TS_TYPE_COL or col in PRIMARY_COL:
+            return f" where cast( {tbname}.{col} as binary(16) ) is not null "
+
+        return ""
+
+
+    def __group_condition(self, col, having = None):
+        if isinstance(col, str):
+            if col.startswith("count"):
+                col = col[6:-1]
+            elif col.startswith("max"):
+                col = col[4:-1]
+            elif col.startswith("sum"):
+                col = col[4:-1]
+            elif col.startswith("min"):
+                col = col[4:-1]
+        return f" group by {col} having {having}" if having else f" group by {col} "
+
+    def __single_sql(self, select_clause, from_clause, where_condition="", group_condition=""):
+        if isinstance(select_clause, str) and "on" not in from_clause and select_clause.split(".")[0] != from_clause.split(".")[0]:
+            return
+        return f"select {select_clause} from {from_clause} {where_condition} {group_condition}"
+
+
+    @property
+    def __join_tblist(self):
+        return [
+            ["ct1", "ct2"],
+            ["ct1", "ct4"],
+            ["ct1", "t1"],
+            ["ct2", "ct4"],
+            ["ct2", "t1"],
+            ["ct4", "t1"],
+            # ["ct1", "ct2", "ct4"],
+            # ["ct1", "ct2", "t1"],
+            # ["ct1", "ct4", "t1"],
+            # ["ct2", "ct4", "t1"],
+            # ["ct1", "ct2", "ct4", "t1"],
+        ]
+
+    @property
+    def __tb_liast(self):
+        return [
+            "ct1",
+            "ct2",
+            "ct4",
+            "t1",
+        ]
+
+    def sql_list(self):
+        sqls = []
+        __join_tblist = self.__join_tblist
+        for join_tblist in __join_tblist:
+            for join_tb in join_tblist:
+                select_claus_list = self.__query_condition(join_tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition( col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition( col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist), where_claus, having_claus),
+                            self.__single_sql(select_claus, self.__join_condition(join_tblist, INNER=True), where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb),
+
+                        )
+                    )
+        __no_join_tblist = self.__tb_liast
+        for tb in __no_join_tblist:
+                select_claus_list = self.__query_condition(tb)
+                for select_claus in select_claus_list:
+                    group_claus = self.__group_condition(col=select_claus)
+                    where_claus = self.__where_condition(query_conditon=select_claus)
+                    having_claus = self.__group_condition(col=select_claus, having=f"{select_claus} is not null")
+                    sqls.extend(
+                        (
+                            self.__single_sql(select_claus, join_tb, where_claus, group_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus, having_claus),
+                            self.__single_sql(select_claus, join_tb, where_claus),
+                            self.__single_sql(select_claus, join_tb, group_claus),
+                            self.__single_sql(select_claus, join_tb, having_claus),
+                            self.__single_sql(select_claus, join_tb),
+                        )
+                    )
+
+        # return filter(None, sqls)
+        return list(filter(None, sqls))
+
+    def __get_type(self, col):
+        if tdSql.cursor.istype(col, "BOOL"):
+            return "BOOL"
+        if tdSql.cursor.istype(col, "INT"):
+            return "INT"
+        if tdSql.cursor.istype(col, "BIGINT"):
+            return "BIGINT"
+        if tdSql.cursor.istype(col, "TINYINT"):
+            return "TINYINT"
+        if tdSql.cursor.istype(col, "SMALLINT"):
+            return "SMALLINT"
+        if tdSql.cursor.istype(col, "FLOAT"):
+            return "FLOAT"
+        if tdSql.cursor.istype(col, "DOUBLE"):
+            return "DOUBLE"
+        if tdSql.cursor.istype(col, "BINARY"):
+            return "BINARY"
+        if tdSql.cursor.istype(col, "NCHAR"):
+            return "NCHAR"
+        if tdSql.cursor.istype(col, "TIMESTAMP"):
+            return "TIMESTAMP"
+        if tdSql.cursor.istype(col, "JSON"):
+            return "JSON"
+        if tdSql.cursor.istype(col, "TINYINT UNSIGNED"):
+            return "TINYINT UNSIGNED"
+        if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"):
+            return "SMALLINT UNSIGNED"
+        if tdSql.cursor.istype(col, "INT UNSIGNED"):
+            return "INT UNSIGNED"
+        if tdSql.cursor.istype(col, "BIGINT UNSIGNED"):
+            return "BIGINT UNSIGNED"
+
+    def union_check(self):
+        sqls = self.sql_list()
+        for i in range(len(sqls)):
+            tdSql.query(sqls[i])
+            res1_type = self.__get_type(0)
+            for j in range(len(sqls[i:])):
+                tdSql.query(sqls[j+i])
+                order_union_type = False
+                rev_order_type = False
+                all_union_type = False
+                res2_type =  self.__get_type(0)
+
+                if res2_type == res1_type:
+                    all_union_type = True
+                elif res1_type in ( "BIGINT" , "NCHAR" ) and res2_type in ("BIGINT" , "NCHAR"):
+                    all_union_type = True
+                elif res1_type in ("BIGINT", "NCHAR"):
+                    order_union_type = True
+                elif res2_type in ("BIGINT", "NCHAR"):
+                    rev_order_type = True
+                elif res1_type == "TIMESAMP" and res2_type not in ("BINARY", "NCHAR"):
+                    order_union_type = True
+                elif res2_type == "TIMESAMP" and res1_type not in ("BINARY", "NCHAR"):
+                    rev_order_type = True
+                elif res1_type == "BINARY" and res2_type != "NCHAR":
+                    order_union_type = True
+                elif res2_type == "BINARY" and res1_type != "NCHAR":
+                    rev_order_type = True
+
+                if all_union_type:
+                    tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
+                    tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
+                    tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
+                    tdSql.checkCols(1)
+                elif order_union_type:
+                    tdSql.query(f"{sqls[i]} union {sqls[j+i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[i]} union all {sqls[j+i]}")
+                    tdSql.checkCols(1)
+                elif rev_order_type:
+                    tdSql.query(f"{sqls[j+i]} union {sqls[i]}")
+                    tdSql.checkCols(1)
+                    tdSql.query(f"{sqls[j+i]} union all {sqls[i]}")
+                    tdSql.checkCols(1)
+                else:
+                    tdSql.error(f"{sqls[i]} union {sqls[j+i]}")
+
+    def __test_error(self):
+
+        tdSql.error( "show tables union show tables" )
+        tdSql.error( "create table errtb1 union all create table errtb2" )
+        tdSql.error( "drop table ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all drop table ct3" )
+        tdSql.error( "select c1 from ct1 union all '' " )
+        tdSql.error( " '' union all select c1 from ct1 " )
+        tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
+
+    def all_test(self):
+        self.__test_error()
+        self.union_check()
+
+
+    def __create_tb(self):
+
+        tdLog.printNoPrefix("==========step1:create table")
+        create_stb_sql  =  f'''create table stb1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            ) tags (t1 int)
+            '''
+        create_ntb_sql = f'''create table t1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            )
+            '''
+        tdSql.execute(create_stb_sql)
+        tdSql.execute(create_ntb_sql)
+
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+            { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}
+
+    def __insert_data(self, rows):
+        now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)
+        for i in range(rows):
+            tdSql.execute(
+                f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+            tdSql.execute(
+                f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i},  {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )"
+            )
+        tdSql.execute(
+            f'''insert into ct1 values
+            ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } )
+            ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct4 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time +  7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}
+                )
+            (
+                { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}
+                )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct2 values
+            ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
+                { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
+                { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+        for i in range(rows):
+            insert_data = f'''insert into t1 values
+                ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
+                "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } )
+                '''
+            tdSql.execute(insert_data)
+        tdSql.execute(
+            f'''insert into t1 values
+            ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
+                "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }
+                )
+            (
+                { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
+                "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }
+                )
+            '''
+        )
+
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        self.__create_tb()
+
+        tdLog.printNoPrefix("==========step2:insert data")
+        self.rows = 10
+        self.__insert_data(self.rows)
+
+        tdLog.printNoPrefix("==========step3:all check")
+        self.all_test()
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.execute("use db")
+
+        tdLog.printNoPrefix("==========step4:after wal, all check again ")
+        self.all_test()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/fulltest.sh

@@ -22,8 +22,12 @@ python3 ./test.py -f 2-query/upper.py
 python3 ./test.py -f 2-query/lower.py
 python3 ./test.py -f 2-query/join.py
 python3 ./test.py -f 2-query/cast.py
-# python3 ./test.py -f 2-query/concat.py  # after wal ,crash occured
-# python3 ./test.py -f 2-query/concat_ws.py
+python3 ./test.py -f 2-query/concat.py
+python3 ./test.py -f 2-query/concat_ws.py
+# python3 ./test.py -f 2-query/union.py
+# python3 ./test.py -f 2-query/union2.py
+# python3 ./test.py -f 2-query/union3.py
+# python3 ./test.py -f 2-query/union4.py
 
 python3 ./test.py -f 2-query/timezone.py
 python3 ./test.py -f 2-query/Now.py
@@ -58,4 +62,4 @@ python3 ./test.py -f 2-query/nestedQuery.py
 
 python3 ./test.py -f 7-tmq/basic5.py
 python3 ./test.py -f 7-tmq/subscribeDb.py
-
+python3 ./test.py -f 7-tmq/subscribeDb1.py

tests/test/c/tmqSim.c

@@ -278,7 +278,7 @@ void build_consumer(SThreadInfo* pInfo) {
 
   //tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
 
-  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
+  tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
 
   // tmq_conf_set(conf, "group.id", "cgrp1");
   for (int32_t i = 0; i < pInfo->numOfKey; i++) {
@@ -322,8 +322,11 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) {
   sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName,
           pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult);
 
-  taosFprintfFile(g_fp, "== save result sql: %s \n", sqlStr);
-  
+  time_t	tTime = taosGetTimestampSec();
+  struct tm tm = *taosLocalTime(&tTime, NULL);
+  taosFprintfFile(g_fp, "# save result:	%d-%02d-%02d %02d:%02d:%02d, sql: %s\n", tm.tm_year + 1900, tm.tm_mon + 1,
+						  tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, sqlStr);
+
   TAOS_RES* pRes = taos_query(pConn, sqlStr);
   if (taos_errno(pRes) != 0) {
     pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes));