Merge pull request #15260 from taosdata/feature/TD-11274-3.0

fix: merge dup rows in client

include/common/tmsg.h

@@ -438,7 +438,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW
   return 0;
 }
 
-STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
+STSchema* tdGetSTSChemaFromSSChema(SSchema* pSchema, int32_t nCols, int32_t sver);
 
 typedef struct {
   char     name[TSDB_TABLE_FNAME_LEN];

source/common/src/tmsg.c

@@ -4941,14 +4941,14 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
   return 0;
 }
 
-STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
+STSchema *tdGetSTSChemaFromSSChema(SSchema *pSchema, int32_t nCols, int32_t sver) {
   STSchemaBuilder schemaBuilder = {0};
-  if (tdInitTSchemaBuilder(&schemaBuilder, 1) < 0) {
+  if (tdInitTSchemaBuilder(&schemaBuilder, sver) < 0) {
     return NULL;
   }
 
   for (int i = 0; i < nCols; i++) {
-    SSchema *schema = *pSchema + i;
+    SSchema *schema = pSchema + i;
     if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) {
       tdDestroyTSchemaBuilder(&schemaBuilder);
       return NULL;

source/common/src/trow.c

@@ -568,6 +568,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
   int32_t   maxVarDataLen = 0;
   int32_t   iColVal = 0;
   void     *varBuf = NULL;
+  bool      isAlloc = false;
 
   ASSERT(nColVal > 1);
 
@@ -610,8 +611,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
     ++iColVal;
   }
 
-  *ppRow = (STSRow *)taosMemoryCalloc(
-      1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1));
+  if (!(*ppRow)) {
+    *ppRow = (STSRow *)taosMemoryCalloc(
+        1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1));
+    isAlloc = true;
+  }
 
   if (!(*ppRow)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -621,7 +625,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
   if (maxVarDataLen > 0) {
     varBuf = taosMemoryMalloc(maxVarDataLen);
     if (!varBuf) {
-      taosMemoryFreeClear(*ppRow);
+      if(isAlloc) {
+        taosMemoryFreeClear(*ppRow);
+      }
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
@@ -1323,12 +1329,11 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV
   SCellVal  cv;
   SValue    value;
 
-  ASSERT(iCol > 0);
+  ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0));
 
   if (TD_IS_TP_ROW(pRow)) {
     tdSTpRowGetVal(pRow, pTColumn->colId, pTColumn->type, pTSchema->flen, pTColumn->offset, iCol - 1, &cv);
   } else if (TD_IS_KV_ROW(pRow)) {
-    ASSERT(iCol > 0);
     tdSKvRowGetVal(pRow, pTColumn->colId, iCol - 1, &cv);
   } else {
     ASSERT(0);

source/common/test/dataformatTest.cpp

@@ -116,7 +116,7 @@ STSchema *genSTSchema(int16_t nCols) {
   }
 
   STSchema *pResult = NULL;
-  pResult = tdGetSTSChemaFromSSChema(&pSchema, nCols);
+  pResult = tdGetSTSChemaFromSSChema(pSchema, nCols, 1);
 
   taosMemoryFree(pSchema);
   return pResult;

source/dnode/vnode/src/tq/tq.c

@@ -634,7 +634,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
     ASSERT(pTask->tbSink.pSchemaWrapper->pSchema);
 
     pTask->tbSink.pTSchema =
-        tdGetSTSChemaFromSSChema(&pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols);
+        tdGetSTSChemaFromSSChema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1);
     ASSERT(pTask->tbSink.pTSchema);
   }
 

source/libs/parser/src/parInsert.c

@@ -1200,7 +1200,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
 
     *gotRow = true;
 #ifdef TD_DEBUG_PRINT_ROW
-    STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&schema, spd->numOfCols);
+    STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1);
     tdSRowPrint(row, pSTSchema, __func__);
     taosMemoryFree(pSTSchema);
 #endif
@@ -1972,7 +1972,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in
       }
     }
 #ifdef TD_DEBUG_PRINT_ROW
-    STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols);
+    STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1);
     tdSRowPrint(row, pSTSchema, __func__);
     taosMemoryFree(pSTSchema);
 #endif
@@ -2057,7 +2057,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu
 
 #ifdef TD_DEBUG_PRINT_ROW
     if (rowEnd) {
-      STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols);
+      STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1);
       tdSRowPrint(row, pSTSchema, __func__);
       taosMemoryFree(pSTSchema);
     }

source/libs/parser/src/parInsertData.c

@@ -19,6 +19,7 @@
 #include "parInt.h"
 #include "parUtil.h"
 #include "querynodes.h"
+#include "tRealloc.h"
 
 #define IS_RAW_PAYLOAD(t) \
   (((int)(t)) == PAYLOAD_TYPE_RAW)  // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
@@ -34,6 +35,32 @@ typedef struct SBlockKeyInfo {
   SBlockKeyTuple* pKeyTuple;
 } SBlockKeyInfo;
 
+typedef struct {
+  int32_t   index;
+  SArray*   rowArray;  // array of merged rows(mem allocated by tRealloc/free by tFree)
+  STSchema* pSchema;
+  int64_t   tbUid;  // suid for child table, uid for normal table
+} SBlockRowMerger;
+
+static FORCE_INLINE void tdResetSBlockRowMerger(SBlockRowMerger* pMerger) {
+  if (pMerger) {
+    pMerger->index = -1;
+  }
+}
+
+static void tdFreeSBlockRowMerger(SBlockRowMerger* pMerger) {
+  if (pMerger) {
+    int32_t size = taosArrayGetSize(pMerger->rowArray);
+    for (int32_t i = 0; i < size; ++i) {
+      tFree(*(void**)taosArrayGet(pMerger->rowArray, i));
+    }
+    taosArrayDestroy(pMerger->rowArray);
+
+    taosMemoryFreeClear(pMerger->pSchema);
+    taosMemoryFree(pMerger);
+  }
+}
+
 static int32_t rowDataCompar(const void* lhs, const void* rhs) {
   TSKEY left = *(TSKEY*)lhs;
   TSKEY right = *(TSKEY*)rhs;
@@ -328,7 +355,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) {
 }
 
 // data block is disordered, sort it in ascending order
-int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) {
+static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) {
   SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData;
   int16_t     nRows = pBlocks->numOfRows;
 
@@ -396,6 +423,201 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey
   return 0;
 }
 
+static void* tdGetCurRowFromBlockMerger(SBlockRowMerger* pBlkRowMerger) {
+  if (pBlkRowMerger && (pBlkRowMerger->index >= 0)) {
+    ASSERT(pBlkRowMerger->index < taosArrayGetSize(pBlkRowMerger->rowArray));
+    return *(void**)taosArrayGet(pBlkRowMerger->rowArray, pBlkRowMerger->index);
+  }
+  return NULL;
+}
+
+static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp, int32_t nDupRows,
+                               SBlockRowMerger** pBlkRowMerger, int32_t rowSize) {
+  ASSERT(nDupRows > 1);
+  SBlockKeyTuple* pStartKeyTp = pEndKeyTp - (nDupRows - 1);
+  ASSERT(pStartKeyTp->skey == pEndKeyTp->skey);
+
+  // TODO: optimization if end row is all normal
+#if 0
+  STSRow* pEndRow = (STSRow*)pEndKeyTp->payloadAddr;
+  if(isNormal(pEndRow)) { // set the end row if it is normal and return directly
+    pStartKeyTp->payloadAddr = pEndKeyTp->payloadAddr;
+    return TSDB_CODE_SUCCESS;
+  }
+#endif
+
+  if (!(*pBlkRowMerger)) {
+    (*pBlkRowMerger) = taosMemoryCalloc(1, sizeof(**pBlkRowMerger));
+    if (!(*pBlkRowMerger)) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+    (*pBlkRowMerger)->index = -1;
+    if (!(*pBlkRowMerger)->rowArray) {
+      (*pBlkRowMerger)->rowArray = taosArrayInit(1, sizeof(void*));
+      if (!(*pBlkRowMerger)->rowArray) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return TSDB_CODE_FAILED;
+      }
+    }
+  }
+
+  if ((*pBlkRowMerger)->pSchema) {
+    if ((*pBlkRowMerger)->pSchema->version != pTableMeta->sversion) {
+      taosMemoryFreeClear((*pBlkRowMerger)->pSchema);
+    } else {
+      if ((*pBlkRowMerger)->tbUid != (pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid)) {
+        taosMemoryFreeClear((*pBlkRowMerger)->pSchema);
+      }
+    }
+  }
+
+  if (!(*pBlkRowMerger)->pSchema) {
+    (*pBlkRowMerger)->pSchema =
+        tdGetSTSChemaFromSSChema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion);
+
+    if (!(*pBlkRowMerger)->pSchema) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+    (*pBlkRowMerger)->tbUid = pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid;
+  }
+
+  void* pDestRow = NULL;
+  ++((*pBlkRowMerger)->index);
+  if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) {
+    void* pAlloc = *(void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index);
+    if (tRealloc((uint8_t**)&pAlloc, rowSize) != 0) {
+      return TSDB_CODE_FAILED;
+    }
+    pDestRow = pAlloc;
+  } else {
+    if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) {
+      return TSDB_CODE_FAILED;
+    }
+    taosArrayPush((*pBlkRowMerger)->rowArray, &pDestRow);
+  }
+
+  // merge rows to pDestRow
+  STSchema* pSchema = (*pBlkRowMerger)->pSchema;
+  SArray*   pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal));
+  for (int32_t i = 0; i < pSchema->numOfCols; ++i) {
+    SColVal colVal = {0};
+    for (int32_t j = 0; j < nDupRows; ++j) {
+      tTSRowGetVal((pEndKeyTp - j)->payloadAddr, pSchema, i, &colVal);
+      if (!colVal.isNone) {
+        break;
+      }
+    }
+    taosArrayPush(pArray, &colVal);
+  }
+  if (tdSTSRowNew(pArray, pSchema, (STSRow**)&pDestRow) < 0) {
+    taosArrayDestroy(pArray);
+    return TSDB_CODE_FAILED;
+  }
+
+  taosArrayDestroy(pArray);
+  return TSDB_CODE_SUCCESS;
+}
+
+// data block is disordered, sort it in ascending order, and merge dup rows if exists
+static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo,
+                                     SBlockRowMerger** ppBlkRowMerger) {
+  SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData;
+  STableMeta* pTableMeta = dataBuf->pTableMeta;
+  int16_t     nRows = pBlocks->numOfRows;
+
+  // size is less than the total size, since duplicated rows may be removed.
+
+  // allocate memory
+  size_t nAlloc = nRows * sizeof(SBlockKeyTuple);
+  if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) {
+    char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc);
+    if (tmp == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp;
+    pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc;
+  }
+  memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc);
+
+  tdResetSBlockRowMerger(*ppBlkRowMerger);
+
+  int32_t         extendedRowSize = getExtendedRowSize(dataBuf);
+  SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
+  char*           pBlockData = pBlocks->data + pBlocks->schemaLen;
+  int             n = 0;
+  while (n < nRows) {
+    pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData);
+    pBlkKeyTuple->payloadAddr = pBlockData;
+    pBlkKeyTuple->index = n;
+
+    // next loop
+    pBlockData += extendedRowSize;
+    ++pBlkKeyTuple;
+    ++n;
+  }
+
+  if (!dataBuf->ordered) {
+    pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
+
+    taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable);
+
+    pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
+    bool    hasDup = false;
+    int32_t nextPos = 0;
+    int32_t i = 0;
+    int32_t j = 1;
+
+    while (j < nRows) {
+      TSKEY ti = (pBlkKeyTuple + i)->skey;
+      TSKEY tj = (pBlkKeyTuple + j)->skey;
+
+      if (ti == tj) {
+        ++j;
+        continue;
+      }
+
+      if ((j - i) > 1) {
+        if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) {
+          return TSDB_CODE_FAILED;
+        }
+        (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger);
+        if (!hasDup) {
+          hasDup = true;
+        }
+        i = j;
+      } else {
+        if (hasDup) {
+          memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple));
+        }
+        ++i;
+      }
+
+      ++nextPos;
+      ++j;
+    }
+
+    if ((j - i) > 1) {
+      ASSERT((pBlkKeyTuple + i)->skey == (pBlkKeyTuple + j - 1)->skey);
+      if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) {
+        return TSDB_CODE_FAILED;
+      }
+      (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger);
+    } else if (hasDup) {
+      memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple));
+    }
+
+    dataBuf->ordered = true;
+    pBlocks->numOfRows = nextPos + 1;
+  }
+
+  dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize;
+  dataBuf->prevTS = INT64_MIN;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 // Erase the empty space reserved for binary data
 static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple,
                          bool isRawPayload) {
@@ -464,6 +686,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
   STableDataBlocks** p = taosHashIterate(pHashObj, NULL);
   STableDataBlocks*  pOneTableBlock = *p;
   SBlockKeyInfo      blkKeyInfo = {0};  // share by pOneTableBlock
+  SBlockRowMerger    *pBlkRowMerger = NULL;
+
   while (pOneTableBlock) {
     SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData;
     if (pBlocks->numOfRows > 0) {
@@ -473,6 +697,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
           getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0,
                                pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL);
       if (ret != TSDB_CODE_SUCCESS) {
+        tdFreeSBlockRowMerger(pBlkRowMerger);
         taosHashCleanup(pVnodeDataBlockHashList);
         destroyBlockArrayList(pVnodeDataBlockList);
         taosMemoryFreeClear(blkKeyInfo.pKeyTuple);
@@ -490,6 +715,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
         if (tmp != NULL) {
           dataBuf->pData = tmp;
         } else {  // failed to allocate memory, free already allocated memory and return error code
+          tdFreeSBlockRowMerger(pBlkRowMerger);
           taosHashCleanup(pVnodeDataBlockHashList);
           destroyBlockArrayList(pVnodeDataBlockList);
           taosMemoryFreeClear(dataBuf->pData);
@@ -501,7 +727,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
       if (isRawPayload) {
         sortRemoveDataBlockDupRowsRaw(pOneTableBlock);
       } else {
-        if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) {
+        if ((code = sortMergeDataBlockDupRows(pOneTableBlock, &blkKeyInfo, &pBlkRowMerger)) != 0) {
+          tdFreeSBlockRowMerger(pBlkRowMerger);
           taosHashCleanup(pVnodeDataBlockHashList);
           destroyBlockArrayList(pVnodeDataBlockList);
           taosMemoryFreeClear(dataBuf->pData);
@@ -529,6 +756,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
   }
 
   // free the table data blocks;
+  tdFreeSBlockRowMerger(pBlkRowMerger);
   taosHashCleanup(pVnodeDataBlockHashList);
   taosMemoryFreeClear(blkKeyInfo.pKeyTuple);
   *pVgDataBlocks = pVnodeDataBlockList;

tests/script/tsim/insert/update0.sim

@@ -79,8 +79,8 @@ if $rows != 3 then
   return -1
 endi
 
-if $data01 != 103 then
-  print data01 $data01 != 103
+if $data01 != 303 then
+  print data01 $data01 != 303
   return -1
 endi
 
@@ -89,8 +89,8 @@ if $data11 != 80 then
   return -1
 endi
 
-if $data21 != 40 then
-  print data21 $data21 != 40
+if $data21 != 60 then
+  print data21 $data21 != 60
   return -1
 endi
 
@@ -138,8 +138,8 @@ if $rows != 3 then
   return -1
 endi
 
-if $data01 != 103 then
-  print data01 $data01 != 103
+if $data01 != 303 then
+  print data01 $data01 != 303
   return -1
 endi
 
@@ -148,8 +148,8 @@ if $data11 != 80 then
   return -1
 endi
 
-if $data21 != 40 then
-  print data21 $data21 != 40
+if $data21 != 60 then
+  print data21 $data21 != 60
   return -1
 endi
 
@@ -208,8 +208,8 @@ if $data01 != 10 then
   return -1
 endi
 
-if $data11 != 103 then
-  print data11 $data11 != 103
+if $data11 != 303 then
+  print data11 $data11 != 303
   return -1
 endi
 
@@ -218,8 +218,8 @@ if $data21 != NULL then
   return -1
 endi
 
-if $data31 != 40 then
-  print data31 $data31 != 40
+if $data31 != 60 then
+  print data31 $data31 != 60
   return -1
 endi
 

tests/script/tsim/insert/update1_sort_merge.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print =============== create database
+sql drop database if exists d0
+sql create database d0 keep 365000d,365000d,365000d
+sql use d0
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int unsigned, c2 double, c3 binary(10), c4 nchar(10), c5 double) tags (city binary(20),district binary(20));
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct1 using stb tags("BeiJing", "ChaoYang")
+sql create table ct2 using stb tags("BeiJing", "HaiDian")
+sql create table ct3 using stb tags("BeiJing", "PingGu")
+sql create table ct4 using stb tags("BeiJing", "YanQing")
+
+sql show tables
+if $rows != 4 then 
+  print rows $rows != 4
+  return -1
+endi
+
+print =============== step 1 insert records into ct1 - taosd merge
+sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10, 20);
+sql insert into ct1(ts,c1,c2,c3,c4) values('2022-05-03 16:59:00.011', 11, NULL, 'binary', 'nchar');
+sql insert into ct1 values('2022-05-03 16:59:00.016', 16, NULL, NULL, 'nchar', NULL);
+sql insert into ct1 values('2022-05-03 16:59:00.016', 17, NULL, NULL, 'nchar', 170);
+sql insert into ct1 values('2022-05-03 16:59:00.020', 20, NULL, NULL, 'nchar', 200);
+sql insert into ct1 values('2022-05-03 16:59:00.016', 18, NULL, NULL, 'nchar', 180);
+sql insert into ct1 values('2022-05-03 16:59:00.021', 21, NULL, NULL, 'nchar', 210);
+sql insert into ct1 values('2022-05-03 16:59:00.022', 22, NULL, NULL, 'nchar', 220);
+
+print =============== step 2 insert records into ct1/ct2 - taosc merge for 2022-05-03 16:59:00.010
+sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10,10), ('2022-05-03 16:59:00.010',20,10.0), ('2022-05-03 16:59:00.010',30,NULL) ct2(ts,c1) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',20) ct1(ts,c2) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',100) ct1(ts,c3) values('2022-05-03 16:59:00.010','bin1'), ('2022-05-03 16:59:00.010','bin2') ct1(ts,c4,c5) values('2022-05-03 16:59:00.010',NULL,NULL), ('2022-05-03 16:59:00.010','nchar4',1000.01) ct2(ts,c2,c3,c4,c5) values('2022-05-03 16:59:00.010',20,'xkl','zxc',10);
+
+print =============== step 3 insert records into ct3
+sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.020', 10,10);
+sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.021', 10,10), ('2022-05-03 16:59:00.021',20,20.0);
+sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.022', 30,30), ('2022-05-03 16:59:00.022',40,40.0),('2022-05-03 16:59:00.022',50,50.0);
+sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.023', 60,60), ('2022-05-03 16:59:00.023',70,70.0),('2022-05-03 16:59:00.023',80,80.0), ('2022-05-03 16:59:00.023',90,90.0);
+sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.024', 100,100), ('2022-05-03 16:59:00.025',110,110.0),('2022-05-03 16:59:00.025',120,120.0), ('2022-05-03 16:59:00.025',130,130.0);
+sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.030', 140,140), ('2022-05-03 16:59:00.030',150,150.0),('2022-05-03 16:59:00.031',160,160.0), ('2022-05-03 16:59:00.030',170,170.0), ('2022-05-03 16:59:00.031',180,180.0);
+sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.042', 190,190), ('2022-05-03 16:59:00.041',200,200.0),('2022-05-03 16:59:00.040',210,210.0);
+sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.050', 220,220), ('2022-05-03 16:59:00.051',230,230.0),('2022-05-03 16:59:00.052',240,240.0);
+
+print =============== step 4 insert records into ct4
+sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.020', 10,'b0','n0');
+sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.021', 20,'b1','n1'), ('2022-05-03 16:59:00.021',30,'b2','n2');
+sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.022', 40,'b3','n3'), ('2022-05-03 16:59:00.022',40,'b4','n4'),('2022-05-03 16:59:00.022',50,'b5','n5');
+sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.023', 60,'b6','n6'), ('2022-05-03 16:59:00.024',70,'b7','n7'),('2022-05-03 16:59:00.024',80,'b8','n8'), ('2022-05-03 16:59:00.023',90,'b9','n9');
+
+
+
+print =============== step 5 query records of ct1 from memory(taosc and taosd merge)
+sql select * from ct1;
+print $data00 $data01 $data02 $data03 $data04 $data05
+print $data10 $data11 $data12 $data13 $data14 $data15
+print $data20 $data21 $data22 $data23 $data24 $data25
+print $data30 $data31 $data32 $data33 $data34 $data35
+print $data40 $data41 $data42 $data43 $data44 $data45
+print $data50 $data51 $data52 $data53 $data54 $data55
+
+if $rows != 6 then 
+  print rows $rows != 6
+  return -1
+endi
+
+if $data01 != 30 then
+  print data01 $data01 != 30
+  return -1
+endi
+
+if $data02 != 100.000000000 then
+  print data02 $data02 != 100.000000000
+  return -1
+endi
+
+if $data03 != bin2 then
+  print data03 $data03 != bin2
+  return -1
+endi
+
+if $data04 != nchar4 then
+  print data04 $data04 != nchar4
+  return -1
+endi
+
+if $data05 != 1000.010000000 then
+  print data05 $data05 != 1000.010000000
+  return -1
+endi
+
+if $data11 != 11 then
+  print data11 $data11 != 11
+  return -1
+endi
+
+if $data12 != NULL then
+  print data12 $data12 != NULL
+  return -1
+endi
+
+if $data13 != binary then
+  print data13 $data13 != binary
+  return -1
+endi
+
+if $data14 != nchar then
+  print data14 $data14 != nchar
+  return -1
+endi
+
+if $data15 != NULL then
+  print data15 $data15 != NULL
+  return -1
+endi
+
+if $data51 != 22 then
+  print data51 $data51 != 22
+  return -1
+endi
+
+if $data52 != NULL then
+  print data52 $data52 != NULL
+  return -1
+endi
+
+if $data53 != NULL then
+  print data53 $data53 != NULL
+  return -1
+endi
+
+if $data54 != nchar then
+  print data54 $data54 != nchar
+  return -1
+endi
+
+if $data55 != 220.000000000 then
+  print data55 $data55 != 220.000000000
+  return -1
+endi
+
+
+print =============== step 6 query records of ct2 from memory(taosc and taosd merge)
+sql select * from ct2;
+print $data00 $data01 $data02 $data03 $data04 $data05
+
+if $rows != 1 then 
+  print rows $rows != 1
+  return -1
+endi
+
+if $data01 != 20 then
+  print data01 $data01 != 20
+  return -1
+endi
+
+if $data02 != 20.000000000 then
+  print data02 $data02 != 20.000000000
+  return -1
+endi
+
+if $data03 != xkl then
+  print data03 $data03 != xkl
+  return -1
+endi
+
+if $data04 != zxc then
+  print data04 $data04 != zxc
+  return -1
+endi
+
+if $data05 != 10.000000000 then
+  print data05 $data05 != 10.000000000
+  return -1
+endi
+
+print =============== step 7 query records of ct3 from memory
+sql select * from ct3;
+print $data00 $data01 $data02 $data03 $data04 $data05
+print $data10 $data11 $data12 $data13 $data14 $data15
+print $data20 $data21 $data22 $data23 $data24 $data25
+print $data30 $data31 $data32 $data33 $data34 $data35
+print $data40 $data41 $data42 $data43 $data44 $data45
+print $data50 $data51 $data52 $data53 $data54 $data55
+print $data60 $data61 $data62 $data63 $data64 $data65
+print $data70 $data71 $data72 $data73 $data74 $data75
+print $data80 $data81 $data82 $data83 $data84 $data85
+print $data90 $data91 $data92 $data93 $data94 $data95
+print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5] 
+print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5] 
+print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5] 
+print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5] 
+
+if $rows != 14 then 
+  print rows $rows != 14
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data11 != 20 then
+  print data11 $data1 != 20
+  return -1
+endi
+
+if $data21 != 50 then
+  print data21 $data21 != 50
+  return -1
+endi
+
+if $data31 != 90 then
+  print data31 $data31 != 90
+  return -1
+endi
+
+if $data41 != 100 then
+  print data41 $data41 != 100
+  return -1
+endi
+
+if $data51 != 130 then
+  print data51 $data51 != 130
+  return -1
+endi
+
+if $data61 != 170 then
+  print data61 $data61 != 170
+  return -1
+endi
+
+if $data71 != 180 then
+  print data71 $data71 != 180
+  return -1
+endi
+
+if $data81 != 210 then
+  print data81 $data81 != 210
+  return -1
+endi
+
+if $data91 != 200 then
+  print data91 $data91 != 200
+  return -1
+endi
+
+if $data[10][1] != 190 then
+  print data[10][1] $data[10][1] != 190
+  return -1
+endi
+
+if $data[11][1] != 220 then
+  print data[11][1] $data[11][1] != 220
+  return -1
+endi
+
+if $data[12][1] != 230 then
+  print data[12][1] $data[12][1] != 230
+  return -1
+endi
+
+if $data[13][1] != 240 then
+  print data[13][1] $data[13][1] != 240
+  return -1
+endi
+
+if $data05 != 10.000000000 then
+  print data05 $data05 != 10.000000000
+  return -1
+endi
+
+if $data15 != 20.000000000 then
+  print data15 $data5 != 20.000000000
+  return -1
+endi
+
+if $data25 != 50.000000000 then
+  print data25 $data25 != 50.000000000
+  return -1
+endi
+
+if $data35 != 90.000000000 then
+  print data35 $data35 != 90.000000000
+  return -1
+endi
+
+if $data45 != 100.000000000 then
+  print data45 $data45 != 100.000000000
+  return -1
+endi
+
+if $data55 != 130.000000000 then
+  print data55 $data55 != 130.000000000
+  return -1
+endi
+
+if $data65 != 170.000000000 then
+  print data65 $data65 != 170.000000000
+  return -1
+endi
+
+if $data75 != 180.000000000 then
+  print data75 $data75 != 180.000000000
+  return -1
+endi
+
+if $data85 != 210.000000000 then
+  print data85 $data85 != 210.000000000
+  return -1
+endi
+
+if $data95 != 200.000000000 then
+  print data95 $data95 != 200.000000000
+  return -1
+endi
+
+if $data[10][5] != 190.000000000 then
+  print data[10][5] $data[10][5] != 190.000000000
+  return -1
+endi
+
+if $data[11][5] != 220.000000000 then
+  print data[11][5] $data[11][5] != 220.000000000
+  return -1
+endi
+
+if $data[12][5] != 230.000000000 then
+  print data[12][5] $data[12][5] != 230.000000000
+  return -1
+endi
+
+if $data[13][5] != 240.000000000 then
+  print data[13][5] $data[13][5] != 240.000000000
+  return -1
+endi
+
+
+print =============== step 8 query records of ct4 from memory
+sql select * from ct4;
+print $data00 $data01 $data02 $data03 $data04 $data05
+print $data10 $data11 $data12 $data13 $data14 $data15
+print $data20 $data21 $data22 $data23 $data24 $data25
+print $data30 $data31 $data32 $data33 $data34 $data35
+print $data40 $data41 $data42 $data43 $data44 $data45
+
+
+if $rows != 5 then 
+  print rows $rows != 5
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data11 != 30 then
+  print data11 $data11 != 30
+  return -1
+endi
+
+if $data21 != 50 then
+  print data21 $data21 != 50
+  return -1
+endi
+
+if $data31 != 90 then
+  print data31 $data31 != 90
+  return -1
+endi
+
+if $data41 != 80 then
+  print data41 $data41 != 80
+  return -1
+endi
+
+if $data03 != b0 then
+  print data03 $data03 != b0
+  return -1
+endi
+
+if $data13 != b2 then
+  print data13 $data13 != b2
+  return -1
+endi
+
+if $data23 != b5 then
+  print data23 $data23 != b5
+  return -1
+endi
+
+if $data33 != b9 then
+  print data33 $data33 != b9
+  return -1
+endi
+
+if $data43 != b8  then
+  print data43 $data43 != b8
+  return -1
+endi
+
+if $data04 != n0 then
+  print data04 $data04 != n0
+  return -1
+endi
+
+if $data14 != n2 then
+  print data14 $data14 != n2
+  return -1
+endi
+
+if $data24 != n5 then
+  print data24 $data24 != n5
+  return -1
+endi
+
+if $data34 != n9 then
+  print data34 $data34 != n9
+  return -1
+endi
+
+if $data44 != n8 then
+  print data44 $data44 != n8
+  return -1
+endi
+
+#==================== reboot to trigger commit data to file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+
+
+print =============== step 9 query records of ct1 from file
+sql select * from ct1;
+print $data00 $data01 $data02 $data03 $data04 $data05
+print $data10 $data11 $data12 $data13 $data14 $data15
+print $data20 $data21 $data22 $data23 $data24 $data25
+print $data30 $data31 $data32 $data33 $data34 $data35
+print $data40 $data41 $data42 $data43 $data44 $data45
+print $data50 $data51 $data52 $data53 $data54 $data55
+
+if $rows != 6 then 
+  print rows $rows != 6
+  return -1
+endi
+
+if $data01 != 30 then
+  print data01 $data01 != 30
+  return -1
+endi
+
+if $data02 != 100.000000000 then
+  print data02 $data02 != 100.000000000
+  return -1
+endi
+
+if $data03 != bin2 then
+  print data03 $data03 != bin2
+  return -1
+endi
+
+if $data04 != nchar4 then
+  print data04 $data04 != nchar4
+  return -1
+endi
+
+if $data05 != 1000.010000000 then
+  print data05 $data05 != 1000.010000000
+  return -1
+endi
+
+if $data11 != 11 then
+  print data11 $data11 != 11
+  return -1
+endi
+
+if $data12 != NULL then
+  print data12 $data12 != NULL
+  return -1
+endi
+
+if $data13 != binary then
+  print data13 $data13 != binary
+  return -1
+endi
+
+if $data14 != nchar then
+  print data14 $data14 != nchar
+  return -1
+endi
+
+if $data15 != NULL then
+  print data15 $data15 != NULL
+  return -1
+endi
+
+if $data51 != 22 then
+  print data51 $data51 != 22
+  return -1
+endi
+
+if $data52 != NULL then
+  print data52 $data52 != NULL
+  return -1
+endi
+
+if $data53 != NULL then
+  print data53 $data53 != NULL
+  return -1
+endi
+
+if $data54 != nchar then
+  print data54 $data54 != nchar
+  return -1
+endi
+
+if $data55 != 220.000000000 then
+  print data55 $data55 != 220.000000000
+  return -1
+endi
+
+
+print =============== step 10 query records of ct2 from file
+sql select * from ct2;
+print $data00 $data01 $data02 $data03 $data04 $data05
+
+if $rows != 1 then 
+  print rows $rows != 1
+  return -1
+endi
+
+if $data01 != 20 then
+  print data01 $data01 != 20
+  return -1
+endi
+
+if $data02 != 20.000000000 then
+  print data02 $data02 != 20.000000000
+  return -1
+endi
+
+if $data03 != xkl then
+  print data03 $data03 != xkl
+  return -1
+endi
+
+if $data04 != zxc then
+  print data04 $data04 != zxc
+  return -1
+endi
+
+if $data05 != 10.000000000 then
+  print data05 $data05 != 10.000000000
+  return -1
+endi
+
+print =============== step 11 query records of ct3 from file
+sql select * from ct3;
+print $data00 $data01 $data02 $data03 $data04 $data05
+print $data10 $data11 $data12 $data13 $data14 $data15
+print $data20 $data21 $data22 $data23 $data24 $data25
+print $data30 $data31 $data32 $data33 $data34 $data35
+print $data40 $data41 $data42 $data43 $data44 $data45
+print $data50 $data51 $data52 $data53 $data54 $data55
+print $data60 $data61 $data62 $data63 $data64 $data65
+print $data70 $data71 $data72 $data73 $data74 $data75
+print $data80 $data81 $data82 $data83 $data84 $data85
+print $data90 $data91 $data92 $data93 $data94 $data95
+print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5] 
+print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5] 
+print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5] 
+print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5] 
+
+if $rows != 14 then 
+  print rows $rows != 14
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data11 != 20 then
+  print data11 $data1 != 20
+  return -1
+endi
+
+if $data21 != 50 then
+  print data21 $data21 != 50
+  return -1
+endi
+
+if $data31 != 90 then
+  print data31 $data31 != 90
+  return -1
+endi
+
+if $data41 != 100 then
+  print data41 $data41 != 100
+  return -1
+endi
+
+if $data51 != 130 then
+  print data51 $data51 != 130
+  return -1
+endi
+
+if $data61 != 170 then
+  print data61 $data61 != 170
+  return -1
+endi
+
+if $data71 != 180 then
+  print data71 $data71 != 180
+  return -1
+endi
+
+if $data81 != 210 then
+  print data81 $data81 != 210
+  return -1
+endi
+
+if $data91 != 200 then
+  print data91 $data91 != 200
+  return -1
+endi
+
+if $data[10][1] != 190 then
+  print data[10][1] $data[10][1] != 190
+  return -1
+endi
+
+if $data[11][1] != 220 then
+  print data[11][1] $data[11][1] != 220
+  return -1
+endi
+
+if $data[12][1] != 230 then
+  print data[12][1] $data[12][1] != 230
+  return -1
+endi
+
+if $data[13][1] != 240 then
+  print data[13][1] $data[13][1] != 240
+  return -1
+endi
+
+if $data05 != 10.000000000 then
+  print data05 $data05 != 10.000000000
+  return -1
+endi
+
+if $data15 != 20.000000000 then
+  print data15 $data5 != 20.000000000
+  return -1
+endi
+
+if $data25 != 50.000000000 then
+  print data25 $data25 != 50.000000000
+  return -1
+endi
+
+if $data35 != 90.000000000 then
+  print data35 $data35 != 90.000000000
+  return -1
+endi
+
+if $data45 != 100.000000000 then
+  print data45 $data45 != 100.000000000
+  return -1
+endi
+
+if $data55 != 130.000000000 then
+  print data55 $data55 != 130.000000000
+  return -1
+endi
+
+if $data65 != 170.000000000 then
+  print data65 $data65 != 170.000000000
+  return -1
+endi
+
+if $data75 != 180.000000000 then
+  print data75 $data75 != 180.000000000
+  return -1
+endi
+
+if $data85 != 210.000000000 then
+  print data85 $data85 != 210.000000000
+  return -1
+endi
+
+if $data95 != 200.000000000 then
+  print data95 $data95 != 200.000000000
+  return -1
+endi
+
+if $data[10][5] != 190.000000000 then
+  print data[10][5] $data[10][5] != 190.000000000
+  return -1
+endi
+
+if $data[11][5] != 220.000000000 then
+  print data[11][5] $data[11][5] != 220.000000000
+  return -1
+endi
+
+if $data[12][5] != 230.000000000 then
+  print data[12][5] $data[12][5] != 230.000000000
+  return -1
+endi
+
+if $data[13][5] != 240.000000000 then
+  print data[13][5] $data[13][5] != 240.000000000
+  return -1
+endi
+
+
+print =============== step 12 query records of ct4 from file
+sql select * from ct4;
+print $data00 $data01 $data02 $data03 $data04 $data05
+print $data10 $data11 $data12 $data13 $data14 $data15
+print $data20 $data21 $data22 $data23 $data24 $data25
+print $data30 $data31 $data32 $data33 $data34 $data35
+print $data40 $data41 $data42 $data43 $data44 $data45
+
+
+if $rows != 5 then 
+  print rows $rows != 5
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data11 != 30 then
+  print data11 $data11 != 30
+  return -1
+endi
+
+if $data21 != 50 then
+  print data21 $data21 != 50
+  return -1
+endi
+
+if $data31 != 90 then
+  print data31 $data31 != 90
+  return -1
+endi
+
+if $data41 != 80 then
+  print data41 $data41 != 80
+  return -1
+endi
+
+if $data03 != b0 then
+  print data03 $data03 != b0
+  return -1
+endi
+
+if $data13 != b2 then
+  print data13 $data13 != b2
+  return -1
+endi
+
+if $data23 != b5 then
+  print data23 $data23 != b5
+  return -1
+endi
+
+if $data33 != b9 then
+  print data33 $data33 != b9
+  return -1
+endi
+
+if $data43 != b8  then
+  print data43 $data43 != b8
+  return -1
+endi
+
+if $data04 != n0 then
+  print data04 $data04 != n0
+  return -1
+endi
+
+if $data14 != n2 then
+  print data14 $data14 != n2
+  return -1
+endi
+
+if $data24 != n5 then
+  print data24 $data24 != n5
+  return -1
+endi
+
+if $data34 != n9 then
+  print data34 $data34 != n9
+  return -1
+endi
+
+if $data44 != n8 then
+  print data44 $data44 != n8
+  return -1
+endi
\ No newline at end of file

tests/script/tsim/stream/basic1.sim

@@ -391,13 +391,13 @@ if $data02 != 4 then
   return -1
 endi
 
-if $data03 != 14 then
-  print ======$data03
+if $data03 != 50 then
+  print ======$data03 != 50
   return -1
 endi
 
-if $data04 != 4 then
-  print ======$data04
+if $data04 != 20 then
+  print ======$data04 != 20
   return -1
 endi
 
@@ -421,13 +421,13 @@ if $data12 != 4 then
   return -1
 endi
 
-if $data13 != 10 then
-  print ======$data13
+if $data13 != 46 then
+  print ======$data13 != 46
   return -1
 endi
 
-if $data14 != 3 then
-  print ======$data14
+if $data14 != 20 then
+  print ======$data14 != 20
   return -1
 endi