Merge branch '3.0' of github.com:taosdata/TDengine into test/chr/TD-14699

cmake/cmake.options

@@ -79,10 +79,16 @@ ENDIF ()
 
 option(
     BUILD_SANITIZER
-    "If build addr2line"
+    "If build sanitizer"
     OFF
     )
 
+option(
+    TDENGINE_3
+    "TDengine 3.x"
+    ON
+    )
+
 option(
     BUILD_ADDR2LINE
     "If build addr2line"

examples/c/stream_demo.c

@@ -98,10 +98,10 @@ int32_t create_stream() {
   /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
   /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
-  pRes = taos_query(
-      pConn,
-      "create stream stream1 trigger max_delay 10s into outstb as select _wstartts, sum(k) from st1 partition "
-      "by tbname session(ts, 10s) ");
+  pRes =
+      taos_query(pConn,
+                 "create stream stream1 trigger max_delay 10s into outstb as select _wstart, sum(k) from st1 partition "
+                 "by tbname session(ts, 10s) ");
   if (taos_errno(pRes) != 0) {
     printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;

examples/c/tmq.c

@@ -30,21 +30,35 @@ static void msg_process(TAOS_RES* msg) {
   if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
     tmq_raw_data *raw = tmq_get_raw_meta(msg);
     if(raw){
-      TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", "abc1", 0);
+      TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0);
       if (pConn == NULL) {
         return;
       }
+
+      TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 5");
+      if (taos_errno(pRes) != 0) {
+        printf("error in create db, reason:%s\n", taos_errstr(pRes));
+        return;
+      }
+      taos_free_result(pRes);
+
+      pRes = taos_query(pConn, "use abc1");
+      if (taos_errno(pRes) != 0) {
+        printf("error in use db, reason:%s\n", taos_errstr(pRes));
+        return;
+      }
+      taos_free_result(pRes);
+
       int32_t ret = taos_write_raw_meta(pConn, raw);
       printf("write raw data: %s\n", tmq_err2str(ret));
-      free(raw);
       taos_close(pConn);
     }
+    tmq_free_raw_meta(raw);
     char* result = tmq_get_json_meta(msg);
     if(result){
       printf("meta result: %s\n", result);
-      free(result);
     }
-    printf("meta:%p\n", raw);
+    tmq_free_json_meta(result);
     return;
   }
   while (1) {
@@ -68,7 +82,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 5");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -82,15 +96,14 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
-  pRes =
-      taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int)");
+  pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 nchar(8), t4 bool)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000)");
+  pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -104,13 +117,20 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)");
+  pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
 
+  pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
   pRes = taos_query(pConn, "insert into ct1 values(now, 3, 4, 'b')");
   if (taos_errno(pRes) != 0) {
     printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes));
@@ -118,7 +138,7 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)");
+  pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -202,6 +222,13 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
+  pRes = taos_query(pConn, "alter table n1 comment 'hello'");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
   pRes = taos_query(pConn, "alter table n1 drop column c1");
   if (taos_errno(pRes) != 0) {
     printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes));
@@ -230,6 +257,27 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
+  pRes = taos_query(pConn, "create table jt2 using jt tags('')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 nchar(8), t4 bool)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "drop table st1");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
   return 0;
 }
 

include/client/taos.h

@@ -265,7 +265,9 @@ typedef struct tmq_raw_data tmq_raw_data;
 DLL_EXPORT tmq_res_t     tmq_get_res_type(TAOS_RES *res);
 DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res);
 DLL_EXPORT int32_t       taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta);
-DLL_EXPORT char         *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed.
+DLL_EXPORT void          tmq_free_raw_meta(tmq_raw_data *rawMeta);
+DLL_EXPORT char         *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed by tmq_free_json_meta
+DLL_EXPORT void          tmq_free_json_meta(char* jsonMeta);
 DLL_EXPORT const char   *tmq_get_topic_name(TAOS_RES *res);
 DLL_EXPORT const char   *tmq_get_db_name(TAOS_RES *res);
 DLL_EXPORT int32_t       tmq_get_vgroup_id(TAOS_RES *res);

include/common/tcommon.h

@@ -40,7 +40,6 @@ enum {
   || x == TDMT_VND_CREATE_TABLE   \
   || x == TDMT_VND_ALTER_TABLE    \
   || x == TDMT_VND_DROP_TABLE     \
-  || x == TDMT_VND_DROP_TTL_TABLE \
 )
 // clang-format on
 
@@ -108,6 +107,7 @@ typedef struct SDataBlockInfo {
   // TODO: optimize and remove following
   int32_t     childId;  // used for stream, do not serialize
   EStreamType type;     // used for stream, do not serialize
+  STimeWindow calWin;   // used for stream, do not serialize
 } SDataBlockInfo;
 
 typedef struct SSDataBlock {
@@ -161,7 +161,6 @@ typedef struct SQueryTableDataCond {
   int64_t      endVersion;
 } SQueryTableDataCond;
 
-void*   blockDataDestroy(SSDataBlock* pBlock);
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
 void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
 
@@ -169,19 +168,6 @@ int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
 void*   tDecodeDataBlocks(const void* buf, SArray** blocks);
 void    colDataDestroy(SColumnInfoData* pColData);
 
-static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
-  int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock);
-  for (int32_t i = 0; i < numOfOutput; ++i) {
-    SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
-    colDataDestroy(pColInfoData);
-  }
-
-  taosArrayDestroy(pBlock->pDataBlock);
-  taosMemoryFreeClear(pBlock->pBlockAgg);
-}
-
-static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); }
-
 //======================================================================================================================
 // the following structure shared by parser and executor
 typedef struct SColumn {

include/common/tdatablock.h

@@ -226,8 +226,12 @@ int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
 
 int32_t      assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
 int32_t      copyDataBlock(SSDataBlock* dst, const SSDataBlock* src);
-SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
+
 SSDataBlock* createDataBlock();
+void*        blockDataDestroy(SSDataBlock* pBlock);
+void         blockDataFreeRes(SSDataBlock* pBlock);
+SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
+
 int32_t      blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
 
 SColumnInfoData  createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);

include/common/tdataformat.h

@@ -64,18 +64,22 @@ int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type);
 int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type);
 int     tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type);
 
-// STSRow2
+// SColVal
 #define COL_VAL_NONE(CID, TYPE)     ((SColVal){.cid = (CID), .type = (TYPE), .isNone = 1})
 #define COL_VAL_NULL(CID, TYPE)     ((SColVal){.cid = (CID), .type = (TYPE), .isNull = 1})
 #define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)})
 
+// STSRow2
+#define TSROW_LEN(PROW, V)  tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL)
+#define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL)
+
 int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow);
 int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow);
 void    tTSRowFree(STSRow2 *pRow);
 void    tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray);
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
-int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow);
 
 // STSRowBuilder
 #define tsRowBuilderInit() ((STSRowBuilder){0})
@@ -97,7 +101,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
 int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
 int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
 void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);  // TODO: remove
-int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf);
+int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf);
 
 // STRUCT =================
 struct STColumn {
@@ -123,16 +127,16 @@ struct STSchema {
 #define TSROW_KV_SMALL ((uint8_t)0x10U)
 #define TSROW_KV_MID   ((uint8_t)0x20U)
 #define TSROW_KV_BIG   ((uint8_t)0x40U)
+#pragma pack(push, 1)
 struct STSRow2 {
-  TSKEY    ts;
-  uint8_t  flags;
-  int32_t  sver;
-  uint32_t nData;
-  uint8_t *pData;
+  TSKEY   ts;
+  uint8_t flags;
+  uint8_t data[];
 };
+#pragma pack(pop)
 
 struct STSRowBuilder {
-  STSRow2  tsRow;
+  // STSRow2  tsRow;
   int32_t  szBuf;
   uint8_t *pBuf;
 };
@@ -167,6 +171,7 @@ struct SColVal {
 
 #pragma pack(push, 1)
 struct STagVal {
+//  char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta
   union {
     int16_t cid;
     char   *pKey;
@@ -226,50 +231,6 @@ struct STag {
     memcpy(varDataVal(x), (str), (_size));      \
   } while (0);
 
-// ----------------- TSDB COLUMN DEFINITION
-
-#define colType(col)   ((col)->type)
-#define colFlags(col)  ((col)->flags)
-#define colColId(col)  ((col)->colId)
-#define colBytes(col)  ((col)->bytes)
-#define colOffset(col) ((col)->offset)
-
-#define colSetType(col, t)   (colType(col) = (t))
-#define colSetFlags(col, f)  (colFlags(col) = (f))
-#define colSetColId(col, id) (colColId(col) = (id))
-#define colSetBytes(col, b)  (colBytes(col) = (b))
-#define colSetOffset(col, o) (colOffset(col) = (o))
-
-// ----------------- TSDB SCHEMA DEFINITION
-
-#define schemaNCols(s)    ((s)->numOfCols)
-#define schemaVersion(s)  ((s)->version)
-#define schemaTLen(s)     ((s)->tlen)
-#define schemaFLen(s)     ((s)->flen)
-#define schemaVLen(s)     ((s)->vlen)
-#define schemaColAt(s, i) ((s)->columns + i)
-#define tdFreeSchema(s)   taosMemoryFreeClear((s))
-
-STSchema *tdDupSchema(const STSchema *pSchema);
-int32_t   tdEncodeSchema(void **buf, STSchema *pSchema);
-void     *tdDecodeSchema(void *buf, STSchema **pRSchema);
-
-static FORCE_INLINE int32_t comparColId(const void *key1, const void *key2) {
-  if (*(int16_t *)key1 > ((STColumn *)key2)->colId) {
-    return 1;
-  } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) {
-    return -1;
-  } else {
-    return 0;
-  }
-}
-
-static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) {
-  void *ptr = bsearch(&colId, (void *)pSchema->columns, schemaNCols(pSchema), sizeof(STColumn), comparColId);
-  if (ptr == NULL) return NULL;
-  return (STColumn *)ptr;
-}
-
 // ----------------- SCHEMA BUILDER DEFINITION
 typedef struct {
   int32_t      tCols;
@@ -299,141 +260,6 @@ void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version)
 int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
-// ----------------- Semantic timestamp key definition
-// typedef uint64_t TKEY;
-#define TKEY TSKEY
-
-#define TKEY_INVALID       UINT64_MAX
-#define TKEY_NULL          TKEY_INVALID
-#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
-#define TKEY_VALUE_FILTER  (~(TKEY_NEGATIVE_FLAG))
-
-#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
-#define TKEY_IS_DELETED(tkey)  (false)
-
-#define tdGetTKEY(key)  (key)
-#define tdGetKey(tskey) (tskey)
-
-#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
-#define MAX_TS_KEY ((TSKEY)0x7fffffffffffffff)
-
-#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
-
-static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
-  TSKEY lkey = key;
-  if (key > MAX_TS_KEY) {
-    lkey = MAX_TS_KEY;
-  } else if (key < MIN_TS_KEY) {
-    lkey = MIN_TS_KEY;
-  }
-
-  return tdGetTKEY(lkey);
-}
-
-static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) {
-  TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
-  TSKEY key2 = tdGetKey(*(TKEY *)tkey2);
-
-  if (key1 < key2) {
-    return -1;
-  } else if (key1 > key2) {
-    return 1;
-  } else {
-    return 0;
-  }
-}
-
-// ----------------- Data column structure
-// SDataCol arrangement: data => bitmap => dataOffset
-typedef struct SDataCol {
-  int8_t          type;        // column type
-  uint8_t         bitmap : 1;  // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
-  uint8_t         reserve : 7;
-  int16_t         colId;      // column ID
-  int32_t         bytes;      // column data bytes defined
-  int32_t         offset;     // data offset in a SDataRow (including the header size)
-  int32_t         spaceSize;  // Total space size for this column
-  int32_t         len;        // column data length
-  VarDataOffsetT *dataOff;    // For binary and nchar data, the offset in the data column
-  void           *pData;      // Actual data pointer
-  void           *pBitmap;    // Bitmap pointer
-  TSKEY           ts;         // only used in last NULL column
-} SDataCol;
-
-#define isAllRowsNull(pCol) ((pCol)->len == 0)
-#define isAllRowsNone(pCol) ((pCol)->len == 0)
-static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
-
-int32_t tdAllocMemForCol(SDataCol *pCol, int32_t maxPoints);
-
-void    dataColInit(SDataCol *pDataCol, STColumn *pCol, int32_t maxPoints);
-int32_t dataColAppendVal(SDataCol *pCol, const void *value, int32_t numOfRows, int32_t maxPoints);
-void   *dataColSetOffset(SDataCol *pCol, int32_t nEle);
-
-bool isNEleNull(SDataCol *pCol, int32_t nEle);
-
-typedef struct {
-  col_id_t  maxCols;    // max number of columns
-  col_id_t  numOfCols;  // Total number of cols
-  int32_t   maxPoints;  // max number of points
-  int32_t   numOfRows;
-  int32_t   bitmapMode : 1;  // default is 0(2 bits), otherwise 1(1 bit)
-  int32_t   sversion : 31;   // TODO: set sversion(not used yet)
-  SDataCol *cols;
-} SDataCols;
-
-static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != TSDB_BITMODE_DEFAULT; }
-static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = TSDB_BITMODE_ONE_BIT; }
-static FORCE_INLINE bool tdIsBitmapModeI(int8_t bitmapMode) { return bitmapMode != TSDB_BITMODE_DEFAULT; }
-
-#define keyCol(pCols)              (&((pCols)->cols[0]))                    // Key column
-#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]  // the idx row of column-wised data
-#define dataColsKeyAt(pCols, idx)  tdGetKey(dataColsTKeyAt(pCols, idx))
-static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) {
-  if (pCols->numOfRows) {
-    return dataColsTKeyAt(pCols, 0);
-  } else {
-    return TKEY_INVALID;
-  }
-}
-
-static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int32_t row) {
-  assert(row < pCols->numOfRows);
-  return dataColsKeyAt(pCols, row);
-}
-
-static FORCE_INLINE TSKEY dataColsKeyFirst(SDataCols *pCols) {
-  if (pCols->numOfRows) {
-    return dataColsKeyAt(pCols, 0);
-  } else {
-    return TSDB_DATA_TIMESTAMP_NULL;
-  }
-}
-
-static FORCE_INLINE TKEY dataColsTKeyLast(SDataCols *pCols) {
-  if (pCols->numOfRows) {
-    return dataColsTKeyAt(pCols, pCols->numOfRows - 1);
-  } else {
-    return TKEY_INVALID;
-  }
-}
-
-static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) {
-  if (pCols->numOfRows) {
-    return dataColsKeyAt(pCols, pCols->numOfRows - 1);
-  } else {
-    return TSDB_DATA_TIMESTAMP_NULL;
-  }
-}
-
-SDataCols *tdNewDataCols(int32_t maxCols, int32_t maxRows);
-void       tdResetDataCols(SDataCols *pCols);
-int32_t    tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
-SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
-SDataCols *tdFreeDataCols(SDataCols *pCols);
-int32_t    tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update,
-                           TDRowVerT maxVer);
-
 #endif
 
 #ifdef __cplusplus

include/common/tglobal.h

@@ -64,6 +64,7 @@ extern int32_t tsNumOfMnodeQueryThreads;
 extern int32_t tsNumOfMnodeFetchThreads;
 extern int32_t tsNumOfMnodeReadThreads;
 extern int32_t tsNumOfVnodeQueryThreads;
+extern int32_t tsNumOfVnodeStreamThreads;
 extern int32_t tsNumOfVnodeFetchThreads;
 extern int32_t tsNumOfVnodeWriteThreads;
 extern int32_t tsNumOfVnodeSyncThreads;

include/common/tmsg.h

@@ -1939,6 +1939,8 @@ typedef struct SVCreateStbReq {
   SSchemaWrapper schemaRow;
   SSchemaWrapper schemaTag;
   SRSmaParam     rsmaParam;
+  int32_t        alterOriDataLen;
+  void*          alterOriData;
 } SVCreateStbReq;
 
 int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
@@ -1966,7 +1968,9 @@ typedef struct SVCreateTbReq {
   int8_t   type;
   union {
     struct {
+      char*    name;    // super table name
       tb_uid_t suid;
+      SArray*  tagName;
       uint8_t* pTag;
     } ctb;
     struct {
@@ -1983,6 +1987,9 @@ static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) {
   taosMemoryFreeClear(req->comment);
   if (req->type == TSDB_CHILD_TABLE) {
     taosMemoryFreeClear(req->ctb.pTag);
+    taosMemoryFreeClear(req->ctb.name);
+    taosArrayDestroy(req->ctb.tagName);
+    req->ctb.tagName = NULL;
   } else if (req->type == TSDB_NORMAL_TABLE) {
     taosMemoryFreeClear(req->ntb.schemaRow.pSchema);
   }
@@ -2066,12 +2073,14 @@ typedef struct {
   int32_t bytes;
   // TSDB_ALTER_TABLE_DROP_COLUMN
   // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES
+  int8_t  colModType;
   int32_t colModBytes;
   // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME
   char* colNewName;
   // TSDB_ALTER_TABLE_UPDATE_TAG_VAL
   char*    tagName;
   int8_t   isNull;
+  int8_t   tagType;
   uint32_t nTagVal;
   uint8_t* pTagVal;
   // TSDB_ALTER_TABLE_UPDATE_OPTIONS
@@ -2858,8 +2867,8 @@ typedef struct {
 
 static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) {
   int32_t tlen = 0;
-  // tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
-  // tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
+  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
+  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
   tlen += taosEncodeFixedI16(buf, pRsp->resMsgType);
   tlen += taosEncodeFixedI32(buf, pRsp->metaRspLen);
   tlen += taosEncodeBinary(buf, pRsp->metaRsp, pRsp->metaRspLen);
@@ -2867,8 +2876,7 @@ static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp
 }
 
 static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) {
-  // buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
-  // buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
+  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
   buf = taosDecodeFixedI16(buf, &pRsp->resMsgType);
   buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen);
   buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen);

include/common/tmsgcb.h

@@ -34,6 +34,7 @@ typedef enum {
   WRITE_QUEUE,
   APPLY_QUEUE,
   SYNC_QUEUE,
+  STREAM_QUEUE,
   QUEUE_MAX,
 } EQueueType;
 

include/common/trow.h

@@ -168,7 +168,7 @@ typedef struct {
 
 // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
 // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
-#define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN)
+#define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) ((s)->tlen + TD_ROW_HEAD_LEN)
 
 #define TD_ROW_SET_INFO(r, i)  (TD_ROW_INFO(r) = (i))
 #define TD_ROW_SET_TYPE(r, t)  (TD_ROW_TYPE(r) = (t))
@@ -223,9 +223,10 @@ int32_t                     tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDR
 static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType,
                                                int8_t bitmapMode);
 bool                        tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode);
-int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints,
-                             int8_t bitmapMode, bool isMerge);
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge);
+// int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t
+// maxPoints,
+//                              int8_t bitmapMode, bool isMerge);
+// int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge);
 
 int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
 int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType);
@@ -318,12 +319,9 @@ bool    tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SC
 bool    tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal);
 bool    tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal);
 bool    tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal);
-STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2);
-int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode);
 bool    tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx,
                        SCellVal *pVal);
 bool    tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal);
-int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode);
 void    tdSCellValPrint(SCellVal *pVal, int8_t colType);
 void    tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag);
 

include/common/ttime.h

@@ -72,6 +72,8 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
 }
 
 int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);
+int64_t taosTimeSub(int64_t t, int64_t duration, char unit, int32_t precision);
+
 int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision);
 int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision);
 

include/common/ttokendef.h

@@ -213,63 +213,64 @@
 #define TK_NK_ARROW       195
 #define TK_ROWTS          196
 #define TK_TBNAME         197
-#define TK_QSTARTTS       198
-#define TK_QENDTS         199
-#define TK_WSTARTTS       200
-#define TK_WENDTS         201
-#define TK_WDURATION      202
-#define TK_CAST           203
-#define TK_NOW            204
-#define TK_TODAY          205
-#define TK_TIMEZONE       206
-#define TK_CLIENT_VERSION 207
-#define TK_SERVER_VERSION 208
-#define TK_SERVER_STATUS  209
-#define TK_CURRENT_USER   210
-#define TK_COUNT          211
-#define TK_LAST_ROW       212
-#define TK_BETWEEN        213
-#define TK_IS             214
-#define TK_NK_LT          215
-#define TK_NK_GT          216
-#define TK_NK_LE          217
-#define TK_NK_GE          218
-#define TK_NK_NE          219
-#define TK_MATCH          220
-#define TK_NMATCH         221
-#define TK_CONTAINS       222
-#define TK_JOIN           223
-#define TK_INNER          224
-#define TK_SELECT         225
-#define TK_DISTINCT       226
-#define TK_WHERE          227
-#define TK_PARTITION      228
-#define TK_BY             229
-#define TK_SESSION        230
-#define TK_STATE_WINDOW   231
-#define TK_SLIDING        232
-#define TK_FILL           233
-#define TK_VALUE          234
-#define TK_NONE           235
-#define TK_PREV           236
-#define TK_LINEAR         237
-#define TK_NEXT           238
-#define TK_HAVING         239
-#define TK_RANGE          240
-#define TK_EVERY          241
-#define TK_ORDER          242
-#define TK_SLIMIT         243
-#define TK_SOFFSET        244
-#define TK_LIMIT          245
-#define TK_OFFSET         246
-#define TK_ASC            247
-#define TK_NULLS          248
-#define TK_ID             249
-#define TK_NK_BITNOT      250
-#define TK_VALUES         251
-#define TK_IMPORT         252
-#define TK_NK_SEMI        253
-#define TK_FILE           254
+#define TK_QSTART         198
+#define TK_QEND           199
+#define TK_QDURATION      200
+#define TK_WSTART         201
+#define TK_WEND           202
+#define TK_WDURATION      203
+#define TK_CAST           204
+#define TK_NOW            205
+#define TK_TODAY          206
+#define TK_TIMEZONE       207
+#define TK_CLIENT_VERSION 208
+#define TK_SERVER_VERSION 209
+#define TK_SERVER_STATUS  210
+#define TK_CURRENT_USER   211
+#define TK_COUNT          212
+#define TK_LAST_ROW       213
+#define TK_BETWEEN        214
+#define TK_IS             215
+#define TK_NK_LT          216
+#define TK_NK_GT          217
+#define TK_NK_LE          218
+#define TK_NK_GE          219
+#define TK_NK_NE          220
+#define TK_MATCH          221
+#define TK_NMATCH         222
+#define TK_CONTAINS       223
+#define TK_JOIN           224
+#define TK_INNER          225
+#define TK_SELECT         226
+#define TK_DISTINCT       227
+#define TK_WHERE          228
+#define TK_PARTITION      229
+#define TK_BY             230
+#define TK_SESSION        231
+#define TK_STATE_WINDOW   232
+#define TK_SLIDING        233
+#define TK_FILL           234
+#define TK_VALUE          235
+#define TK_NONE           236
+#define TK_PREV           237
+#define TK_LINEAR         238
+#define TK_NEXT           239
+#define TK_HAVING         240
+#define TK_RANGE          241
+#define TK_EVERY          242
+#define TK_ORDER          243
+#define TK_SLIMIT         244
+#define TK_SOFFSET        245
+#define TK_LIMIT          246
+#define TK_OFFSET         247
+#define TK_ASC            248
+#define TK_NULLS          249
+#define TK_ID             250
+#define TK_NK_BITNOT      251
+#define TK_VALUES         252
+#define TK_IMPORT         253
+#define TK_NK_SEMI        254
+#define TK_FILE           255
 
 #define TK_NK_SPACE   300
 #define TK_NK_COMMENT 301

include/libs/executor/executor.h

@@ -42,25 +42,28 @@ typedef struct SReadHandle {
   bool    initTqReader;
 } SReadHandle;
 
+// in queue mode, data streams are seperated by msg
 typedef enum {
   OPTR_EXEC_MODEL_BATCH = 0x1,
   OPTR_EXEC_MODEL_STREAM = 0x2,
+  OPTR_EXEC_MODEL_QUEUE = 0x3,
 } EOPTR_EXEC_MODEL;
 
 /**
- * Create the exec task for streaming mode
+ * Create the exec task for stream mode
  * @param pMsg
- * @param streamReadHandle
+ * @param SReadHandle
  * @return
  */
 qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers);
 
 /**
- * Switch the stream scan to snapshot mode
- * @param tinfo
+ * Create the exec task for queue mode
+ * @param pMsg
+ * @param SReadHandle
  * @return
  */
-int32_t qStreamScanSnapshot(qTaskInfo_t tinfo);
+qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers);
 
 /**
  * Set the input data block for the stream scan.
@@ -111,7 +114,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
  * @return
  */
 int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion,
-                                      int32_t* tversion);
+                                    int32_t* tversion);
 
 /**
  * The main task execution function, including query on both table and multiple tables,
@@ -183,9 +186,12 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);
 
 void* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
 
-void*   qExtractReaderFromStreamScanner(void* scanner);
+void* qExtractReaderFromStreamScanner(void* scanner);
+
 int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);
 
+int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/function/functionMgt.h

@@ -34,7 +34,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_ELAPSED,
   FUNCTION_TYPE_IRATE,
   FUNCTION_TYPE_LAST_ROW,
-  FUNCTION_TYPE_LAST_ROWT, //TODO: removed
+  FUNCTION_TYPE_LAST_ROWT,  // TODO: removed
   FUNCTION_TYPE_MAX,
   FUNCTION_TYPE_MIN,
   FUNCTION_TYPE_MODE,
@@ -114,10 +114,11 @@ typedef enum EFunctionType {
   // pseudo column function
   FUNCTION_TYPE_ROWTS = 3500,
   FUNCTION_TYPE_TBNAME,
-  FUNCTION_TYPE_QSTARTTS,
-  FUNCTION_TYPE_QENDTS,
-  FUNCTION_TYPE_WSTARTTS,
-  FUNCTION_TYPE_WENDTS,
+  FUNCTION_TYPE_QSTART,
+  FUNCTION_TYPE_QEND,
+  FUNCTION_TYPE_QDURATION,
+  FUNCTION_TYPE_WSTART,
+  FUNCTION_TYPE_WEND,
   FUNCTION_TYPE_WDURATION,
 
   // internal function
@@ -197,6 +198,7 @@ bool fmIsInterpFunc(int32_t funcId);
 bool fmIsLastRowFunc(int32_t funcId);
 bool fmIsSystemInfoFunc(int32_t funcId);
 bool fmIsImplicitTsFunc(int32_t funcId);
+bool fmIsClientPseudoColumnFunc(int32_t funcId);
 
 int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc);
 

include/libs/nodes/querynodes.h

@@ -228,7 +228,7 @@ typedef struct SFillNode {
   ENodeType   type;  // QUERY_NODE_FILL
   EFillMode   mode;
   SNode*      pValues;    // SNodeListNode
-  SNode*      pWStartTs;  // _wstartts pseudo column
+  SNode*      pWStartTs;  // _wstart pseudo column
   STimeWindow timeRange;
 } SFillNode;
 
@@ -248,6 +248,7 @@ typedef struct SSelectStmt {
   SNodeList*  pOrderByList;  // SOrderByExprNode
   SLimitNode* pLimit;
   SLimitNode* pSlimit;
+  STimeWindow timeRange;
   char        stmtName[TSDB_TABLE_NAME_LEN];
   uint8_t     precision;
   int32_t     selectFuncNum;

include/libs/parser/parser.h

@@ -29,7 +29,7 @@ struct SMetaData;
 typedef struct SStmtCallback {
   TAOS_STMT* pStmt;
   int32_t (*getTbNameFn)(TAOS_STMT*, char**);
-  int32_t (*setInfoFn)(TAOS_STMT*, STableMeta*, void*, char*, bool, SHashObj*, SHashObj*);
+  int32_t (*setInfoFn)(TAOS_STMT*, STableMeta*, void*, char*, bool, SHashObj*, SHashObj*, const char*);
   int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
 } SStmtCallback;
 
@@ -84,7 +84,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu
                                 int32_t rowNum);
 int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields);
 int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields);
-int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tName, TAOS_MULTI_BIND* bind,
+int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, TAOS_MULTI_BIND* bind,
                            char* msgBuf, int32_t msgBufLen);
 void    destroyBoundColumnInfo(void* pBoundInfo);
 int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
@@ -93,7 +93,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
 void*   smlInitHandle(SQuery* pQuery);
 void    smlDestroyHandle(void* pHandle);
 int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
-                    char* tableName, char* msgBuf, int16_t msgBufLen);
+                    char* tableName, const char* sTableName, int32_t sTableNameLen, char* msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
 
 int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);

include/libs/scalar/scalar.h

@@ -32,6 +32,7 @@ pNode will be freed in API;
 *pRes need to freed in caller
 */
 int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
+int32_t scalarCalculateConstantsFromDual(SNode *pNode, SNode **pRes);
 
 /*
 pDst need to freed in caller

include/libs/stream/tstream.h

@@ -13,7 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "executor.h"
 #include "os.h"
+#include "query.h"
 #include "tdatablock.h"
 #include "tmsg.h"
 #include "tmsgcb.h"
@@ -305,13 +307,18 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
       atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
       return -1;
     }
+    qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data);
     taosWriteQitem(pTask->inputQueue->queue, pSubmitClone);
+    // qStreamInput(pTask->exec.executor, pSubmitClone);
   } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
     taosWriteQitem(pTask->inputQueue->queue, pItem);
+    // qStreamInput(pTask->exec.executor, pItem);
   } else if (pItem->type == STREAM_INPUT__CHECKPOINT) {
     taosWriteQitem(pTask->inputQueue->queue, pItem);
+    // qStreamInput(pTask->exec.executor, pItem);
   } else if (pItem->type == STREAM_INPUT__TRIGGER) {
     taosWriteQitem(pTask->inputQueue->queue, pItem);
+    // qStreamInput(pTask->exec.executor, pItem);
   }
 
   if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) {
@@ -333,12 +340,12 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc
   if (pTask->sinkType == TASK_SINK__TABLE) {
     ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
     pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
-    taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock);
+    taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
     taosFreeQitem(pBlock);
   } else if (pTask->sinkType == TASK_SINK__SMA) {
     ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
     pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
-    taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock);
+    taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
     taosFreeQitem(pBlock);
   } else {
     ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE);

include/libs/sync/sync.h

@@ -26,6 +26,8 @@ extern "C" {
 
 extern bool gRaftDetailLog;
 
+#define SYNC_RESP_TTL_MS 5000
+
 #define SYNC_MAX_BATCH_SIZE 500
 #define SYNC_INDEX_BEGIN 0
 #define SYNC_INDEX_INVALID -1

include/os/osDef.h

@@ -22,7 +22,10 @@ extern "C" {
 
 #if defined(_TD_DARWIN_64)
   // specific
+#ifndef __COMPAR_FN_T
+#define __COMPAR_FN_T
   typedef int(*__compar_fn_t)(const void *, const void *);
+#endif
 
   // for send function in tsocket.c
   #if defined(MSG_NOSIGNAL)
@@ -41,7 +44,10 @@ extern "C" {
 #endif
 
 #if defined(_ALPINE)
+#ifndef __COMPAR_FN_T
+#define __COMPAR_FN_T
   typedef int(*__compar_fn_t)(const void *, const void *);
+#endif
   void  error (int, int, const char *);
   #ifndef PTHREAD_MUTEX_RECURSIVE_NP
     #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
@@ -54,7 +60,10 @@ extern "C" {
   char *stpncpy (char *dest, const char *src, size_t n);
 
   // specific
+#ifndef __COMPAR_FN_T
+#define __COMPAR_FN_T
   typedef int (*__compar_fn_t)(const void *, const void *);
+#endif
   #define ssize_t int
   #define _SSIZE_T_
   #define bzero(ptr, size) memset((ptr), 0, (size))
@@ -69,7 +78,6 @@ extern "C" {
   char *     strsep(char **stringp, const char *delim);
   char *     getpass(const char *prefix);
   char *     strndup(const char *s, size_t n);
-  int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 
   // for send function in tsocket.c
   #define MSG_NOSIGNAL             0

include/util/taoserror.h

@@ -584,7 +584,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_INTERP_CLAUSE     TAOS_DEF_ERROR_CODE(0, 0x265D)
 #define TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN      TAOS_DEF_ERROR_CODE(0, 0x265E)
 #define TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE TAOS_DEF_ERROR_CODE(0, 0x265F)
-#define TSDB_CODE_PAR_INVALID_SMA_INDEX         TAOS_DEF_ERROR_CODE(0, 0x265C)
+#define TSDB_CODE_PAR_INVALID_SMA_INDEX         TAOS_DEF_ERROR_CODE(0, 0x2660)
+#define TSDB_CODE_PAR_INVALID_SELECTED_EXPR     TAOS_DEF_ERROR_CODE(0, 0x2661)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tencode.h

@@ -440,7 +440,7 @@ static FORCE_INLINE bool tDecodeIsEnd(SDecoder* pCoder) { return (pCoder->size =
 
 static FORCE_INLINE void* tEncoderMalloc(SEncoder* pCoder, int32_t size) {
   void*      p = NULL;
-  SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + size);
+  SCoderMem* pMem = (SCoderMem*)taosMemoryCalloc(1, sizeof(*pMem) + size);
   if (pMem) {
     pMem->next = pCoder->mList;
     pCoder->mList = pMem;
@@ -451,7 +451,7 @@ static FORCE_INLINE void* tEncoderMalloc(SEncoder* pCoder, int32_t size) {
 
 static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) {
   void*      p = NULL;
-  SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + size);
+  SCoderMem* pMem = (SCoderMem*)taosMemoryCalloc(1, sizeof(*pMem) + size);
   if (pMem) {
     pMem->next = pCoder->mList;
     pCoder->mList = pMem;

include/util/tlog.h

@@ -94,7 +94,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
 // clang-format on
-#define BUF_PAGE_DEBUG
+//#define BUF_PAGE_DEBUG
 #ifdef __cplusplus
 }
 #endif

include/util/tqueue.h

@@ -44,6 +44,8 @@ typedef struct STaosQset  STaosQset;
 typedef struct STaosQall  STaosQall;
 typedef struct {
   void   *ahandle;
+  void   *fp;
+  void   *queue;
   int32_t workerId;
   int32_t threadNum;
   int64_t timestamp;
@@ -65,6 +67,7 @@ void        taosFreeQitem(void *pItem);
 void        taosWriteQitem(STaosQueue *queue, void *pItem);
 int32_t     taosReadQitem(STaosQueue *queue, void **ppItem);
 bool        taosQueueEmpty(STaosQueue *queue);
+void        taosUpdateItemSize(STaosQueue *queue, int32_t items);
 int32_t     taosQueueItemSize(STaosQueue *queue);
 int64_t     taosQueueMemorySize(STaosQueue *queue);
 
@@ -81,8 +84,8 @@ int32_t    taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle);
 void       taosRemoveFromQset(STaosQset *qset, STaosQueue *queue);
 int32_t    taosGetQueueNumber(STaosQset *qset);
 
-int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, int64_t *ts, void **ahandle, FItem *itemFp);
-int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp);
+int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo);
+int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);
 
 extern int64_t tsRpcQueueMemoryAllowed;

source/client/inc/clientInt.h

@@ -181,6 +181,7 @@ typedef struct SRequestSendRecvBody {
   tsem_t            rspSem;  // not used now
   __taos_async_fn_t queryFp;
   __taos_async_fn_t fetchFp;
+  EQueryExecMode    execMode;
   void*             param;
   SDataBuf          requestMsg;
   int64_t           queryJob;  // query job, created according to sql query DAG.
@@ -223,8 +224,8 @@ typedef struct SRequestObj {
   SArray*              tableList;
   SQueryExecMetric     metric;
   SRequestSendRecvBody body;
-  bool                 stableQuery;  // todo refactor
-  bool                 validateOnly; // todo refactor
+  bool                 stableQuery;   // todo refactor
+  bool                 validateOnly;  // todo refactor
 
   bool     killed;
   uint32_t prevCode;  // previous error code: todo refactor, add update flag for catalog
@@ -325,7 +326,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
 
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
 
-int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql, SRequestObj** pRequest);
+int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql,
+                     SRequestObj** pRequest);
 
 void taos_close_internal(void* taos);
 
@@ -359,9 +361,6 @@ int32_t      removeMeta(STscObj* pTscObj, SArray* tbList);  // todo move to clie
 int32_t      handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);  // todo move to xxx
 bool         qnodeRequired(SRequestObj* pRequest);
 
-void initTscQhandle();
-void cleanupTscQhandle();
-
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -35,22 +35,10 @@ SAppInfo appInfo;
 int32_t  clientReqRefPool = -1;
 int32_t  clientConnRefPool = -1;
 
-void *tscQhandle = NULL;
-
 static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
 volatile int32_t    tscInitRes = 0;
 
-void initTscQhandle() {
-  // init handle
-  tscQhandle = taosInitScheduler(4096, 5, "tsc");
-}
-
-void cleanupTscQhandle() {
-  // destroy handle
-  taosCleanUpScheduler(tscQhandle);
-}
-
-static int32_t registerRequest(SRequestObj *pRequest, STscObj* pTscObj) {
+static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) {
   // connection has been released already, abort creating request.
   pRequest->self = taosAddRef(clientReqRefPool, pRequest);
 
@@ -72,7 +60,7 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj* pTscObj) {
 static void deregisterRequest(SRequestObj *pRequest) {
   assert(pRequest != NULL);
 
-  STscObj *           pTscObj = pRequest->pTscObj;
+  STscObj            *pTscObj = pRequest->pTscObj;
   SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary;
 
   int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
@@ -97,7 +85,8 @@ void closeTransporter(SAppInstInfo *pAppInfo) {
 
 static bool clientRpcRfp(int32_t code, tmsg_t msgType) {
   if (NEED_REDIRECT_ERROR(code)) {
-    if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) {
+    if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
+        msgType == TDMT_SCH_MERGE_FETCH) {
       return false;
     }
     return true;
@@ -251,7 +240,7 @@ void *createRequest(uint64_t connId, int32_t type) {
     return NULL;
   }
 
-  STscObj* pTscObj = acquireTscObj(connId);
+  STscObj *pTscObj = acquireTscObj(connId);
   if (pTscObj == NULL) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
@@ -348,7 +337,6 @@ void taos_init_imp(void) {
   // In the APIs of other program language, taos_cleanup is not available yet.
   // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning.
   atexit(taos_cleanup);
-  initTscQhandle();
   errno = TSDB_CODE_SUCCESS;
   taosSeedRand(taosGetTimestampSec());
 
@@ -407,7 +395,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
     return 0;
   }
 
-  SConfig *    pCfg = taosGetCfg();
+  SConfig     *pCfg = taosGetCfg();
   SConfigItem *pItem = NULL;
 
   switch (option) {

source/client/src/clientImpl.c

@@ -153,7 +153,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   *pRequest = createRequest(connId, TSDB_SQL_SELECT);
   if (*pRequest == NULL) {
     tscError("failed to malloc sqlObj, %s", sql);
-    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return terrno;
   }
 
   (*pRequest)->sqlstr = taosMemoryMalloc(sqlLen + 1);
@@ -933,6 +933,8 @@ SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool vali
 void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) {
   int32_t code = 0;
 
+  pRequest->body.execMode = pQuery->execMode;
+  
   switch (pQuery->execMode) {
     case QUERY_EXEC_MODE_LOCAL:
       asyncExecLocalCmd(pRequest, pQuery);
@@ -1149,7 +1151,6 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
   SRequestObj* pRequest = createRequest(pTscObj->id, TDMT_MND_CONNECT);
   if (pRequest == NULL) {
     destroyTscObj(pTscObj);
-    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -1274,8 +1275,8 @@ typedef struct SchedArg {
   SEpSet* pEpset;
 } SchedArg;
 
-void doProcessMsgFromServer(SSchedMsg* schedMsg) {
-  SchedArg* arg = (SchedArg*)schedMsg->ahandle;
+int32_t doProcessMsgFromServer(void* param) {
+  SchedArg* arg = (SchedArg*)param;
   SRpcMsg*  pMsg = &arg->msg;
   SEpSet*   pEpSet = arg->pEpset;
 
@@ -1328,11 +1329,10 @@ void doProcessMsgFromServer(SSchedMsg* schedMsg) {
   rpcFreeCont(pMsg->pCont);
   destroySendMsgInfo(pSendInfo);
   taosMemoryFree(arg);
+  return TSDB_CODE_SUCCESS;
 }
 
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
-  SSchedMsg schedMsg = {0};
-
   SEpSet* tEpSet = NULL;
   if (pEpSet != NULL) {
     tEpSet = taosMemoryCalloc(1, sizeof(SEpSet));
@@ -1343,9 +1343,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
   arg->msg = *pMsg;
   arg->pEpset = tEpSet;
 
-  schedMsg.fp = doProcessMsgFromServer;
-  schedMsg.ahandle = arg;
-  taosScheduleTask(tscQhandle, &schedMsg);
+  taosAsyncExec(doProcessMsgFromServer, arg, NULL);
 }
 
 TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, const char* db, uint16_t port) {

source/client/src/clientMain.c

@@ -49,7 +49,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
 }
 // this function may be called by user or system, or by both simultaneously.
 void taos_cleanup(void) {
-  tscInfo("start  to cleanup client environment");
+  tscDebug("start to cleanup client environment");
   if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
     return;
   }
@@ -58,7 +58,10 @@ void taos_cleanup(void) {
   clientReqRefPool = -1;
   taosCloseRef(id);
 
-  cleanupTaskQueue();
+  hbMgrCleanUp();
+
+  catalogDestroy();
+  schedulerDestroy();
 
   fmFuncMgtDestroy();
   qCleanupKeywordsTable();
@@ -67,13 +70,11 @@ void taos_cleanup(void) {
   clientConnRefPool = -1;
   taosCloseRef(id);
 
-  hbMgrCleanUp();
+  rpcCleanup();
+  tscDebug("rpc cleanup");
 
-  catalogDestroy();
-  schedulerDestroy();
+  cleanupTaskQueue();
 
-  cleanupTscQhandle();
-  rpcCleanup();
   tscInfo("all local resources released");
   taosCleanupCfg();
   taosCloseLog();
@@ -242,7 +243,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
 #endif
 
   } else if (TD_RES_TMQ(res)) {
-    SMqRspObj *     msg = ((SMqRspObj *)res);
+    SMqRspObj      *msg = ((SMqRspObj *)res);
     SReqResultInfo *pResultInfo;
     if (msg->resIter == -1) {
       pResultInfo = tmqGetNextResInfo(res, true);
@@ -418,7 +419,7 @@ int taos_affected_rows(TAOS_RES *res) {
     return 0;
   }
 
-  SRequestObj *   pRequest = (SRequestObj *)res;
+  SRequestObj    *pRequest = (SRequestObj *)res;
   SReqResultInfo *pResInfo = &pRequest->body.resInfo;
   return pResInfo->numOfRows;
 }
@@ -601,7 +602,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
   }
 
   SReqResultInfo *pResInfo = tscGetCurResInfo(res);
-  TAOS_FIELD *    pField = &pResInfo->userFields[columnIndex];
+  TAOS_FIELD     *pField = &pResInfo->userFields[columnIndex];
   if (!IS_VAR_DATA_TYPE(pField->type)) {
     return 0;
   }
@@ -645,8 +646,8 @@ const char *taos_get_server_info(TAOS *taos) {
 typedef struct SqlParseWrapper {
   SParseContext *pCtx;
   SCatalogReq    catalogReq;
-  SRequestObj *  pRequest;
-  SQuery *       pQuery;
+  SRequestObj   *pRequest;
+  SQuery        *pQuery;
 } SqlParseWrapper;
 
 static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) {
@@ -665,8 +666,8 @@ static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) {
 
 void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
   SqlParseWrapper *pWrapper = (SqlParseWrapper *)param;
-  SQuery *         pQuery = pWrapper->pQuery;
-  SRequestObj *    pRequest = pWrapper->pRequest;
+  SQuery          *pQuery = pWrapper->pQuery;
+  SRequestObj     *pRequest = pWrapper->pRequest;
 
   if (code == TSDB_CODE_SUCCESS) {
     code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery);
@@ -684,7 +685,8 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
 
     destorySqlParseWrapper(pWrapper);
 
-    tscDebug("0x%"PRIx64" analysis semantics completed, start async query, reqId:0x%"PRIx64, pRequest->self, pRequest->requestId);
+    tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, reqId:0x%" PRIx64, pRequest->self,
+             pRequest->requestId);
     launchAsyncQuery(pRequest, pQuery, pResultMeta);
   } else {
     destorySqlParseWrapper(pWrapper);
@@ -705,7 +707,7 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
 }
 
 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
-  int64_t  connId = *(int64_t*)taos;
+  int64_t connId = *(int64_t *)taos;
   taosAsyncQueryImpl(connId, sql, fp, param, false);
 }
 
@@ -739,7 +741,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
 
 void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
   SParseContext *pCxt = NULL;
-  STscObj *      pTscObj = pRequest->pTscObj;
+  STscObj       *pTscObj = pRequest->pTscObj;
   int32_t        code = 0;
 
   if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) {
@@ -852,31 +854,28 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
   }
 
   // all data has returned to App already, no need to try again
-  if (pResultInfo->completed && (pRequest->body.queryJob != 0)) {
-    pResultInfo->numOfRows = 0;
-    pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows);
-    return;
-  }
-
-  // it is a local executed query, no need to do async fetch
-  if (pRequest->body.queryJob == 0) {
-    ASSERT(pResultInfo->completed && pResultInfo->numOfRows >= 0);
-    if (pResultInfo->localResultFetched) {
-      pResultInfo->numOfRows = 0;
-      pResultInfo->current = 0;
-      pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows);
+  if (pResultInfo->completed) {
+    // it is a local executed query, no need to do async fetch
+    if (QUERY_EXEC_MODE_LOCAL == pRequest->body.execMode) {
+      ASSERT(pResultInfo->numOfRows >= 0);
+      if (pResultInfo->localResultFetched) {
+        pResultInfo->numOfRows = 0;
+        pResultInfo->current = 0;
+      } else {
+        pResultInfo->localResultFetched = true;
+      }
     } else {
-      pResultInfo->localResultFetched = true;
-      pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows);
+      pResultInfo->numOfRows = 0;
     }
+    
+    pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows);
     return;
   }
 
-
   SSchedulerReq req = {
-    .syncReq = false,
-    .fetchFp = fetchCallback,
-    .cbParam = pRequest,
+      .syncReq = false,
+      .fetchFp = fetchCallback,
+      .cbParam = pRequest,
   };
 
   schedulerFetchRows(pRequest->body.queryJob, &req);
@@ -886,7 +885,7 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
   ASSERT(res != NULL && fp != NULL);
   ASSERT(TD_RES_QUERY(res));
 
-  SRequestObj *pRequest = res;
+  SRequestObj    *pRequest = res;
   SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
 
   // set the current block is all consumed
@@ -928,7 +927,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   int64_t       connId = *(int64_t *)taos;
   const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024;  // 12MB list
   int32_t       code = 0;
-  SRequestObj * pRequest = NULL;
+  SRequestObj  *pRequest = NULL;
   SCatalogReq   catalogReq = {0};
 
   if (NULL == tableNameList) {
@@ -950,7 +949,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
     goto _return;
   }
 
-  STscObj* pTscObj = pRequest->pTscObj;
+  STscObj *pTscObj = pRequest->pTscObj;
   code = transferTableNameList(tableNameList, pTscObj->acctId, pTscObj->db, &catalogReq.pTableMeta);
   if (code) {
     goto _return;
@@ -972,7 +971,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
     goto _return;
   }
 
-  SSyncQueryParam* pParam = pRequest->body.param;
+  SSyncQueryParam *pParam = pRequest->body.param;
   tsem_wait(&pParam->sem);
 
 _return:

source/client/src/clientMsgHandler.c

@@ -179,7 +179,6 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   if (code != 0) {
     terrno = code;
     if (output.dbVgroup) taosHashCleanup(output.dbVgroup->vgHash);
-    taosMemoryFreeClear(output.dbVgroup);
 
     tscError("0x%" PRIx64 " failed to build use db output since %s", pRequest->requestId, terrstr());
   } else if (output.dbVgroup && output.dbVgroup->vgHash) {
@@ -189,12 +188,14 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     if (code1 != TSDB_CODE_SUCCESS) {
       tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
               tstrerror(code1));
-      taosMemoryFreeClear(output.dbVgroup);
     } else {
       catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
+      output.dbVgroup = NULL;
     }
   }
 
+  taosMemoryFreeClear(output.dbVgroup);
+
   tFreeSUsedbRsp(&usedbRsp);
 
   char db[TSDB_DB_NAME_LEN] = {0};

source/client/src/clientSml.c

@@ -2256,7 +2256,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
     (*pMeta)->tableMeta->uid = tableData->uid;  // one table merge data block together according uid
 
     code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat,
-                       (*pMeta)->tableMeta, tableData->childTableName, info->msgBuf.buf, info->msgBuf.len);
+                       (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->msgBuf.buf, info->msgBuf.len);
     if (code != TSDB_CODE_SUCCESS) {
       uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
       return code;

source/client/src/clientStmt.c

@@ -136,7 +136,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName) {
+int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, const char* sTableName) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   strncpy(pStmt->bInfo.tbFName, tbFName, sizeof(pStmt->bInfo.tbFName) - 1);
@@ -147,6 +147,7 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags,
   pStmt->bInfo.tbType = pTableMeta->tableType;
   pStmt->bInfo.boundTags = tags;
   pStmt->bInfo.tagsCached = false;
+  strcpy(pStmt->bInfo.stbFName, sTableName);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -162,10 +163,10 @@ int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockH
 }
 
 int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, char* tbFName, bool autoCreateTbl,
-                       SHashObj* pVgHash, SHashObj* pBlockHash) {
+                       SHashObj* pVgHash, SHashObj* pBlockHash, const char* sTableName) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName));
+  STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbFName, sTableName));
   STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl));
 
   pStmt->sql.autoCreateTbl = autoCreateTbl;
@@ -253,7 +254,7 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) {
     destroyBoundColumnInfo(pStmt->bInfo.boundTags);
     taosMemoryFreeClear(pStmt->bInfo.boundTags);
   }
-
+  memset(pStmt->bInfo.stbFName, 0, TSDB_TABLE_FNAME_LEN);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -592,7 +593,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
   }
 
   tscDebug("start to bind stmt tag values");
-  STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.sname.tname,
+  STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, pStmt->bInfo.sname.tname,
                                   tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
 
   return TSDB_CODE_SUCCESS;

source/common/src/tdatablock.c

@@ -320,7 +320,9 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
     memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
   } else {
     memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows));
-    memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
+    if (pSource->pData) {
+      memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
+    }
   }
 
   pColumnInfoData->hasNull = pSource->hasNull;
@@ -540,8 +542,10 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
 }
 
 int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
-  pBlock->info.rows = *(int32_t*)buf;
+  int32_t numOfRows = *(int32_t*) buf;
+  blockDataEnsureCapacity(pBlock, numOfRows);
 
+  pBlock->info.rows = numOfRows;
   size_t      numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   const char* pStart = buf + sizeof(uint32_t);
 
@@ -587,6 +591,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
   return TSDB_CODE_SUCCESS;
 }
 
+// todo remove this
 int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) {
   pBlock->info.rows = *(int32_t*)buf;
   pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t));
@@ -1172,8 +1177,6 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows)
 
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
   int32_t code = 0;
-  // ASSERT(numOfRows > 0);
-
   if (numOfRows == 0) {
     return TSDB_CODE_SUCCESS;
   }
@@ -1194,15 +1197,28 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
   return TSDB_CODE_SUCCESS;
 }
 
+void blockDataFreeRes(SSDataBlock* pBlock) {
+  int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock);
+  for (int32_t i = 0; i < numOfOutput; ++i) {
+    SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
+    colDataDestroy(pColInfoData);
+  }
+
+  taosArrayDestroy(pBlock->pDataBlock);
+  taosMemoryFreeClear(pBlock->pBlockAgg);
+  memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
+}
+
 void* blockDataDestroy(SSDataBlock* pBlock) {
   if (pBlock == NULL) {
     return NULL;
   }
 
-  blockDestroyInner(pBlock);
+  blockDataFreeRes(pBlock);
   taosMemoryFreeClear(pBlock);
   return NULL;
 }
+
 int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
   ASSERT(src != NULL);
 
@@ -1736,56 +1752,57 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
   int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
   int32_t rows = pDataBlock->info.rows;
   int32_t len = 0;
-  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id:%" PRIu64 "|\n", flag,
-                  (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
+  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld\n", flag,
+                  (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId,
+                  pDataBlock->info.uid);
   if (len >= size - 1) return dumpBuf;
 
   for (int32_t j = 0; j < rows; j++) {
     len += snprintf(dumpBuf + len, size - len, "%s |", flag);
-    if (len >= size -1) return dumpBuf;
+    if (len >= size - 1) return dumpBuf;
 
     for (int32_t k = 0; k < colNum; k++) {
       SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
       void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
       if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) {
         len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL");
-        if (len >= size -1) return dumpBuf;
+        if (len >= size - 1) return dumpBuf;
         continue;
       }
       switch (pColInfoData->info.type) {
         case TSDB_DATA_TYPE_TIMESTAMP:
           formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
           len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf);
-          if (len >= size -1) return dumpBuf;
+          if (len >= size - 1) return dumpBuf;
           break;
         case TSDB_DATA_TYPE_INT:
           len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var);
-          if (len >= size -1) return dumpBuf;
+          if (len >= size - 1) return dumpBuf;
           break;
         case TSDB_DATA_TYPE_UINT:
           len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var);
-          if (len >= size -1) return dumpBuf;
+          if (len >= size - 1) return dumpBuf;
           break;
         case TSDB_DATA_TYPE_BIGINT:
           len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var);
-          if (len >= size -1) return dumpBuf;
+          if (len >= size - 1) return dumpBuf;
           break;
         case TSDB_DATA_TYPE_UBIGINT:
           len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var);
-          if (len >= size -1) return dumpBuf;
+          if (len >= size - 1) return dumpBuf;
           break;
         case TSDB_DATA_TYPE_FLOAT:
           len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var);
-          if (len >= size -1) return dumpBuf;
+          if (len >= size - 1) return dumpBuf;
           break;
         case TSDB_DATA_TYPE_DOUBLE:
           len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var);
-          if (len >= size -1) return dumpBuf;
+          if (len >= size - 1) return dumpBuf;
           break;
       }
     }
     len += snprintf(dumpBuf + len, size - len, "\n");
-    if (len >= size -1) return dumpBuf;
+    if (len >= size - 1) return dumpBuf;
   }
   len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag);
   return dumpBuf;

source/common/src/tdataformat.c

@@ -175,7 +175,8 @@ static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) {
   } while (0)
 
 int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) {
-  int32_t   code = 0;
+  int32_t code = 0;
+#if 0
   STColumn *pTColumn;
   SColVal  *pColVal;
   int32_t   nColVal = taosArrayGetSize(pArray);
@@ -462,30 +463,22 @@ int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, S
     }
   }
 
+#endif
 _exit:
   return code;
 }
 
 int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow) {
   int32_t code = 0;
+  int32_t rLen;
 
-  (*ppRow) = (STSRow2 *)taosMemoryMalloc(sizeof(**ppRow));
+  TSROW_LEN(pRow, rLen);
+  (*ppRow) = (STSRow2 *)taosMemoryMalloc(rLen);
   if (*ppRow == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
-  **ppRow = *pRow;
-  (*ppRow)->pData = NULL;
-
-  if (pRow->nData) {
-    (*ppRow)->pData = taosMemoryMalloc(pRow->nData);
-    if ((*ppRow)->pData == NULL) {
-      taosMemoryFree(*ppRow);
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _exit;
-    }
-    memcpy((*ppRow)->pData, pRow->pData, pRow->nData);
-  }
+  memcpy(*ppRow, pRow, rLen);
 
 _exit:
   return code;
@@ -493,12 +486,12 @@ _exit:
 
 void tTSRowFree(STSRow2 *pRow) {
   if (pRow) {
-    if (pRow->pData) taosMemoryFree(pRow->pData);
     taosMemoryFree(pRow);
   }
 }
 
 void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
+#if 0
   uint8_t   isTuple = ((pRow->flags & 0xf0) == 0) ? 1 : 0;
   STColumn *pTColumn = &pTSchema->columns[iCol];
   uint8_t   flags = pRow->flags & (uint8_t)0xf;
@@ -643,10 +636,12 @@ _return_null:
 _return_value:
   *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value);
   return;
+#endif
 }
 
 int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) {
   int32_t code = 0;
+#if 0
   SColVal cv;
 
   (*ppArray) = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal));
@@ -660,52 +655,27 @@ int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray) {
     taosArrayPush(*ppArray, &cv);
   }
 
+#endif
 _exit:
   return code;
 }
 
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) {
-  int32_t n = 0;
+  int32_t n;
 
-  n += tPutI64(p ? p + n : p, pRow->ts);
-  n += tPutI8(p ? p + n : p, pRow->flags);
-  n += tPutI32v(p ? p + n : p, pRow->sver);
-
-  ASSERT(pRow->flags & 0xf);
-
-  switch (pRow->flags & 0xf) {
-    case TSROW_HAS_NONE:
-    case TSROW_HAS_NULL:
-      ASSERT(pRow->nData == 0);
-      ASSERT(pRow->pData == NULL);
-      break;
-    default:
-      ASSERT(pRow->nData && pRow->pData);
-      n += tPutBinary(p ? p + n : p, pRow->pData, pRow->nData);
-      break;
+  TSROW_LEN(pRow, n);
+  if (p) {
+    memcpy(p, pRow, n);
   }
 
   return n;
 }
 
-int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow) {
-  int32_t n = 0;
+int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) {
+  int32_t n;
 
-  n += tGetI64(p + n, &pRow->ts);
-  n += tGetI8(p + n, &pRow->flags);
-  n += tGetI32v(p + n, &pRow->sver);
-
-  ASSERT(pRow->flags);
-  switch (pRow->flags & 0xf) {
-    case TSROW_HAS_NONE:
-    case TSROW_HAS_NULL:
-      pRow->nData = 0;
-      pRow->pData = NULL;
-      break;
-    default:
-      n += tGetBinary(p + n, &pRow->pData, &pRow->nData);
-      break;
-  }
+  *ppRow = (STSRow2 *)p;
+  TSROW_LEN(*ppRow, n);
 
   return n;
 }
@@ -904,15 +874,13 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
   return n;
 }
 
-bool tTagIsJson(const void *pTag){
-  return (((const STag *)pTag)->flags & TD_TAG_JSON);
-}
+bool tTagIsJson(const void *pTag) { return (((const STag *)pTag)->flags & TD_TAG_JSON); }
 
-bool tTagIsJsonNull(void *data){
-  STag *pTag = (STag*)data;
-  int8_t   isJson = tTagIsJson(pTag);
-  if(!isJson) return false;
-  return ((STag*)data)->nTag == 0;
+bool tTagIsJsonNull(void *data) {
+  STag  *pTag = (STag *)data;
+  int8_t isJson = tTagIsJson(pTag);
+  if (!isJson) return false;
+  return ((STag *)data)->nTag == 0;
 }
 
 int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) {
@@ -1097,112 +1065,6 @@ _err:
 }
 
 #if 1  // ===================================================================================================================
-static void dataColSetNEleNull(SDataCol *pCol, int nEle);
-int         tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
-          int spaceNeeded = pCol->bytes * maxPoints;
-          if (IS_VAR_DATA_TYPE(pCol->type)) {
-            spaceNeeded += sizeof(VarDataOffsetT) * maxPoints;
-  }
-#ifdef TD_SUPPORT_BITMAP
-  int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(maxPoints);
-  spaceNeeded += (int)nBitmapBytes;
-  // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more
-  // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered.
-  // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus
-  // remove the additional space
-#endif
-
-  if (pCol->spaceSize < spaceNeeded) {
-    void *ptr = taosMemoryRealloc(pCol->pData, spaceNeeded);
-    if (ptr == NULL) {
-      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, strerror(errno));
-      return -1;
-    } else {
-      pCol->pData = ptr;
-      pCol->spaceSize = spaceNeeded;
-    }
-  }
-#ifdef TD_SUPPORT_BITMAP
-
-  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
-    pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes);
-  } else {
-    pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
-  }
-#else
-  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    pCol->dataOff = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
-  }
-#endif
-  return 0;
-}
-
-/**
- * Duplicate the schema and return a new object
- */
-STSchema *tdDupSchema(const STSchema *pSchema) {
-  int       tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema);
-  STSchema *tSchema = (STSchema *)taosMemoryMalloc(tlen);
-  if (tSchema == NULL) return NULL;
-
-  memcpy((void *)tSchema, (void *)pSchema, tlen);
-
-  return tSchema;
-}
-
-/**
- * Encode a schema to dst, and return the next pointer
- */
-int tdEncodeSchema(void **buf, STSchema *pSchema) {
-  int tlen = 0;
-  tlen += taosEncodeFixedI32(buf, schemaVersion(pSchema));
-  tlen += taosEncodeFixedI32(buf, schemaNCols(pSchema));
-
-  for (int i = 0; i < schemaNCols(pSchema); i++) {
-    STColumn *pCol = schemaColAt(pSchema, i);
-    tlen += taosEncodeFixedI8(buf, colType(pCol));
-    tlen += taosEncodeFixedI8(buf, colFlags(pCol));
-    tlen += taosEncodeFixedI16(buf, colColId(pCol));
-    tlen += taosEncodeFixedI16(buf, colBytes(pCol));
-  }
-
-  return tlen;
-}
-
-/**
- * Decode a schema from a binary.
- */
-void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
-  int             version = 0;
-  int             numOfCols = 0;
-  STSchemaBuilder schemaBuilder;
-
-  buf = taosDecodeFixedI32(buf, &version);
-  buf = taosDecodeFixedI32(buf, &numOfCols);
-
-  if (tdInitTSchemaBuilder(&schemaBuilder, version) < 0) return NULL;
-
-  for (int i = 0; i < numOfCols; i++) {
-    col_type_t  type = 0;
-    int8_t      flags = 0;
-    col_id_t    colId = 0;
-    col_bytes_t bytes = 0;
-    buf = taosDecodeFixedI8(buf, &type);
-    buf = taosDecodeFixedI8(buf, &flags);
-    buf = taosDecodeFixedI16(buf, &colId);
-    buf = taosDecodeFixedI32(buf, &bytes);
-    if (tdAddColToSchema(&schemaBuilder, type, flags, colId, bytes) < 0) {
-      tdDestroyTSchemaBuilder(&schemaBuilder);
-      return NULL;
-    }
-  }
-
-  *pRSchema = tdGetSchemaFromBuilder(&schemaBuilder);
-  tdDestroyTSchemaBuilder(&schemaBuilder);
-  return buf;
-}
-
 int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   if (pBuilder == NULL) return -1;
 
@@ -1239,22 +1101,22 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, c
   }
 
   STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]);
-  colSetType(pCol, type);
-  colSetColId(pCol, colId);
-  colSetFlags(pCol, flags);
+  pCol->type = type;
+  pCol->colId = colId;
+  pCol->flags = flags;
   if (pBuilder->nCols == 0) {
-    colSetOffset(pCol, 0);
+    pCol->offset = 0;
   } else {
     STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols - 1]);
-    colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
+    pCol->offset = pTCol->offset + TYPE_BYTES[pTCol->type];
   }
 
   if (IS_VAR_DATA_TYPE(type)) {
-    colSetBytes(pCol, bytes);
+    pCol->bytes = bytes;
     pBuilder->tlen += (TYPE_BYTES[type] + bytes);
     pBuilder->vlen += bytes - sizeof(VarDataLenT);
   } else {
-    colSetBytes(pCol, TYPE_BYTES[type]);
+    pCol->bytes = TYPE_BYTES[type];
     pBuilder->tlen += TYPE_BYTES[type];
     pBuilder->vlen += TYPE_BYTES[type];
   }
@@ -1275,151 +1137,19 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
   STSchema *pSchema = (STSchema *)taosMemoryMalloc(tlen);
   if (pSchema == NULL) return NULL;
 
-  schemaVersion(pSchema) = pBuilder->version;
-  schemaNCols(pSchema) = pBuilder->nCols;
-  schemaTLen(pSchema) = pBuilder->tlen;
-  schemaFLen(pSchema) = pBuilder->flen;
-  schemaVLen(pSchema) = pBuilder->vlen;
+  pSchema->version = pBuilder->version;
+  pSchema->numOfCols = pBuilder->nCols;
+  pSchema->tlen = pBuilder->tlen;
+  pSchema->flen = pBuilder->flen;
+  pSchema->vlen = pBuilder->vlen;
 
 #ifdef TD_SUPPORT_BITMAP
-  schemaTLen(pSchema) += (int)TD_BITMAP_BYTES(schemaNCols(pSchema));
+  pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols);
 #endif
 
-  memcpy(schemaColAt(pSchema, 0), pBuilder->columns, sizeof(STColumn) * pBuilder->nCols);
+  memcpy(&pSchema->columns[0], pBuilder->columns, sizeof(STColumn) * pBuilder->nCols);
 
   return pSchema;
 }
 
-void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
-  pDataCol->type = colType(pCol);
-  pDataCol->colId = colColId(pCol);
-  pDataCol->bytes = colBytes(pCol);
-  pDataCol->offset = colOffset(pCol) + 0;  // TD_DATA_ROW_HEAD_SIZE;
-
-  pDataCol->len = 0;
-}
-
-static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) {
-  if (IS_VAR_DATA_TYPE(pCol->type)) {
-    return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
-  } else {
-    return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
-  }
-}
-
-bool isNEleNull(SDataCol *pCol, int nEle) {
-  if (isAllRowsNull(pCol)) return true;
-  for (int i = 0; i < nEle; ++i) {
-    if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false;
-  }
-  return true;
-}
-
-void *dataColSetOffset(SDataCol *pCol, int nEle) {
-  ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR)));
-
-  void *tptr = pCol->pData;
-  // char *tptr = (char *)(pCol->pData);
-
-  VarDataOffsetT offset = 0;
-  for (int i = 0; i < nEle; ++i) {
-    pCol->dataOff[i] = offset;
-    offset += varDataTLen(tptr);
-    tptr = POINTER_SHIFT(tptr, varDataTLen(tptr));
-  }
-  return POINTER_SHIFT(tptr, varDataTLen(tptr));
-}
-
-SDataCols *tdNewDataCols(int maxCols, int maxRows) {
-  SDataCols *pCols = (SDataCols *)taosMemoryCalloc(1, sizeof(SDataCols));
-  if (pCols == NULL) {
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno));
-    return NULL;
-  }
-
-  pCols->maxPoints = maxRows;
-  pCols->maxCols = maxCols;
-  pCols->numOfRows = 0;
-  pCols->numOfCols = 0;
-  pCols->bitmapMode = TSDB_BITMODE_DEFAULT;
-
-  if (maxCols > 0) {
-    pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol));
-    if (pCols->cols == NULL) {
-      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
-             strerror(errno));
-      tdFreeDataCols(pCols);
-      return NULL;
-    }
-#if 0  // no need as calloc used
-    int i;
-    for (i = 0; i < maxCols; i++) {
-      pCols->cols[i].spaceSize = 0;
-      pCols->cols[i].len = 0;
-      pCols->cols[i].pData = NULL;
-      pCols->cols[i].dataOff = NULL;
-    }
-#endif
-  }
-
-  return pCols;
-}
-
-int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
-  int i;
-  int oldMaxCols = pCols->maxCols;
-  if (schemaNCols(pSchema) > oldMaxCols) {
-    pCols->maxCols = schemaNCols(pSchema);
-    void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
-    if (ptr == NULL) return -1;
-    pCols->cols = ptr;
-    for (i = oldMaxCols; i < pCols->maxCols; ++i) {
-      pCols->cols[i].pData = NULL;
-      pCols->cols[i].dataOff = NULL;
-      pCols->cols[i].pBitmap = NULL;
-      pCols->cols[i].spaceSize = 0;
-    }
-  }
-#if 0
-  tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...)
-#endif
-
-  pCols->numOfRows = 0;
-  pCols->bitmapMode = TSDB_BITMODE_DEFAULT;
-  pCols->numOfCols = schemaNCols(pSchema);
-
-  for (i = 0; i < schemaNCols(pSchema); ++i) {
-    dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints);
-  }
-
-  return 0;
-}
-
-SDataCols *tdFreeDataCols(SDataCols *pCols) {
-  int i;
-  if (pCols) {
-    if (pCols->cols) {
-      int maxCols = pCols->maxCols;
-      for (i = 0; i < maxCols; ++i) {
-        SDataCol *pCol = &pCols->cols[i];
-        taosMemoryFreeClear(pCol->pData);
-      }
-      taosMemoryFree(pCols->cols);
-      pCols->cols = NULL;
-    }
-    taosMemoryFree(pCols);
-  }
-  return NULL;
-}
-
-void tdResetDataCols(SDataCols *pCols) {
-  if (pCols != NULL) {
-    pCols->numOfRows = 0;
-    pCols->bitmapMode = 0;
-    for (int i = 0; i < pCols->maxCols; ++i) {
-      dataColReset(pCols->cols + i);
-    }
-  }
-}
-
 #endif
\ No newline at end of file

source/common/src/tglobal.c

@@ -55,6 +55,7 @@ int32_t tsNumOfMnodeQueryThreads = 2;
 int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 2;
+int32_t tsNumOfVnodeStreamThreads = 2;
 int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeWriteThreads = 2;
 int32_t tsNumOfVnodeSyncThreads = 2;
@@ -412,7 +413,12 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 2);
   if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1;
 
-  tsNumOfVnodeFetchThreads = TRANGE(tsNumOfVnodeFetchThreads, 1, 1);
+  tsNumOfVnodeStreamThreads = tsNumOfCores / 4;
+  tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4);
+  if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 1, 1024, 0) != 0) return -1;
+
+  tsNumOfVnodeFetchThreads = tsNumOfCores / 4;
+  tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1024, 0) != 0) return -1;
 
   tsNumOfVnodeWriteThreads = tsNumOfCores;
@@ -586,6 +592,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsNumOfMnodeQueryThreads = cfgGetItem(pCfg, "numOfMnodeQueryThreads")->i32;
   tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
   tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
+  tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32;
   tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
   tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
   tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
@@ -1116,10 +1123,44 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
   if (strncasecmp(option, "debugFlag", 9) == 0) {
     int32_t flag = atoi(value);
     taosSetAllDebugFlag(flag);
+    return;
   }
 
   if (strcasecmp(option, "resetlog") == 0) {
     taosResetLog();
     cfgDumpCfg(tsCfg, 0, false);
+    return;
+  }
+
+  if (strcasecmp(option, "monitor") == 0) {
+    int32_t monitor = atoi(value);
+    uInfo("monitor set from %d to %d", tsEnableMonitor, monitor);
+    tsEnableMonitor = monitor;
+    return;
+  }
+
+  const char *options[] = {
+      "dDebugFlag",  "vDebugFlag",   "mDebugFlag",   "wDebugFlag",   "sDebugFlag",   "tsdbDebugFlag",
+      "tqDebugFlag", "fsDebugFlag",  "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tmrDebugFlag",
+      "uDebugFlag",  "smaDebugFlag", "rpcDebugFlag", "qDebugFlag",
+  };
+  int32_t *optionVars[] = {
+      &dDebugFlag,  &vDebugFlag,   &mDebugFlag,   &wDebugFlag,   &sDebugFlag,   &tsdbDebugFlag,
+      &tqDebugFlag, &fsDebugFlag,  &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tmrDebugFlag,
+      &uDebugFlag,  &smaDebugFlag, &rpcDebugFlag, &qDebugFlag,
+  };
+
+  int32_t optionSize = tListLen(options);
+  for (int32_t d = 0; d < optionSize; ++d) {
+    const char *optName = options[d];
+    int32_t     optLen = strlen(optName);
+    if (strncasecmp(option, optName, optLen) != 0) continue;
+
+    int32_t flag = atoi(value);
+    uInfo("%s set from %d to %d", optName, *optionVars[d], flag);
+    *optionVars[d] = flag;
+    return;
   }
+
+  uError("failed to cfg dynamic option:%s value:%s", option, value);
 }

source/common/src/tmsg.c

@@ -4871,6 +4871,11 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
     if (tEncodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1;
   }
 
+  if (tEncodeI32(pCoder, pReq->alterOriDataLen) < 0) return -1;
+  if (pReq->alterOriDataLen > 0) {
+    if (tEncodeBinary(pCoder, pReq->alterOriData, pReq->alterOriDataLen) < 0) return -1;
+  }
+
   tEndEncode(pCoder);
   return 0;
 }
@@ -4887,6 +4892,11 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
     if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1;
   }
 
+  if (tDecodeI32(pCoder, &pReq->alterOriDataLen) < 0) return -1;
+  if (pReq->alterOriDataLen > 0) {
+    if (tDecodeBinary(pCoder, (uint8_t **)&pReq->alterOriData, NULL) < 0) return -1;
+  }
+
   tEndDecode(pCoder);
   return 0;
 }
@@ -4930,8 +4940,15 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
   }
 
   if (pReq->type == TSDB_CHILD_TABLE) {
+    if (tEncodeCStr(pCoder, pReq->ctb.name) < 0) return -1;
     if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1;
     if (tEncodeTag(pCoder, (const STag *)pReq->ctb.pTag) < 0) return -1;
+    int32_t len = taosArrayGetSize(pReq->ctb.tagName);
+    if (tEncodeI32(pCoder, len) < 0) return -1;
+    for (int32_t i = 0; i < len; i++){
+      char* name = taosArrayGet(pReq->ctb.tagName, i);
+      if (tEncodeCStr(pCoder, name) < 0) return -1;
+    }
   } else if (pReq->type == TSDB_NORMAL_TABLE) {
     if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1;
   } else {
@@ -4959,10 +4976,22 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
   }
 
   if (pReq->type == TSDB_CHILD_TABLE) {
+    if (tDecodeCStr(pCoder, &pReq->ctb.name) < 0) return -1;
     if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1;
     if (tDecodeTag(pCoder, (STag **)&pReq->ctb.pTag) < 0) return -1;
+    int32_t len = 0;
+    if (tDecodeI32(pCoder, &len) < 0) return -1;
+    pReq->ctb.tagName = taosArrayInit(len, TSDB_COL_NAME_LEN);
+    if(pReq->ctb.tagName == NULL) return -1;
+    for (int32_t i = 0; i < len; i++){
+      char name[TSDB_COL_NAME_LEN] = {0};
+      char *tmp = NULL;
+      if (tDecodeCStr(pCoder, &tmp) < 0) return -1;
+      strcpy(name, tmp);
+      taosArrayPush(pReq->ctb.tagName, name);
+    }
   } else if (pReq->type == TSDB_NORMAL_TABLE) {
-    if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1;
+    if (tDecodeSSchemaWrapperEx(pCoder, &pReq->ntb.schemaRow) < 0) return -1;
   } else {
     ASSERT(0);
   }
@@ -5292,6 +5321,7 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
       break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
       if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->colModType) < 0) return -1;
       if (tEncodeI32v(pEncoder, pReq->colModBytes) < 0) return -1;
       break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
@@ -5301,6 +5331,7 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
     case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
       if (tEncodeCStr(pEncoder, pReq->tagName) < 0) return -1;
       if (tEncodeI8(pEncoder, pReq->isNull) < 0) return -1;
+      if (tEncodeI8(pEncoder, pReq->tagType) < 0) return -1;
       if (!pReq->isNull) {
         if (tEncodeBinary(pEncoder, pReq->pTagVal, pReq->nTagVal) < 0) return -1;
       }
@@ -5340,6 +5371,7 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
       break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
       if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->colModType) < 0) return -1;
       if (tDecodeI32v(pDecoder, &pReq->colModBytes) < 0) return -1;
       break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:
@@ -5349,6 +5381,7 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
     case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:
       if (tDecodeCStr(pDecoder, &pReq->tagName) < 0) return -1;
       if (tDecodeI8(pDecoder, &pReq->isNull) < 0) return -1;
+      if (tDecodeI8(pDecoder, &pReq->tagType) < 0) return -1;
       if (!pReq->isNull) {
         if (tDecodeBinary(pDecoder, &pReq->pTagVal, &pReq->nTagVal) < 0) return -1;
       }
@@ -5526,6 +5559,11 @@ bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) {
       ASSERT(0);
       // TODO
       return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts;
+    } else {
+      ASSERT(0);
+      /*ASSERT(pLeft->type == TMQ_OFFSET__RESET_NONE || pLeft->type == TMQ_OFFSET__RESET_EARLIEAST ||*/
+      /*pLeft->type == TMQ_OFFSET__RESET_LATEST);*/
+      /*return true;*/
     }
   }
   return false;

source/common/src/ttime.c

@@ -710,6 +710,32 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
   return (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision));
 }
 
+int64_t taosTimeSub(int64_t t, int64_t duration, char unit, int32_t precision) {
+  if (duration == 0) {
+    return t;
+  }
+
+  if (unit != 'n' && unit != 'y') {
+    return t - duration;
+  }
+
+  // The following code handles the y/n time duration
+  int64_t numOfMonth = duration;
+  if (unit == 'y') {
+    numOfMonth *= 12;
+  }
+
+  struct tm tm;
+  time_t    tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision));
+  taosLocalTime(&tt, &tm);
+  int32_t mon = tm.tm_year * 12 + tm.tm_mon - (int32_t)numOfMonth;
+  tm.tm_year = mon / 12;
+  tm.tm_mon = mon % 12;
+
+  return (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision));
+}
+
+
 int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision) {
   if (ekey < skey) {
     int64_t tmp = ekey;

source/common/test/dataformatTest.cpp

@@ -285,8 +285,8 @@ int32_t debugPrintSColVal(SColVal *cv, int8_t type) {
 }
 
 void debugPrintTSRow(STSRow2 *row, STSchema *pTSchema, const char *tags, int32_t ln) {
-  printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData);
-  for (int16_t i = 0; i < schemaNCols(pTSchema); ++i) {
+  // printf("%s:%d %s:v%d:%d ", tags, ln, (row->flags & 0xf0) ? "KV" : "TP", row->sver, row->nData);
+  for (int16_t i = 0; i < pTSchema->numOfCols; ++i) {
     SColVal cv = {0};
     tTSRowGet(row, pTSchema, i, &cv);
     debugPrintSColVal(&cv, pTSchema->columns[i].type);
@@ -393,7 +393,7 @@ static int32_t checkSColVal(const char *rawVal, SColVal *cv, int8_t type) {
 }
 
 static void checkTSRow(const char **data, STSRow2 *row, STSchema *pTSchema) {
-  for (int16_t i = 0; i < schemaNCols(pTSchema); ++i) {
+  for (int16_t i = 0; i < pTSchema->numOfCols; ++i) {
     SColVal cv = {0};
     tTSRowGet(row, pTSchema, i, &cv);
     checkSColVal(data[i], &cv, pTSchema->columns[i].type);

source/dnode/mgmt/mgmt_mnode/src/mmWorker.c

@@ -168,7 +168,12 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   memcpy(pMsg, pRpc, sizeof(SRpcMsg));
 
   dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
-  return mmPutMsgToWorker(pMgmt, pWorker, pMsg);
+  int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg);
+  if (code != 0) {
+    dTrace("msg:%p, is freed", pMsg);
+    taosFreeQitem(pMsg);
+  }
+  return code;
 }
 
 int32_t mmStartWorker(SMnodeMgmt *pMgmt) {

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -31,7 +31,8 @@ typedef struct SVnodeMgmt {
   const char    *path;
   const char    *name;
   SQWorkerPool   queryPool;
-  SQWorkerPool   fetchPool;
+  SQWorkerPool   streamPool;
+  SWWorkerPool   fetchPool;
   SWWorkerPool   syncPool;
   SWWorkerPool   writePool;
   SWWorkerPool   applyPool;
@@ -61,6 +62,7 @@ typedef struct {
   STaosQueue *pSyncQ;
   STaosQueue *pApplyQ;
   STaosQueue *pQueryQ;
+  STaosQueue *pStreamQ;
   STaosQueue *pFetchQ;
 } SVnodeObj;
 
@@ -105,6 +107,7 @@ int32_t vmPutMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
 int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
 int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
 int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
+int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
 int32_t vmPutMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
 int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
 int32_t vmPutMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);

source/dnode/mgmt/mgmt_vnode/src/vmFile.c

@@ -31,7 +31,7 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
     SVnodeObj  *pVnode = *ppVnode;
     if (pVnode && num < size) {
       int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
-      // dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+      // dTrace("vgId:%d, acquire vnode list, ref:%d", pVnode->vgId, refCount);
       pVnodes[num++] = (*ppVnode);
       pIter = taosHashIterate(pMgmt->hash, pIter);
     } else {

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -357,16 +357,16 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
 
-  if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -23,6 +23,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) {
   taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
   if (pVnode == NULL || pVnode->dropped) {
     terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
+    pVnode = NULL;
   } else {
     int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
     // dTrace("vgId:%d, acquire vnode, ref:%d", pVnode->vgId, refCount);
@@ -82,6 +83,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   taosThreadRwlockUnlock(&pMgmt->lock);
   vmReleaseVnode(pMgmt, pVnode);
 
+  dTrace("vgId:%d, wait for vnode ref become 0", pVnode->vgId);
   while (pVnode->refCount > 0) taosMsleep(10);
   dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId);
 
@@ -90,6 +92,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
+  while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10);
   dTrace("vgId:%d, vnode queue is empty", pVnode->vgId);
 
   vmFreeQueue(pMgmt, pVnode);

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -81,15 +81,15 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   taosFreeQitem(pMsg);
 }
 
-static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
+static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   SVnodeObj      *pVnode = pInfo->ahandle;
   const STraceId *trace = &pMsg->info.traceId;
 
-  dGTrace("vgId:%d, msg:%p get from vnode-fetch queue", pVnode->vgId, pMsg);
+  dGTrace("vgId:%d, msg:%p get from vnode-stream queue", pVnode->vgId, pMsg);
   int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
   if (code != 0) {
     if (terrno != 0) code = terrno;
-    dGError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr());
+    dGError("vgId:%d, msg:%p failed to stream since %s", pVnode->vgId, pMsg, terrstr());
     vmSendRsp(pMsg, code);
   }
 
@@ -98,6 +98,28 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   taosFreeQitem(pMsg);
 }
 
+static void vmProcessFetchQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+  SVnodeObj *pVnode = pInfo->ahandle;
+  SRpcMsg   *pMsg = NULL;
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+    const STraceId *trace = &pMsg->info.traceId;
+    dGTrace("vgId:%d, msg:%p get from vnode-fetch queue", pVnode->vgId, pMsg);
+
+    int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
+    if (code != 0) {
+      if (terrno != 0) code = terrno;
+      dGError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr());
+      vmSendRsp(pMsg, code);
+    }
+
+    dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
+  }
+}
+
 static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
   SRpcMsg   *pMsg = NULL;
@@ -135,6 +157,10 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
       dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg);
       taosWriteQitem(pVnode->pQueryQ, pMsg);
       break;
+    case STREAM_QUEUE:
+      dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg);
+      taosWriteQitem(pVnode->pStreamQ, pMsg);
+      break;
     case FETCH_QUEUE:
       dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg);
       taosWriteQitem(pVnode->pFetchQ, pMsg);
@@ -169,6 +195,8 @@ int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsg
 
 int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); }
 
+int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, STREAM_QUEUE); }
+
 int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   const STraceId *trace = &pMsg->info.traceId;
   dGTrace("msg:%p, put into vnode-mgmt queue", pMsg);
@@ -201,9 +229,9 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   int32_t code = vmPutMsgToQueue(pMgmt, pMsg, qtype);
   if (code != 0) {
     dTrace("msg:%p, is freed", pMsg);
-    taosFreeQitem(pMsg);
     rpcFreeCont(pMsg->pCont);
     pRpc->pCont = NULL;
+    taosFreeQitem(pMsg);
   }
 
   return code;
@@ -229,11 +257,14 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
       case FETCH_QUEUE:
         size = taosQueueItemSize(pVnode->pFetchQ);
         break;
+      case STREAM_QUEUE:
+        size = taosQueueItemSize(pVnode->pStreamQ);
+        break;
       default:
         break;
     }
+    vmReleaseVnode(pMgmt, pVnode);
   }
-  vmReleaseVnode(pMgmt, pVnode);
   return size;
 }
 
@@ -242,15 +273,21 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
   pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyWriteMsg);
   pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
-  pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue);
+  pVnode->pStreamQ = tQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue);
+  pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue);
 
   if (pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pApplyQ == NULL || pVnode->pQueryQ == NULL ||
-      pVnode->pFetchQ == NULL) {
+      pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
-  dDebug("vgId:%d, queue is alloced", pVnode->vgId);
+  dDebug("vgId:%d, write-queue:%p is alloced", pVnode->vgId, pVnode->pWriteQ);
+  dDebug("vgId:%d, sync-queue:%p is alloced", pVnode->vgId, pVnode->pSyncQ);
+  dDebug("vgId:%d, apply-queue:%p is alloced", pVnode->vgId, pVnode->pApplyQ);
+  dDebug("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
+  dDebug("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ);
+  dDebug("vgId:%d, fetch-queue:%p is alloced", pVnode->vgId, pVnode->pFetchQ);
   return 0;
 }
 
@@ -259,11 +296,13 @@ void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   tWWorkerFreeQueue(&pMgmt->applyPool, pVnode->pApplyQ);
   tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
   tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
-  tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
+  tQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ);
+  tWWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
   pVnode->pWriteQ = NULL;
   pVnode->pSyncQ = NULL;
   pVnode->pApplyQ = NULL;
   pVnode->pQueryQ = NULL;
+  pVnode->pStreamQ = NULL;
   pVnode->pFetchQ = NULL;
   dDebug("vgId:%d, queue is freed", pVnode->vgId);
 }
@@ -275,11 +314,16 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
   pQPool->max = tsNumOfVnodeQueryThreads;
   if (tQWorkerInit(pQPool) != 0) return -1;
 
-  SQWorkerPool *pFPool = &pMgmt->fetchPool;
+  SQWorkerPool *pStreamPool = &pMgmt->streamPool;
+  pStreamPool->name = "vnode-stream";
+  pStreamPool->min = tsNumOfVnodeStreamThreads;
+  pStreamPool->max = tsNumOfVnodeStreamThreads;
+  if (tQWorkerInit(pStreamPool) != 0) return -1;
+
+  SWWorkerPool *pFPool = &pMgmt->fetchPool;
   pFPool->name = "vnode-fetch";
-  pFPool->min = tsNumOfVnodeFetchThreads;
   pFPool->max = tsNumOfVnodeFetchThreads;
-  if (tQWorkerInit(pFPool) != 0) return -1;
+  if (tWWorkerInit(pFPool) != 0) return -1;
 
   SWWorkerPool *pWPool = &pMgmt->writePool;
   pWPool->name = "vnode-write";
@@ -325,6 +369,7 @@ void vmStopWorker(SVnodeMgmt *pMgmt) {
   tWWorkerCleanup(&pMgmt->applyPool);
   tWWorkerCleanup(&pMgmt->syncPool);
   tQWorkerCleanup(&pMgmt->queryPool);
-  tQWorkerCleanup(&pMgmt->fetchPool);
+  tQWorkerCleanup(&pMgmt->streamPool);
+  tWWorkerCleanup(&pMgmt->fetchPool);
   dDebug("vnode workers are closed");
 }

source/dnode/mnode/impl/src/mndDnode.c

@@ -781,7 +781,13 @@ _OVER:
 }
 
 static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
-  SMnode *pMnode = pReq->info.node;
+  SMnode     *pMnode = pReq->info.node;
+  const char *options[] = {
+      "debugFlag",     "dDebugFlag",  "vDebugFlag",   "mDebugFlag",   "wDebugFlag",   "sDebugFlag",
+      "tsdbDebugFlag", "tqDebugFlag", "fsDebugFlag",  "udfDebugFlag", "smaDebugFlag", "idxDebugFlag",
+      "tmrDebugFlag",  "uDebugFlag",  "smaDebugFlag", "rpcDebugFlag", "qDebugFlag",
+  };
+  int32_t optionSize = tListLen(options);
 
   SMCfgDnodeReq cfgReq = {0};
   if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
@@ -802,27 +808,52 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
   SEpSet epSet = mndGetDnodeEpset(pDnode);
   mndReleaseDnode(pMnode, pDnode);
 
+
   SDCfgDnodeReq dcfgReq = {0};
-  if (strncasecmp(cfgReq.config, "debugFlag", 9) == 0) {
+  if (strcasecmp(cfgReq.config, "resetlog") == 0) {
+    strcpy(dcfgReq.config, "resetlog");
+  } else if (strncasecmp(cfgReq.config, "monitor", 7) == 0) {
     const char *value = cfgReq.value;
     int32_t     flag = atoi(value);
     if (flag <= 0) {
-      flag = atoi(cfgReq.config + 10);
+      flag = atoi(cfgReq.config + 8);
     }
-    if (flag <= 0 || flag > 255) {
-      mError("dnode:%d, failed to config debugFlag since value:%d", cfgReq.dnodeId, flag);
+    if (flag < 0 || flag > 2) {
+      mError("dnode:%d, failed to config monitor since value:%d", cfgReq.dnodeId, flag);
       terrno = TSDB_CODE_INVALID_CFG;
       return -1;
     }
 
-    strcpy(dcfgReq.config, "debugFlag");
+    strcpy(dcfgReq.config, "monitor");
     snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strcasecmp(cfgReq.config, "resetlog") == 0) {
-    strcpy(dcfgReq.config, "resetlog");
   } else {
-    terrno = TSDB_CODE_INVALID_CFG;
-    mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr());
-    return -1;
+    bool findOpt = false;
+    for (int32_t d = 0; d < optionSize; ++d) {
+      const char *optName = options[d];
+      int32_t     optLen = strlen(optName);
+      if (strncasecmp(cfgReq.config, optName, optLen) != 0) continue;
+
+      const char *value = cfgReq.value;
+      int32_t flag = atoi(value);
+      if (flag <= 0) {
+        flag = atoi(cfgReq.config + optLen + 1);
+      }
+      if (flag <= 0 || flag > 255) {
+        mError("dnode:%d, failed to config %s since value:%d", cfgReq.dnodeId, optName, flag);
+        terrno = TSDB_CODE_INVALID_CFG;
+        return -1;
+      }
+
+      tstrncpy(dcfgReq.config, optName, optLen + 1);
+      snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
+      findOpt = true;
+    }
+
+    if (!findOpt) {
+      terrno = TSDB_CODE_INVALID_CFG;
+      mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr());
+      return -1;
+    }
   }
 
   int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq);
@@ -831,13 +862,14 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
   if (pBuf == NULL) return -1;
   tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq);
 
-  mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle);
+  mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle,
+        dcfgReq.config, dcfgReq.value);
   SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen};
   return tmsgSendReq(&epSet, &rpcMsg);
 }
 
 static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) {
-  mDebug("config rsp from dnode, app:%p", pRsp->info.ahandle);
+  mInfo("config rsp from dnode, app:%p", pRsp->info.ahandle);
   return 0;
 }
 

source/dnode/mnode/impl/src/mndMnode.c

@@ -218,6 +218,7 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) {
 }
 
 void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
+#if 0  
   SSdb   *pSdb = pMnode->pSdb;
   int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE);
   void   *pIter = NULL;
@@ -237,6 +238,9 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
     addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
     sdbRelease(pSdb, pObj);
   }
+#else
+  syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet);
+#endif
 }
 
 static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {

source/dnode/mnode/impl/src/mndStb.c

@@ -45,7 +45,7 @@ static int32_t  mndProcessTableMetaReq(SRpcMsg *pReq);
 static int32_t  mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextStb(SMnode *pMnode, void *pIter);
 static int32_t  mndProcessTableCfgReq(SRpcMsg *pReq);
-static int32_t  mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp);
+static int32_t  mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void* alterOriData, int32_t alterOriDataLen);
 
 int32_t mndInitStb(SMnode *pMnode) {
   SSdbTable table = {
@@ -409,7 +409,7 @@ static FORCE_INLINE int32_t schemaExColIdCompare(const void *colId, const void *
   return 0;
 }
 
-static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) {
+static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen, void* alterOriData, int32_t alterOriDataLen) {
   SEncoder       encoder = {0};
   int32_t        contLen;
   SName          name = {0};
@@ -422,6 +422,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.name = (char *)tNameGetTableName(&name);
   req.suid = pStb->uid;
   req.rollup = pStb->ast1Len > 0 ? 1 : 0;
+  req.alterOriData = alterOriData;
+  req.alterOriDataLen = alterOriDataLen;
   // todo
   req.schemaRow.nCols = pStb->numOfColumns;
   req.schemaRow.version = pStb->colVer;
@@ -626,7 +628,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
       continue;
     }
 
-    void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen);
+    void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen, NULL, 0);
     if (pReq == NULL) {
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
@@ -706,10 +708,10 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
   pDst->createdTime = taosGetTimestampMs();
   pDst->updateTime = pDst->createdTime;
-  pDst->uid = (pCreate->source == 1) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+  pDst->uid = (pCreate->source == TD_REQ_FROM_TAOX) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
   pDst->dbUid = pDb->uid;
-  pDst->tagVer = (pCreate->source != TD_REQ_FROM_APP) ? pCreate->tagVer : 1;
-  pDst->colVer = (pCreate->source != TD_REQ_FROM_APP) ? pCreate->colVer : 1;
+  pDst->tagVer = 1;
+  pDst->colVer = 1;
   pDst->smaVer = 1;
   pDst->nextColId = 1;
   pDst->maxdelay[0] = pCreate->delay1;
@@ -849,6 +851,75 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
   return 0;
 }
 
+static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) {
+  for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
+    if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) {
+      return tag;
+    }
+  }
+
+  return -1;
+}
+
+static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *colName) {
+  for (int32_t col = 0; col < pStb->numOfColumns; col++) {
+    if (strcasecmp(pStb->pColumns[col].name, colName) == 0) {
+      return col;
+    }
+  }
+
+  return -1;
+}
+
+static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq *createReq) {
+  taosRLockLatch(&pStb->lock);
+  memcpy(pDst, pStb, sizeof(SStbObj));
+  taosRUnLockLatch(&pStb->lock);
+
+  pDst->updateTime = taosGetTimestampMs();
+  pDst->numOfColumns = createReq->numOfColumns;
+  pDst->numOfTags = createReq->numOfTags;
+  pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema));
+  pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema));
+  if (pDst->pColumns == NULL || pDst->pTags == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  for (int32_t i = 0; i < pDst->numOfColumns; ++i) {
+    SField  *pField = taosArrayGet(createReq->pColumns, i);
+    SSchema *pSchema = &pDst->pColumns[i];
+    pSchema->type = pField->type;
+    pSchema->bytes = pField->bytes;
+    pSchema->flags = pField->flags;
+    memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
+    int32_t cIndex = mndFindSuperTableColumnIndex(pStb, pField->name);
+    if (cIndex >= 0){
+      pSchema->colId = pStb->pColumns[cIndex].colId;
+    }else{
+      pSchema->colId = pDst->nextColId++;
+    }
+  }
+
+  for (int32_t i = 0; i < pDst->numOfTags; ++i) {
+    SField  *pField = taosArrayGet(createReq->pTags, i);
+    SSchema *pSchema = &pDst->pTags[i];
+    pSchema->type = pField->type;
+    pSchema->bytes = pField->bytes;
+    memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
+    int32_t cIndex = mndFindSuperTableTagIndex(pStb, pField->name);
+    if (cIndex >= 0){
+      pSchema->colId = pStb->pTags[cIndex].colId;
+    }else{
+      pSchema->colId = pDst->nextColId++;
+    }
+
+  }
+  pDst->tagVer = createReq->tagVer;
+  pDst->colVer = createReq->colVer;
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
   SMnode        *pMnode = pReq->info.node;
   int32_t        code = -1;
@@ -881,9 +952,9 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
         terrno = TSDB_CODE_MND_STABLE_UID_NOT_MATCH;
         goto _OVER;
       } else if (createReq.tagVer > 0 || createReq.colVer > 0) {
-        int32_t tagDelta = pStb->tagVer - createReq.tagVer;
-        int32_t colDelta = pStb->colVer - createReq.colVer;
-        int32_t verDelta = tagDelta + verDelta;
+        int32_t tagDelta = createReq.tagVer - pStb->tagVer;
+        int32_t colDelta = createReq.colVer - pStb->colVer;
+        int32_t verDelta = tagDelta + colDelta;
         mInfo("stb:%s, already exist while create, input tagVer:%d colVer:%d, exist tagVer:%d colVer:%d",
               createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
         if (tagDelta <= 0 && colDelta <= 0) {
@@ -910,6 +981,10 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
     }
   } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) {
     goto _OVER;
+  } else if (createReq.source == TD_REQ_FROM_TAOX && (createReq.tagVer != 1 || createReq.colVer != 1)){
+    mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name);
+    code = 0;
+    goto _OVER;
   }
 
   pDb = mndAcquireDbByStb(pMnode, createReq.name);
@@ -934,7 +1009,16 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
 
   if (isAlter) {
     bool needRsp = false;
-    code = mndAlterStbImp(pMnode, pReq, pDb, pStb, needRsp);
+    SStbObj pDst = {0};
+    if (mndBuildStbFromAlter(pStb, &pDst, &createReq) != 0) {
+      taosMemoryFreeClear(pDst.pTags);
+      taosMemoryFreeClear(pDst.pColumns);
+      goto _OVER;
+    }
+
+    code = mndAlterStbImp(pMnode, pReq, pDb, &pDst, needRsp, NULL, 0);
+    taosMemoryFreeClear(pDst.pTags);
+    taosMemoryFreeClear(pDst.pColumns);
   } else {
     code = mndCreateStb(pMnode, pReq, &createReq, pDb);
   }
@@ -972,26 +1056,6 @@ static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) {
   return 0;
 }
 
-static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) {
-  for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
-    if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) {
-      return tag;
-    }
-  }
-
-  return -1;
-}
-
-static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *colName) {
-  for (int32_t col = 0; col < pStb->numOfColumns; col++) {
-    if (strcasecmp(pStb->pColumns[col].name, colName) == 0) {
-      return col;
-    }
-  }
-
-  return -1;
-}
-
 static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) {
   pNew->pTags = taosMemoryCalloc(pNew->numOfTags, sizeof(SSchema));
   pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema));
@@ -1315,7 +1379,7 @@ static int32_t mndSetAlterStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *
   return 0;
 }
 
-static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, void* alterOriData, int32_t alterOriDataLen) {
   SSdb   *pSdb = pMnode->pSdb;
   SVgObj *pVgroup = NULL;
   void   *pIter = NULL;
@@ -1329,7 +1393,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
       continue;
     }
 
-    void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen);
+    void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen, alterOriData, alterOriDataLen);
     if (pReq == NULL) {
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
@@ -1542,7 +1606,7 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, i
   return 0;
 }
 
-static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp) {
+static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void* alterOriData, int32_t alterOriDataLen) {
   int32_t code = -1;
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq);
   if (pTrans == NULL) goto _OVER;
@@ -1559,7 +1623,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb
 
   if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
-  if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;
@@ -1620,7 +1684,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
   }
 
   if (code != 0) goto _OVER;
-  code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp);
+  code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp, pReq->pCont, pReq->contLen);
 
 _OVER:
   taosMemoryFreeClear(stbObj.pTags);
@@ -1785,8 +1849,8 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
     }
   }
 
-  if (dropReq.source != TD_REQ_FROM_APP && pStb->uid != dropReq.suid) {
-    terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+  if (dropReq.source == TD_REQ_FROM_TAOX && pStb->uid != dropReq.suid) {
+    code = 0;
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndSync.c

@@ -56,20 +56,22 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM
     sdbSetApplyInfo(pMnode->pSdb, cbMeta.index, cbMeta.term, cbMeta.lastConfigIndex);
   }
 
-  if (pMgmt->transId == transId) {
+  if (pMgmt->transId == transId && transId != 0) {
     if (pMgmt->errCode != 0) {
       mError("trans:%d, failed to propose since %s", transId, tstrerror(pMgmt->errCode));
     }
     pMgmt->transId = 0;
     tsem_post(&pMgmt->syncSem);
   } else {
+#if 1
+    mError("trans:%d, invalid commit msg since trandId not match with %d", transId, pMgmt->transId);
+#else
     STrans *pTrans = mndAcquireTrans(pMnode, transId);
     if (pTrans != NULL) {
       mndTransExecute(pMnode, pTrans);
       mndReleaseTrans(pMnode, pTrans);
     }
-#if 0
-    sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA);
+    // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA);
 #endif
   }
 }

source/dnode/mnode/sdb/inc/sdb.h

@@ -163,6 +163,7 @@ typedef struct SSdbRow {
   ESdbType   type;
   ESdbStatus status;
   int32_t    refCount;
+  int64_t    forAlign;
   char       pObj[];
 } SSdbRow;
 

source/dnode/vnode/inc/vnode.h

@@ -137,9 +137,8 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
 void   *tsdbGetIdx(SMeta *pMeta);
 void   *tsdbGetIvtIdx(SMeta *pMeta);
 
-int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t *colId, int32_t numOfCols,
-                              void **pReader);
-int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds);
+int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader);
+int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray* pTableUids);
 int32_t tsdbLastrowReaderClose(void *pReader);
 int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
 

source/dnode/vnode/src/inc/tq.h

@@ -89,8 +89,6 @@ typedef struct {
     STqExecTb  execTb;
     STqExecDb  execDb;
   };
-  // TODO remove it
-  int64_t tsdbEndVer;
 
 } STqExecHandle;
 
@@ -101,6 +99,8 @@ typedef struct {
   int32_t epoch;
   int8_t  fetchMeta;
 
+  int64_t snapshotVer;
+
   // TODO remove
   SWalReader* pWalReader;
 
@@ -131,7 +131,7 @@ typedef struct {
 static STqMgmt tqMgmt = {0};
 
 // tqRead
-int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset);
+int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* offset);
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
 
 // tqExec

source/dnode/vnode/src/meta/metaTable.c

@@ -1125,6 +1125,11 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   skmDbKey.uid = pME->uid;
   skmDbKey.sver = pSW->version;
 
+  // if receive tmq meta message is: create stable1 then delete stable1 then create stable1 with multi vgroups
+  if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), NULL, NULL) == 0) {
+    return rcode;
+  }
+
   // encode schema
   int32_t ret = 0;
   tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen, ret);

source/dnode/vnode/src/tq/tq.c

@@ -244,6 +244,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   STqOffsetVal reqOffset = pReq->reqOffset;
   STqOffsetVal fetchOffsetNew;
 
+  // todo
+  workerId = 0;
+
   // 1.find handle
   STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
   /*ASSERT(pHandle);*/
@@ -284,7 +287,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       fetchOffsetNew = pOffset->val;
       char formatBuf[80];
       tFormatOffset(formatBuf, 80, &fetchOffsetNew);
-      tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, offset reset to %s", consumerId, pHandle->subKey, formatBuf);
+      tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, offset reset to %s", consumerId, pHandle->subKey,
+              TD_VID(pTq->pVnode), formatBuf);
     } else {
       if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
         if (pReq->useSnapshot && pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@@ -299,8 +303,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
         }
       } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
         tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
-        tqDebug("tmq poll: consumer %ld, subkey %s, offset reset to %ld", consumerId, pHandle->subKey,
-                dataRsp.rspOffset.version);
+        tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, offset reset to %ld", consumerId, pHandle->subKey,
+                TD_VID(pTq->pVnode), dataRsp.rspOffset.version);
         if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
           code = -1;
         }
@@ -318,10 +322,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   // 3.query
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
-      fetchOffsetNew.version++;
-    }
-    if (tqScan(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) {
+    /*if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {*/
+    /*fetchOffsetNew.version++;*/
+    /*}*/
+    if (tqScan(pTq, pHandle, &dataRsp, &fetchOffsetNew) < 0) {
       ASSERT(0);
       code = -1;
       goto OVER;
@@ -397,7 +401,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
         tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType);
         SMqMetaRsp metaRsp = {0};
         /*metaRsp.reqOffset = pReq->reqOffset.version;*/
-        /*metaRsp.rspOffset = fetchVer;*/
+        metaRsp.rspOffset = fetchVer;
         /*metaRsp.rspOffsetNew.version = fetchVer;*/
         tqOffsetResetToLog(&metaRsp.reqOffsetNew, pReq->reqOffset.version);
         tqOffsetResetToLog(&metaRsp.rspOffsetNew, fetchVer);
@@ -480,30 +484,28 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
     pHandle->fetchMeta = req.withMeta;
 
     pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
-    /*for (int32_t i = 0; i < 5; i++) {*/
-    /*pHandle->execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/
-    /*}*/
+
+    // TODO version should be assigned in preprocess
     int64_t ver = walGetCommittedVer(pTq->pVnode->pWal);
     if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
       pHandle->execHandle.execCol.qmsg = req.qmsg;
+      pHandle->snapshotVer = ver;
       req.qmsg = NULL;
       for (int32_t i = 0; i < 5; i++) {
         SReadHandle handle = {
-            .tqReader = pHandle->execHandle.pExecReader[i],
             .meta = pTq->pVnode->pMeta,
             .vnode = pTq->pVnode,
             .initTableReader = true,
             .initTqReader = true,
             .version = ver,
         };
-        pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
+        pHandle->execHandle.execCol.task[i] = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
         ASSERT(pHandle->execHandle.execCol.task[i]);
         void* scanner = NULL;
         qExtractStreamScanner(pHandle->execHandle.execCol.task[i], &scanner);
         ASSERT(scanner);
         pHandle->execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner);
         ASSERT(pHandle->execHandle.pExecReader[i]);
-        pHandle->execHandle.tsdbEndVer = ver;
       }
     } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
       for (int32_t i = 0; i < 5; i++) {

source/dnode/vnode/src/tq/tqExec.c

@@ -59,13 +59,13 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
   return 0;
 }
 
-int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
-  qTaskInfo_t task = pExec->execCol.task[0];
+int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
+  const STqExecHandle* pExec = &pHandle->execHandle;
+  qTaskInfo_t          task = pExec->execCol.task[0];
 
   if (qStreamPrepareScan(task, pOffset) < 0) {
     ASSERT(pOffset->type == TMQ_OFFSET__LOG);
     pRsp->rspOffset = *pOffset;
-    pRsp->rspOffset.version--;
     return 0;
   }
 
@@ -73,9 +73,11 @@ int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffset
   while (1) {
     SSDataBlock* pDataBlock = NULL;
     uint64_t     ts = 0;
+    tqDebug("task start to execute");
     if (qExecTask(task, &pDataBlock, &ts) < 0) {
       ASSERT(0);
     }
+    tqDebug("task execute end, get %p", pDataBlock);
 
     if (pDataBlock != NULL) {
       tqAddBlockDataToRsp(pDataBlock, pRsp);
@@ -97,7 +99,7 @@ int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffset
     }
 
     if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-      tqOffsetResetToLog(pOffset, pExec->tsdbEndVer + 1);
+      tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1);
       qStreamPrepareScan(task, pOffset);
       continue;
     }
@@ -116,7 +118,7 @@ int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffset
     if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
       ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version);
     }
-
+    tqDebug("task exec exited");
     break;
   }
 

source/dnode/vnode/src/tq/tqMeta.c

@@ -19,6 +19,7 @@ static int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) {
   if (tStartEncode(pEncoder) < 0) return -1;
   if (tEncodeCStr(pEncoder, pHandle->subKey) < 0) return -1;
   if (tEncodeI64(pEncoder, pHandle->consumerId) < 0) return -1;
+  if (tEncodeI64(pEncoder, pHandle->snapshotVer) < 0) return -1;
   if (tEncodeI32(pEncoder, pHandle->epoch) < 0) return -1;
   if (tEncodeI8(pEncoder, pHandle->execHandle.subType) < 0) return -1;
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@@ -32,6 +33,7 @@ static int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
   if (tStartDecode(pDecoder) < 0) return -1;
   if (tDecodeCStrTo(pDecoder, pHandle->subKey) < 0) return -1;
   if (tDecodeI64(pDecoder, &pHandle->consumerId) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pHandle->snapshotVer) < 0) return -1;
   if (tDecodeI32(pDecoder, &pHandle->epoch) < 0) return -1;
   if (tDecodeI8(pDecoder, &pHandle->execHandle.subType) < 0) return -1;
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@@ -78,19 +80,25 @@ int32_t tqMetaOpen(STQ* pTq) {
     tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
     tDecodeSTqHandle(&decoder, &handle);
     handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
-    for (int32_t i = 0; i < 5; i++) {
-      handle.execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);
-    }
+    /*for (int32_t i = 0; i < 5; i++) {*/
+    /*handle.execHandle.pExecReader[i] = tqOpenReader(pTq->pVnode);*/
+    /*}*/
     if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
       for (int32_t i = 0; i < 5; i++) {
         SReadHandle reader = {
-            .tqReader = handle.execHandle.pExecReader[i],
             .meta = pTq->pVnode->pMeta,
-            .pMsgCb = &pTq->pVnode->msgCb,
             .vnode = pTq->pVnode,
+            .initTableReader = true,
+            .initTqReader = true,
+            .version = handle.snapshotVer,
         };
-        handle.execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(handle.execHandle.execCol.qmsg, &reader);
+        handle.execHandle.execCol.task[i] = qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader);
         ASSERT(handle.execHandle.execCol.task[i]);
+        void* scanner = NULL;
+        qExtractStreamScanner(handle.execHandle.execCol.task[i], &scanner);
+        ASSERT(scanner);
+        handle.execHandle.pExecReader[i] = qExtractReaderFromStreamScanner(scanner);
+        ASSERT(handle.execHandle.pExecReader[i]);
       }
     } else {
       handle.execHandle.execDb.pFilterOutTbUid =

source/dnode/vnode/src/tq/tqRead.c

@@ -319,7 +319,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
   return 0;
 
 FAIL:
-  tDeleteSSDataBlock(pBlock);
+  blockDataFreeRes(pBlock);
   return -1;
 }
 

source/dnode/vnode/src/tq/tqSink.c

@@ -50,6 +50,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
 
       SVCreateTbReq createTbReq = {0};
       createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
+      createTbReq.ctb.name = strdup(stbFullName);
       createTbReq.flags = 0;
       createTbReq.type = TSDB_CHILD_TABLE;
       createTbReq.ctb.suid = suid;

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -22,34 +22,35 @@ typedef struct SLastrowReader {
   SVnode*   pVnode;
   STSchema* pSchema;
   uint64_t  uid;
-  //  int32_t*  pSlotIds;
-  char**  transferBuf;  // todo remove it soon
-  int32_t numOfCols;
-  int32_t type;
-  int32_t tableIndex;  // currently returned result tables
-  SArray* pTableList;  // table id list
+  char**    transferBuf;  // todo remove it soon
+  int32_t   numOfCols;
+  int32_t   type;
+  int32_t   tableIndex;  // currently returned result tables
+  SArray*   pTableList;  // table id list
 } SLastrowReader;
 
 static void saveOneRow(STSRow* pRow, SSDataBlock* pBlock, SLastrowReader* pReader, const int32_t* slotIds) {
+  ASSERT(pReader->numOfCols <= taosArrayGetSize(pBlock->pDataBlock));
   int32_t numOfRows = pBlock->info.rows;
-  size_t  numOfCols = taosArrayGetSize(pBlock->pDataBlock);
 
   SColVal colVal = {0};
-  for (int32_t i = 0; i < numOfCols; ++i) {
+  for (int32_t i = 0; i < pReader->numOfCols; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
 
     if (slotIds[i] == -1) {
       colDataAppend(pColInfoData, numOfRows, (const char*)&pRow->ts, false);
     } else {
-      tTSRowGetVal(pRow, pReader->pSchema, slotIds[i], &colVal);
+      int32_t slotId = slotIds[i];
+
+      tTSRowGetVal(pRow, pReader->pSchema, slotId, &colVal);
 
       if (IS_VAR_DATA_TYPE(colVal.type)) {
         if (colVal.isNull || colVal.isNone) {
           colDataAppendNULL(pColInfoData, numOfRows);
         } else {
-          varDataSetLen(pReader->transferBuf[i], colVal.value.nData);
-          memcpy(varDataVal(pReader->transferBuf[i]), colVal.value.pData, colVal.value.nData);
-          colDataAppend(pColInfoData, numOfRows, pReader->transferBuf[i], false);
+          varDataSetLen(pReader->transferBuf[slotId], colVal.value.nData);
+          memcpy(varDataVal(pReader->transferBuf[slotId]), colVal.value.pData, colVal.value.nData);
+          colDataAppend(pColInfoData, numOfRows, pReader->transferBuf[slotId], false);
         }
       } else {
         colDataAppend(pColInfoData, numOfRows, (const char*)&colVal.value, colVal.isNull || colVal.isNone);
@@ -60,8 +61,7 @@ static void saveOneRow(STSRow* pRow, SSDataBlock* pBlock, SLastrowReader* pReade
   pBlock->info.rows += 1;
 }
 
-int32_t tsdbLastRowReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList, int32_t* colId, int32_t numOfCols,
-                              void** pReader) {
+int32_t tsdbLastRowReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList, int32_t numOfCols, void** pReader) {
   SLastrowReader* p = taosMemoryCalloc(1, sizeof(SLastrowReader));
   if (p == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -70,13 +70,18 @@ int32_t tsdbLastRowReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList,
   p->type = type;
   p->pVnode = pVnode;
   p->numOfCols = numOfCols;
-  p->transferBuf = taosMemoryCalloc(p->numOfCols, POINTER_BYTES);
+
+  if (taosArrayGetSize(pTableIdList) == 0) {
+    *pReader = p;
+    return TSDB_CODE_SUCCESS;
+  }
 
   STableKeyInfo* pKeyInfo = taosArrayGet(pTableIdList, 0);
   p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, pKeyInfo->uid, -1);
   p->pTableList = pTableIdList;
 
-  for (int32_t i = 0; i < p->numOfCols; ++i) {
+  p->transferBuf = taosMemoryCalloc(p->pSchema->numOfCols, POINTER_BYTES);
+  for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) {
     if (IS_VAR_DATA_TYPE(p->pSchema->columns[i].type)) {
       p->transferBuf[i] = taosMemoryMalloc(p->pSchema->columns[i].bytes);
     }
@@ -89,16 +94,20 @@ int32_t tsdbLastRowReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList,
 int32_t tsdbLastrowReaderClose(void* pReader) {
   SLastrowReader* p = pReader;
 
-  for (int32_t i = 0; i < p->numOfCols; ++i) {
-    taosMemoryFreeClear(p->transferBuf[i]);
+  if (p->pSchema != NULL) {
+    for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) {
+      taosMemoryFreeClear(p->transferBuf[i]);
+    }
+
+    taosMemoryFree(p->transferBuf);
+    taosMemoryFree(p->pSchema);
   }
 
-  taosMemoryFree(p->transferBuf);
   taosMemoryFree(pReader);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t* slotIds) {
+int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t* slotIds, SArray* pTableUidList) {
   if (pReader == NULL || pResBlock == NULL) {
     return TSDB_CODE_INVALID_PARA;
   }
@@ -135,14 +144,15 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t
         // appended or not.
         if (internalResult) {
           pResBlock->info.rows -= 1;
+          taosArrayClear(pTableUidList);
         }
 
         saveOneRow(pRow, pResBlock, pr, slotIds);
+        taosArrayPush(pTableUidList, &pKeyInfo->uid);
         internalResult = true;
         lastKey = pRow->ts;
       }
 
-      // taosMemoryFree(pRow);
       tsdbCacheRelease(lruCache, h);
     }
   } else if (pr->type == LASTROW_RETRIEVE_TYPE_ALL) {
@@ -165,6 +175,7 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t
       // tsdbCacheLastArray2Row(pLast, &pRow, pr->pSchema);
 
       saveOneRow(pRow, pResBlock, pr, slotIds);
+      taosArrayPush(pTableUidList, &pKeyInfo->uid);
 
       // taosMemoryFree(pRow);
       tsdbCacheRelease(lruCache, h);

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -427,13 +427,22 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c
          syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state,
          syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType));
 
-  SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen};
-  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
-  memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
-  syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info);
-  rpcMsg.info.conn.applyIndex = cbMeta.index;
-  rpcMsg.info.conn.applyTerm = cbMeta.term;
-  tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
+  if (cbMeta.code == 0) {
+    SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen};
+    rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+    memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
+    syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info);
+    rpcMsg.info.conn.applyIndex = cbMeta.index;
+    rpcMsg.info.conn.applyTerm = cbMeta.term;
+    tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
+  } else {
+    SRpcMsg rsp = {.code = cbMeta.code, .info = pMsg->info};
+    vError("vgId:%d, sync commit error, msgtype:%d,%s, error:0x%X, errmsg:%s", syncGetVgId(pVnode->sync), pMsg->msgType,
+           TMSG_INFO(pMsg->msgType), cbMeta.code, tstrerror(cbMeta.code));
+    if (rsp.info.handle != NULL) {
+      tmsgSendRsp(&rsp);
+    }
+  }
 }
 
 static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {

source/libs/catalog/inc/catalogInt.h

@@ -642,6 +642,7 @@ void    ctgFreeSTableIndex(void *info);
 void    ctgClearSubTaskRes(SCtgSubRes *pRes);
 void    ctgFreeQNode(SCtgQNode *node);
 void    ctgClearHandle(SCatalog* pCtg);
+void    ctgFreeTbCacheImpl(SCtgTbCache *pCache);
 
 
 extern SCatalogMgmt gCtgMgmt;