Merge remote-tracking branch 'origin/3.0' into feature/qnode

example/src/tmq.c

@@ -163,12 +163,13 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
     printf("subscribe err\n");
     return;
   }
-  /*int32_t cnt = 0;*/
+  int32_t cnt = 0;
   /*clock_t startTime = clock();*/
   while (running) {
     tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
     if (tmqmessage) {
-      /*cnt++;*/
+      cnt++;
+      printf("get data\n");
       msg_process(tmqmessage);
       tmq_message_destroy(tmqmessage);
       /*} else {*/

include/common/taosdef.h

@@ -57,6 +57,8 @@ typedef enum {
   TD_ROW_PARTIAL_UPDATE = 2,
 } TDUpdateConfig;
 
+#define TD_SUPPORT_UPDATE(u) ((u) > 0)
+
 typedef enum {
   TSDB_STATIS_OK = 0,    // statis part exist and load successfully
   TSDB_STATIS_NONE = 1,  // statis part not exist

include/common/tcommon.h

@@ -222,7 +222,7 @@ typedef struct SFunctParam {
 // the structure for sql function in select clause
 typedef struct SResSchame {
   int8_t  type;
-  int32_t colId;
+  int32_t slotId;
   int32_t bytes;
   int32_t precision;
   int32_t scale;

include/common/tdatablock.h

@@ -136,7 +136,8 @@ static FORCE_INLINE void colDataAppendInt8(SColumnInfoData* pColumnInfoData, uin
 }
 
 static FORCE_INLINE void colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) {
-  ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT);
+  ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT ||
+         pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT);
   char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
   *(int16_t*)p = *(int16_t*)v;
 }
@@ -210,15 +211,19 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock);
 
 void blockDebugShowData(const SArray* dataBlocks);
 
+static FORCE_INLINE int32_t blockEstimateEncodeSize(const SSDataBlock* pBlock) {
+  return blockDataGetSerialMetaSize(pBlock) + (int32_t)ceil(blockDataGetSerialRowSize(pBlock) * pBlock->info.rows);
+}
+
 static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,
-                                            int8_t compressed) {
+                                                 int8_t compressed) {
   int32_t colSize = colDataGetLength(pColRes, numOfRows);
   return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data,
                                                       colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0);
 }
 
 static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
-                                       int8_t needCompress) {
+                                             int8_t needCompress) {
   int32_t* colSizes = (int32_t*)data;
 
   data += numOfCols * sizeof(int32_t);

include/common/tdataformat.h

@@ -482,7 +482,7 @@ void       tdResetDataCols(SDataCols *pCols);
 int32_t    tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 SDataCols *tdFreeDataCols(SDataCols *pCols);
-int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull);
+int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer);
 
 // ----------------- K-V data row structure
 /* |<-------------------------------------- len -------------------------------------------->|

include/common/tglobal.h

@@ -18,6 +18,7 @@
 
 #include "tarray.h"
 #include "tdef.h"
+#include "tconfig.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -129,6 +130,7 @@ void    taosCfgDynamicOptions(const char *option, const char *value);
 void    taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary);
 
 struct SConfig *taosGetCfg();
+int32_t taosAddClientLogCfg(SConfig *pCfg);
 
 #ifdef __cplusplus
 }

include/common/tmsg.h

@@ -260,6 +260,7 @@ typedef struct {
 
 typedef struct SSchema {
   int8_t   type;
+  int8_t   index;  // default is 0, not index created
   col_id_t colId;
   int32_t  bytes;
   char     name[TSDB_COL_NAME_LEN];

include/common/trow.h

@@ -230,7 +230,7 @@ static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowVa
 static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
                                                   bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset,
                                                   col_id_t colId);
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool forceSetNull);
+int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols);
 
 /**
  * @brief

include/common/ttypes.h

@@ -27,6 +27,7 @@ extern "C" {
 typedef int32_t  VarDataOffsetT;
 typedef uint32_t TDRowLenT;
 typedef uint8_t  TDRowValT;
+typedef uint64_t TDRowVerT;
 typedef int16_t  col_id_t;
 typedef int8_t   col_type_t;
 typedef int32_t  col_bytes_t;
@@ -141,6 +142,43 @@ typedef struct {
     }                                          \
   } while (0)
 
+#define NUM_TO_STRING(_inputType, _input, _outputBytes, _output)     \
+  do {                                         \
+    switch (_inputType) {                      \
+      case TSDB_DATA_TYPE_TINYINT:             \
+        snprintf(_output, (int32_t)(_outputBytes), "%d", *(int8_t *)(_input));     \
+        break;                                 \
+      case TSDB_DATA_TYPE_UTINYINT:            \
+        snprintf(_output, (int32_t)(_outputBytes), "%d", *(uint8_t *)(_input));     \
+        break;                                 \
+      case TSDB_DATA_TYPE_SMALLINT:            \
+        snprintf(_output, (int32_t)(_outputBytes), "%d", *(int16_t *)(_input));     \
+        break;                                 \
+      case TSDB_DATA_TYPE_USMALLINT:           \
+        snprintf(_output, (int32_t)(_outputBytes), "%d", *(uint16_t *)(_input));     \
+        break;                                 \
+      case TSDB_DATA_TYPE_TIMESTAMP:           \
+      case TSDB_DATA_TYPE_BIGINT:              \
+        snprintf(_output, (int32_t)(_outputBytes), "%" PRId64, *(int64_t *)(_input));     \
+        break;                                 \
+      case TSDB_DATA_TYPE_UBIGINT:             \
+        snprintf(_output, (int32_t)(_outputBytes), "%" PRIu64, *(uint64_t *)(_input));     \
+        break;                                 \
+      case TSDB_DATA_TYPE_FLOAT:               \
+        snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input));     \
+        break;                                 \
+      case TSDB_DATA_TYPE_DOUBLE:              \
+        snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input));     \
+        break;                                 \
+      case TSDB_DATA_TYPE_UINT:                \
+        snprintf(_output, (int32_t)(_outputBytes), "%u", *(uint32_t *)(_input));     \
+        break;                                 \
+      default:                                 \
+        snprintf(_output, (int32_t)(_outputBytes), "%d", *(int32_t *)(_input));     \
+        break;                                 \
+    }                                          \
+  } while (0)
+
 #define IS_SIGNED_NUMERIC_TYPE(_t)   ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT)
 #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT)
 #define IS_FLOAT_TYPE(_t)            ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)

include/libs/function/functionMgt.h

@@ -135,8 +135,17 @@ bool fmIsTimeorderFunc(int32_t funcId);
 bool fmIsPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
+bool fmIsSpecialDataRequiredFunc(int32_t funcId);
+bool fmIsDynamicScanOptimizedFunc(int32_t funcId);
 
-int32_t fmFuncScanType(int32_t funcId);
+typedef enum EFuncDataRequired {
+  FUNC_DATA_REQUIRED_ALL_NEEDED = 1,
+  FUNC_DATA_REQUIRED_STATIS_NEEDED,
+  FUNC_DATA_REQUIRED_NO_NEEDED,
+  FUNC_DATA_REQUIRED_DISCARD
+} EFuncDataRequired;
+
+EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);

include/libs/monitor/monitor.h

@@ -78,6 +78,9 @@ typedef struct {
 typedef struct {
   float        uptime;  // day
   int8_t       has_mnode;
+  int8_t       has_qnode;
+  int8_t       has_snode;
+  int8_t       has_bnode;
   SMonDiskDesc logdir;
   SMonDiskDesc tempdir;
 } SMonDnodeInfo;
@@ -134,8 +137,8 @@ typedef struct {
 
 typedef struct {
   int32_t expire_time;
-  int32_t timeseries_used;
-  int32_t timeseries_total;
+  int64_t timeseries_used;
+  int64_t timeseries_total;
 } SMonGrantInfo;
 
 typedef struct {

include/libs/nodes/nodes.h

@@ -216,6 +216,7 @@ SNodeList* nodesMakeList();
 int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
 int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
 int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode);
+int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode);
 int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
 int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
 int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode);

include/libs/nodes/plannodes.h

@@ -30,6 +30,7 @@ typedef struct SLogicNode {
   SNode* pConditions;
   SNodeList* pChildren;
   struct SLogicNode* pParent;
+  int32_t optimizedFlag;
 } SLogicNode;
 
 typedef enum EScanType {
@@ -50,6 +51,8 @@ typedef struct SScanLogicNode {
   SName tableName;
   bool showRewrite;
   double ratio;
+  SNodeList* pDynamicScanFuncs;
+  int32_t dataRequired;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -84,6 +87,7 @@ typedef struct SVnodeModifLogicNode {
 typedef struct SExchangeLogicNode {
   SLogicNode node;
   int32_t srcGroupId;
+  uint8_t precision;
 } SExchangeLogicNode;
 
 typedef enum EWindowType {
@@ -163,7 +167,7 @@ typedef struct SDataBlockDescNode {
   SNodeList* pSlots;
   int32_t totalRowSize;
   int32_t outputRowSize;
-  int16_t precision;
+  uint8_t precision;
 } SDataBlockDescNode;
 
 typedef struct SPhysiNode {
@@ -195,20 +199,13 @@ typedef struct SSystemTableScanPhysiNode {
   int32_t accountId;
 } SSystemTableScanPhysiNode;
 
-typedef enum EScanRequired {
-  SCAN_REQUIRED_DATA_NO_NEEDED = 1,
-  SCAN_REQUIRED_DATA_STATIS_NEEDED,
-  SCAN_REQUIRED_DATA_ALL_NEEDED,
-  SCAN_REQUIRED_DATA_DISCARD,
-} EScanRequired;
-
 typedef struct STableScanPhysiNode {
   SScanPhysiNode scan;
   uint8_t scanFlag;         // denotes reversed scan of data or not
   STimeWindow scanRange;
   double ratio;
-  EScanRequired scanRequired;
-  SNodeList* pScanReferFuncs;
+  int32_t dataRequired;
+  SNodeList* pDynamicScanFuncs;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;
@@ -253,11 +250,11 @@ typedef struct SWinodwPhysiNode {
   SPhysiNode node;
   SNodeList* pExprs;   // these are expression list of parameter expression of function
   SNodeList* pFuncs;
+  SNode* pTspk; // timestamp primary key
 } SWinodwPhysiNode;
 
 typedef struct SIntervalPhysiNode {
   SWinodwPhysiNode window;
-  SNode* pTspk; // timestamp primary key
   int64_t    interval;
   int64_t    offset;
   int64_t    sliding;
@@ -274,7 +271,7 @@ typedef struct SMultiTableIntervalPhysiNode {
 
 typedef struct SSessionWinodwPhysiNode {
   SWinodwPhysiNode window;
-  int64_t    gap;
+  int64_t gap;
 } SSessionWinodwPhysiNode;
 
 typedef struct SStateWinodwPhysiNode {

include/libs/nodes/querynodes.h

@@ -190,6 +190,7 @@ typedef struct SLimitNode {
 
 typedef struct SStateWindowNode {
   ENodeType type; // QUERY_NODE_STATE_WINDOW
+  SNode* pCol; // timestamp primary key
   SNode* pExpr;
 } SStateWindowNode;
 

include/libs/scalar/filter.h

@@ -19,8 +19,8 @@
 extern "C" {
 #endif
 
-#include "tcommon.h"
 #include "nodes.h"
+#include "tcommon.h"
 
 typedef struct SFilterInfo SFilterInfo;
 typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
@@ -31,20 +31,20 @@ enum {
   FLT_OPTION_NEED_UNIQE = 4,
 };
 
-typedef struct SFilterColumnParam{
+typedef struct SFilterColumnParam {
   int32_t numOfCols;
-  SArray* pDataBlock;
+  SArray *pDataBlock;
 } SFilterColumnParam;
 
 extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
-extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
+extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t **p, SColumnDataAgg *statis, int16_t numOfCols);
 extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);
 extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param);
 extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict);
-extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
-extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
-extern void filterFreeInfo(SFilterInfo *info);
-extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
+extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar);
+extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo);
+extern void    filterFreeInfo(SFilterInfo *info);
+extern bool    filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
 
 #ifdef __cplusplus
 }

include/libs/scalar/scalar.h

@@ -70,6 +70,9 @@ int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
 int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
+/* Conversion functions */
+int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
 bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 
 int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

include/os/osFile.h

@@ -44,7 +44,7 @@ extern "C" {
 
 typedef struct TdFile *TdFilePtr;
  
-#define TD_FILE_CTEATE    0x0001
+#define TD_FILE_CREATE    0x0001
 #define TD_FILE_WRITE     0x0002
 #define TD_FILE_READ      0x0004
 #define TD_FILE_TRUNC     0x0008

include/os/osSysinfo.h

@@ -39,7 +39,7 @@ int32_t taosGetEmail(char *email, int32_t maxLen);
 int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
 int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
 int32_t taosGetCpuCores(float *numOfCores);
-int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine);
+void    taosGetCpuUsage(double *cpu_system, double *cpu_engine);
 int32_t taosGetTotalMemory(int64_t *totalKB);
 int32_t taosGetProcMemory(int64_t *usedKB);
 int32_t taosGetSysMemory(int64_t *usedKB);

include/util/taoserror.h

@@ -597,6 +597,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_ROLLUP_OPTION     TAOS_DEF_ERROR_CODE(0, 0x2622)
 #define TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION TAOS_DEF_ERROR_CODE(0, 0x2623)
 #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST    TAOS_DEF_ERROR_CODE(0, 0x2624)
+#define TSDB_CODE_PAR_INVALID_OPTION_UNIT       TAOS_DEF_ERROR_CODE(0, 0x2625)
+#define TSDB_CODE_PAR_INVALID_KEEP_UNIT         TAOS_DEF_ERROR_CODE(0, 0x2626)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR              TAOS_DEF_ERROR_CODE(0, 0x2700)

source/client/inc/clientInt.h

@@ -76,12 +76,12 @@ typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq
 typedef struct {
   int8_t inited;
   // ctl
-  int8_t          threadStop;
-  TdThread       thread;
+  int8_t        threadStop;
+  TdThread      thread;
   TdThreadMutex lock;       // used when app init and cleanup
-  SArray*         appHbMgrs;  // SArray<SAppHbMgr*> one for each cluster
-  FHbReqHandle    reqHandle[HEARTBEAT_TYPE_MAX];
-  FHbRspHandle    rspHandle[HEARTBEAT_TYPE_MAX];
+  SArray*       appHbMgrs;  // SArray<SAppHbMgr*> one for each cluster
+  FHbReqHandle  reqHandle[HEARTBEAT_TYPE_MAX];
+  FHbRspHandle  rspHandle[HEARTBEAT_TYPE_MAX];
 } SClientHbMgr;
 
 typedef struct SQueryExecMetric {
@@ -119,43 +119,43 @@ struct SAppInstInfo {
 };
 
 typedef struct SAppInfo {
-  int64_t         startTime;
-  char            appName[TSDB_APP_NAME_LEN];
-  char*           ep;
-  int32_t         pid;
-  int32_t         numOfThreads;
-  SHashObj*       pInstMap;
+  int64_t       startTime;
+  char          appName[TSDB_APP_NAME_LEN];
+  char*         ep;
+  int32_t       pid;
+  int32_t       numOfThreads;
+  SHashObj*     pInstMap;
   TdThreadMutex mutex;
 } SAppInfo;
 
 typedef struct STscObj {
-  char            user[TSDB_USER_LEN];
-  char            pass[TSDB_PASSWORD_LEN];
-  char            db[TSDB_DB_FNAME_LEN];
-  char            ver[128];
-  int32_t         acctId;
-  uint32_t        connId;
-  int32_t         connType;
-  uint64_t        id;         // ref ID returned by taosAddRef
-  TdThreadMutex   mutex;      // used to protect the operation on db
-  int32_t         numOfReqs;  // number of sqlObj bound to this connection
-  SAppInstInfo*   pAppInfo;
-  SHashObj*       pRequests;
+  char          user[TSDB_USER_LEN];
+  char          pass[TSDB_PASSWORD_LEN];
+  char          db[TSDB_DB_FNAME_LEN];
+  char          ver[128];
+  int32_t       acctId;
+  uint32_t      connId;
+  int32_t       connType;
+  uint64_t      id;         // ref ID returned by taosAddRef
+  TdThreadMutex mutex;      // used to protect the operation on db
+  int32_t       numOfReqs;  // number of sqlObj bound to this connection
+  SAppInstInfo* pAppInfo;
+  SHashObj*     pRequests;
 } STscObj;
 
 typedef struct SResultColumn {
   union {
-    char*       nullbitmap;  // bitmap, one bit for each item in the list
-    int32_t*    offset;
+    char*    nullbitmap;  // bitmap, one bit for each item in the list
+    int32_t* offset;
   };
-  char*         pData;
+  char* pData;
 } SResultColumn;
 
 typedef struct SReqResultInfo {
   const char*    pRspMsg;
   const char*    pData;
-  TAOS_FIELD*    fields;     // todo, column names are not needed.
-  TAOS_FIELD*    userFields; // the fields info that return to user
+  TAOS_FIELD*    fields;      // todo, column names are not needed.
+  TAOS_FIELD*    userFields;  // the fields info that return to user
   uint32_t       numOfCols;
   int32_t*       length;
   char**         convertBuf;
@@ -182,13 +182,30 @@ typedef struct SRequestSendRecvBody {
   SShowReqInfo       showInfo;  // todo this attribute will be removed after the query framework being completed.
   SDataBuf           requestMsg;
   int64_t            queryJob;  // query job, created according to sql query DAG.
-  struct SQueryPlan* pDag;       // the query dag, generated according to the sql statement.
+  struct SQueryPlan* pDag;      // the query dag, generated according to the sql statement.
   SReqResultInfo     resInfo;
 } SRequestSendRecvBody;
 
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
 
+enum {
+  RES_TYPE__QUERY = 1,
+  RES_TYPE__TMQ,
+};
+
+#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY)
+#define TD_RES_TMQ(res)   (*(int8_t*)res == RES_TYPE__TMQ)
+
+typedef struct SMqRspObj {
+  int8_t  resType;
+  char*   topic;
+  void*   vg;
+  SArray* res;  // SArray<SReqResultInfo>
+  int32_t resIter;
+} SMqRspObj;
+
 typedef struct SRequestObj {
+  int8_t               resType;  // query or tmq
   uint64_t             requestId;
   int32_t              type;  // request type
   STscObj*             pTscObj;
@@ -205,6 +222,25 @@ typedef struct SRequestObj {
   SRequestSendRecvBody body;
 } SRequestObj;
 
+static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
+  SMqRspObj* msg = (SMqRspObj*)res;
+  int32_t    resIter = msg->resIter == -1 ? 0 : msg->resIter;
+  return (SReqResultInfo*)taosArrayGet(msg->res, resIter);
+}
+
+static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res) {
+  SMqRspObj* msg = (SMqRspObj*)res;
+  if (++msg->resIter < taosArrayGetSize(msg->res)) {
+    return (SReqResultInfo*)taosArrayGet(msg->res, msg->resIter);
+  }
+  return NULL;
+}
+
+static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) {
+  if (TD_RES_QUERY(res)) return &(((SRequestObj*)res)->body.resInfo);
+  return tmqGetCurResInfo(res);
+}
+
 extern SAppInfo appInfo;
 extern int32_t  clientReqRefPool;
 extern int32_t  clientConnRefPool;
@@ -244,14 +280,17 @@ void initMsgHandleFp();
 TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
                             uint16_t port);
 
-void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr);
-
-int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
+int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery);
+int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
-int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery);
-int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
+void*   doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
+void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
+int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows,
+                         bool convertUcs4);
+void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
 
 // --- heartbeat
 // global, called by mgmt

source/client/src/clientEnv.c

@@ -177,6 +177,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
     return NULL;
   }
 
+  pRequest->resType = RES_TYPE__QUERY;
   pRequest->pDb = getDbOfConnection(pObj);
   pRequest->requestId = generateRequestId();
   pRequest->metric.start = taosGetTimestampUs();

source/client/src/clientImpl.c

@@ -13,7 +13,6 @@
 static int32_t       initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
 static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
 static void          destroySendMsgInfo(SMsgSendInfo* pMsgBody);
-static int32_t       setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
 
 static bool stringLengthCheck(const char* str, size_t maxsize) {
   if (str == NULL) {
@@ -42,7 +41,6 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
 
 static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
                                 SAppInstInfo* pAppInfo);
-static void     setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
 
 TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
                             uint16_t port) {
@@ -174,9 +172,9 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
 
 int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
   SRetrieveTableRsp* pRsp = NULL;
-  int32_t code = qExecCommand(pQuery->pRoot, &pRsp);
+  int32_t            code = qExecCommand(pQuery->pRoot, &pRsp);
   if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
-    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp);
+    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false);
   }
   return code;
 }
@@ -190,7 +188,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
   SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg;
   pRequest->type = pMsgInfo->msgType;
   pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL};
-  pMsgInfo->pMsg = NULL; // pMsg transferred to SMsgSendInfo management
+  pMsgInfo->pMsg = NULL;  // pMsg transferred to SMsgSendInfo management
 
   STscObj*      pTscObj = pRequest->pTscObj;
   SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest);
@@ -211,14 +209,12 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
 
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
   pRequest->type = pQuery->msgType;
-  SPlanContext cxt = {
-    .queryId = pRequest->requestId,
-    .acctId = pRequest->pTscObj->acctId,
-    .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
-    .pAstRoot = pQuery->pRoot,
-    .showRewrite = pQuery->showRewrite
-  };
-  int32_t  code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
+  SPlanContext cxt = {.queryId = pRequest->requestId,
+                      .acctId = pRequest->pTscObj->acctId,
+                      .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
+                      .pAstRoot = pQuery->pRoot,
+                      .showRewrite = pQuery->showRewrite};
+  int32_t      code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
   if (code != 0) {
     return code;
   }
@@ -234,10 +230,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
 
   for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
     pResInfo->fields[i].bytes = pSchema[i].bytes;
-    pResInfo->fields[i].type  = pSchema[i].type;
+    pResInfo->fields[i].type = pSchema[i].type;
 
     pResInfo->userFields[i].bytes = pSchema[i].bytes;
-    pResInfo->userFields[i].type  = pSchema[i].type;
+    pResInfo->userFields[i].type = pSchema[i].type;
 
     if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) {
       pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
@@ -254,7 +250,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
   void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;
 
   SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
-  int32_t      code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, &res);
+  int32_t      code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr,
+                                       pRequest->metric.start, &res);
   if (code != TSDB_CODE_SUCCESS) {
     if (pRequest->body.queryJob != 0) {
       schedulerFreeJob(pRequest->body.queryJob);
@@ -274,14 +271,14 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
   }
 
   pRequest->code = res.code;
-  terrno = res.code;  
+  terrno = res.code;
   return pRequest->code;
 }
 
 SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
   SRequestObj* pRequest = NULL;
-  SQuery* pQuery = NULL;
-  SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
+  SQuery*      pQuery = NULL;
+  SArray*      pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
 
   int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
   if (TSDB_CODE_SUCCESS == code) {
@@ -320,15 +317,15 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
 }
 
 int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
-  SCatalog *pCatalog = NULL;
-  int32_t code = 0;
-  int32_t dbNum = taosArrayGetSize(pRequest->dbList);
-  int32_t tblNum = taosArrayGetSize(pRequest->tableList);
+  SCatalog* pCatalog = NULL;
+  int32_t   code = 0;
+  int32_t   dbNum = taosArrayGetSize(pRequest->dbList);
+  int32_t   tblNum = taosArrayGetSize(pRequest->tableList);
 
   if (dbNum <= 0 && tblNum <= 0) {
     return TSDB_CODE_QRY_APP_ERROR;
   }
-  
+
   code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
@@ -337,8 +334,8 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
   SEpSet epset = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
 
   for (int32_t i = 0; i < dbNum; ++i) {
-    char *dbFName = taosArrayGet(pRequest->dbList, i);
-    
+    char* dbFName = taosArrayGet(pRequest->dbList, i);
+
     code = catalogRefreshDBVgInfo(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, dbFName);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -346,7 +343,7 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
   }
 
   for (int32_t i = 0; i < tblNum; ++i) {
-    SName *tableName = taosArrayGet(pRequest->tableList, i);
+    SName* tableName = taosArrayGet(pRequest->tableList, i);
 
     code = catalogRefreshTableMeta(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, tableName, -1);
     if (code != TSDB_CODE_SUCCESS) {
@@ -357,11 +354,10 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
   return code;
 }
 
-
 SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
   SRequestObj* pRequest = NULL;
-  int32_t retryNum = 0;
-  int32_t code = 0;
+  int32_t      retryNum = 0;
+  int32_t      code = 0;
 
   while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
     pRequest = execQueryImpl(pTscObj, sql, sqlLen);
@@ -377,7 +373,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
 
     destroyRequest(pRequest);
   }
-  
+
   return pRequest;
 }
 
@@ -509,7 +505,8 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
 }
 
 bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) {
-  return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || msgType == TDMT_VND_QUERY_HEARTBEAT_RSP;
+  return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP ||
+         msgType == TDMT_VND_QUERY_HEARTBEAT_RSP;
 }
 
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
@@ -536,10 +533,10 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
     int32_t elapsed = pRequest->metric.rsp - pRequest->metric.start;
     if (pMsg->code == TSDB_CODE_SUCCESS) {
       tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%d ms, reqId:0x%" PRIx64, pRequest->self,
-               TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed/1000, pRequest->requestId);
+               TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed / 1000, pRequest->requestId);
     } else {
       tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%d ms, reqId:0x%" PRIx64, pRequest->self,
-               TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed/1000, pRequest->requestId);
+               TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed / 1000, pRequest->requestId);
     }
 
     taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId);
@@ -590,7 +587,7 @@ TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, c
   return taos_connect(ipStr, userStr, passStr, dbStr, port);
 }
 
-static void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
+void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
   for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
     SResultColumn* pCol = &pResultInfo->pCol[i];
 
@@ -616,7 +613,7 @@ static void doSetOneRowPtr(SReqResultInfo* pResultInfo) {
   }
 }
 
-void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr) {
+void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
   assert(pRequest != NULL);
   SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
 
@@ -637,7 +634,7 @@ void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr) {
         return NULL;
       }
 
-      pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData);
+      pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData, convertUcs4);
       if (pRequest->code != TSDB_CODE_SUCCESS) {
         pResultInfo->numOfRows = 0;
         return NULL;
@@ -722,8 +719,8 @@ _return:
 
 static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
   if (pResInfo->row == NULL) {
-    pResInfo->row    = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES);
-    pResInfo->pCol   = taosMemoryCalloc(pResInfo->numOfCols, sizeof(SResultColumn));
+    pResInfo->row = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES);
+    pResInfo->pCol = taosMemoryCalloc(pResInfo->numOfCols, sizeof(SResultColumn));
     pResInfo->length = taosMemoryCalloc(pResInfo->numOfCols, sizeof(int32_t));
     pResInfo->convertBuf = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES);
 
@@ -735,7 +732,43 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) {
+static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    int32_t type = pResultInfo->fields[i].type;
+    int32_t bytes = pResultInfo->fields[i].bytes;
+
+    if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) {
+      char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
+      if (p == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+
+      pResultInfo->convertBuf[i] = p;
+
+      SResultColumn* pCol = &pResultInfo->pCol[i];
+      for (int32_t j = 0; j < numOfRows; ++j) {
+        if (pCol->offset[j] != -1) {
+          char* pStart = pCol->offset[j] + pCol->pData;
+
+          int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p));
+          ASSERT(len <= bytes);
+
+          varDataSetLen(p, len);
+          pCol->offset[j] = (p - pResultInfo->convertBuf[i]);
+          p += (len + VARSTR_HEADER_SIZE);
+        }
+      }
+
+      pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
+      pResultInfo->row[i] = pResultInfo->pCol[i].pData;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows,
+                         bool convertUcs4) {
   assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL);
   if (numOfRows == 0) {
     return TSDB_CODE_SUCCESS;
@@ -767,37 +800,11 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
   }
 
   // convert UCS4-LE encoded character to native multi-bytes character in current data block.
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    int32_t type = pResultInfo->fields[i].type;
-    int32_t bytes = pResultInfo->fields[i].bytes;
-
-    if (type == TSDB_DATA_TYPE_NCHAR) {
-      char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
-      if (p == NULL) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-      pResultInfo->convertBuf[i] = p;
-
-      SResultColumn* pCol = &pResultInfo->pCol[i];
-      for (int32_t j = 0; j < numOfRows; ++j) {
-        if (pCol->offset[j] != -1) {
-          pStart = pCol->offset[j] + pCol->pData;
-
-          int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p));
-          ASSERT(len <= bytes);
-
-          varDataSetLen(p, len);
-          pCol->offset[j] = (p - pResultInfo->convertBuf[i]);
-          p += (len + VARSTR_HEADER_SIZE);
-        }
-      }
-
-      pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
-      pResultInfo->row[i] = pResultInfo->pCol[i].pData;
-    }
+  if (convertUcs4) {
+    code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength);
   }
 
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 char* getDbOfConnection(STscObj* pObj) {
@@ -829,18 +836,19 @@ void resetConnectDB(STscObj* pTscObj) {
   taosThreadMutexUnlock(&pTscObj->mutex);
 }
 
-int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) {
   assert(pResultInfo != NULL && pRsp != NULL);
 
-  pResultInfo->pRspMsg    = (const char*)pRsp;
-  pResultInfo->pData      = (void*)pRsp->data;
-  pResultInfo->numOfRows  = htonl(pRsp->numOfRows);
-  pResultInfo->current    = 0;
-  pResultInfo->completed  = (pRsp->completed == 1);
+  pResultInfo->pRspMsg = (const char*)pRsp;
+  pResultInfo->pData = (void*)pRsp->data;
+  pResultInfo->numOfRows = htonl(pRsp->numOfRows);
+  pResultInfo->current = 0;
+  pResultInfo->completed = (pRsp->completed == 1);
   pResultInfo->payloadLen = htonl(pRsp->compLen);
-  pResultInfo->precision  = pRsp->precision;
+  pResultInfo->precision = pRsp->precision;
 
   // TODO handle the compressed case
   pResultInfo->totalRows += pResultInfo->numOfRows;
-  return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows);
+  return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows,
+                          convertUcs4);
 }

source/client/src/clientMain.c

@@ -73,7 +73,7 @@ void taos_cleanup(void) {
   tscInfo("all local resources released");
 }
 
-setConfRet   taos_set_config(const char *config) {
+setConfRet taos_set_config(const char *config) {
   // TODO
   setConfRet ret = {SET_CONF_RET_SUCC, {0}};
   return ret;
@@ -135,8 +135,7 @@ int taos_field_count(TAOS_RES *res) {
     return 0;
   }
 
-  SRequestObj    *pRequest = (SRequestObj *)res;
-  SReqResultInfo *pResInfo = &pRequest->body.resInfo;
+  SReqResultInfo *pResInfo = tscGetCurResInfo(res);
   return pResInfo->numOfCols;
 }
 
@@ -147,7 +146,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
     return NULL;
   }
 
-  SReqResultInfo *pResInfo = &(((SRequestObj *)res)->body.resInfo);
+  SReqResultInfo *pResInfo = tscGetCurResInfo(res);
   return pResInfo->userFields;
 }
 
@@ -164,13 +163,36 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return NULL;
   }
 
-  SRequestObj *pRequest = (SRequestObj *)res;
-  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
-      pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
-    return NULL;
-  }
+  if (TD_RES_QUERY(res)) {
+    SRequestObj *pRequest = (SRequestObj *)res;
+    if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
+        pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
+      return NULL;
+    }
+
+    return doFetchRow(pRequest, true, true);
+
+  } else if (TD_RES_TMQ(res)) {
+    SMqRspObj      *msg = ((SMqRspObj *)res);
+    SReqResultInfo *pResultInfo = taosArrayGet(msg->res, msg->resIter);
+
+    doSetOneRowPtr(pResultInfo);
+    pResultInfo->current += 1;
 
-  return doFetchRow(pRequest, true);
+    if (pResultInfo->row == NULL) {
+      msg->resIter++;
+      pResultInfo = taosArrayGet(msg->res, msg->resIter);
+      doSetOneRowPtr(pResultInfo);
+      pResultInfo->current += 1;
+    }
+
+    return pResultInfo->row;
+
+  } else {
+    // assert to avoid uninitialization error
+    ASSERT(0);
+  }
+  return NULL;
 }
 
 int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
@@ -262,12 +284,12 @@ int *taos_fetch_lengths(TAOS_RES *res) {
     return NULL;
   }
 
-  return ((SRequestObj *)res)->body.resInfo.length;
+  SReqResultInfo *pResInfo = tscGetCurResInfo(res);
+  return pResInfo->length;
 }
 
 TAOS_ROW *taos_result_block(TAOS_RES *res) {
-  SRequestObj* pRequest = (SRequestObj*) res;
-  if (pRequest == NULL) {
+  if (res == NULL) {
     terrno = TSDB_CODE_INVALID_PARA;
     return NULL;
   }
@@ -276,7 +298,8 @@ TAOS_ROW *taos_result_block(TAOS_RES *res) {
     return NULL;
   }
 
-  return &pRequest->body.resInfo.row;
+  SReqResultInfo *pResInfo = tscGetCurResInfo(res);
+  return &pResInfo->row;
 }
 
 // todo intergrate with tDataTypes
@@ -315,7 +338,7 @@ const char *taos_data_type(int type) {
 const char *taos_get_client_info() { return version; }
 
 int taos_affected_rows(TAOS_RES *res) {
-  if (res == NULL) {
+  if (res == NULL || TD_RES_TMQ(res)) {
     return 0;
   }
 
@@ -325,12 +348,17 @@ int taos_affected_rows(TAOS_RES *res) {
 }
 
 int taos_result_precision(TAOS_RES *res) {
-  SRequestObj* pRequest = (SRequestObj*) res;
-  if (pRequest == NULL) {
+  if (res == NULL) {
     return TSDB_TIME_PRECISION_MILLI;
   }
-
-  return pRequest->body.resInfo.precision;
+  if (TD_RES_QUERY(res)) {
+    SRequestObj *pRequest = (SRequestObj *)res;
+    return pRequest->body.resInfo.precision;
+  } else if (TD_RES_TMQ(res)) {
+    SReqResultInfo *info = tmqGetCurResInfo(res);
+    return info->precision;
+  }
+  return TSDB_TIME_PRECISION_MILLI;
 }
 
 int taos_select_db(TAOS *taos, const char *db) {
@@ -372,90 +400,115 @@ void taos_stop_query(TAOS_RES *res) {
 }
 
 bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
-  SRequestObj    *pRequestObj = res;
-  SReqResultInfo *pResultInfo = &pRequestObj->body.resInfo;
+  SReqResultInfo *pResultInfo = tscGetCurResInfo(res);
   if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) {
     return true;
   }
 
-  SResultColumn *pCol = &pRequestObj->body.resInfo.pCol[col];
+  SResultColumn *pCol = &pResultInfo->pCol[col];
   return colDataIsNull_f(pCol->nullbitmap, row);
 }
 
-bool taos_is_update_query(TAOS_RES *res) {
-  return taos_num_fields(res) == 0;
-}
+bool taos_is_update_query(TAOS_RES *res) { return taos_num_fields(res) == 0; }
 
 int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
   int32_t numOfRows = 0;
-  /*int32_t code = */taos_fetch_block_s(res, &numOfRows, rows);
+  /*int32_t code = */ taos_fetch_block_s(res, &numOfRows, rows);
   return numOfRows;
 }
 
-int taos_fetch_block_s(TAOS_RES *res, int* numOfRows, TAOS_ROW *rows) {
-  SRequestObj *pRequest = (SRequestObj *)res;
-  if (pRequest == NULL) {
+int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
+  if (res == NULL) {
     return 0;
   }
+  if (TD_RES_QUERY(res)) {
+    SRequestObj *pRequest = (SRequestObj *)res;
 
-  (*rows)      = NULL;
-  (*numOfRows) = 0;
+    (*rows) = NULL;
+    (*numOfRows) = 0;
 
-  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
-      pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
-    return 0;
-  }
+    if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
+        pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
+      return 0;
+    }
 
-  doFetchRow(pRequest, false);
+    doFetchRow(pRequest, false, true);
 
-  // TODO refactor
-  SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
-  pResultInfo->current = pResultInfo->numOfRows;
+    // TODO refactor
+    SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
+    pResultInfo->current = pResultInfo->numOfRows;
 
-  (*rows)      = pResultInfo->row;
-  (*numOfRows) = pResultInfo->numOfRows;
-  return pRequest->code;
-}
+    (*rows) = pResultInfo->row;
+    (*numOfRows) = pResultInfo->numOfRows;
+    return pRequest->code;
+  } else if (TD_RES_TMQ(res)) {
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    if (pResultInfo == NULL) return -1;
 
-int taos_fetch_raw_block(TAOS_RES *res, int* numOfRows, void** pData) {
-  SRequestObj *pRequest = (SRequestObj *)res;
-  if (pRequest == NULL) {
+    pResultInfo->current = pResultInfo->numOfRows;
+    (*rows) = pResultInfo->row;
+    (*numOfRows) = pResultInfo->numOfRows;
     return 0;
+  } else {
+    ASSERT(0);
+    return -1;
   }
+}
 
-  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
-      pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
+int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
+  if (res == NULL) {
     return 0;
   }
+  if (TD_RES_QUERY(res)) {
+    SRequestObj *pRequest = (SRequestObj *)res;
 
-  doFetchRow(pRequest, false);
+    if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
+        pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
+      return 0;
+    }
+
+    doFetchRow(pRequest, false, false);
+
+    SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
 
-  SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
+    pResultInfo->current = pResultInfo->numOfRows;
+    (*numOfRows) = pResultInfo->numOfRows;
+    (*pData) = (void *)pResultInfo->pData;
+
+    return 0;
 
-  pResultInfo->current = pResultInfo->numOfRows;
-  (*numOfRows) = pResultInfo->numOfRows;
-  (*pData) = (void*) pResultInfo->pData;
+  } else if (TD_RES_TMQ(res)) {
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    if (pResultInfo == NULL) return -1;
 
-  return 0;
+    pResultInfo->current = pResultInfo->numOfRows;
+    (*numOfRows) = pResultInfo->numOfRows;
+    (*pData) = (void *)pResultInfo->pData;
+    return 0;
+
+  } else {
+    ASSERT(0);
+    return -1;
+  }
 }
 
 int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
-  SRequestObj *pRequest = (SRequestObj *)res;
-  if (pRequest == NULL) {
+  if (res == NULL) {
     return 0;
   }
 
-  int32_t numOfFields = taos_num_fields(pRequest);
+  int32_t numOfFields = taos_num_fields(res);
   if (columnIndex < 0 || columnIndex >= numOfFields || numOfFields == 0) {
     return 0;
   }
 
-  TAOS_FIELD* pField = &pRequest->body.resInfo.userFields[columnIndex];
+  SReqResultInfo *pResInfo = tscGetCurResInfo(res);
+  TAOS_FIELD     *pField = &pResInfo->userFields[columnIndex];
   if (!IS_VAR_DATA_TYPE(pField->type)) {
     return 0;
   }
 
-  return pRequest->body.resInfo.pCol[columnIndex].offset;
+  return pResInfo->pCol[columnIndex].offset;
 }
 
 int taos_validate_sql(TAOS *taos, const char *sql) { return true; }
@@ -485,18 +538,19 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
   // TODO
 }
 
-TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) {
-    // TODO
-    return NULL;
+TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp,
+                         void *param, int interval) {
+  // TODO
+  return NULL;
 }
 
 TAOS_RES *taos_consume(TAOS_SUB *tsub) {
-    // TODO
-    return NULL;
+  // TODO
+  return NULL;
 }
 
 void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
-    // TODO
+  // TODO
 }
 
 int taos_load_table_info(TAOS *taos, const char *tableNameList) {
@@ -555,26 +609,26 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
 }
 
 int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) {
-    // TODO
-    return -1;
+  // TODO
+  return -1;
 }
 
 int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
-    // TODO
-    return -1;
+  // TODO
+  return -1;
 }
 
-int taos_stmt_add_batch(TAOS_STMT* stmt) {
-    // TODO
-    return -1;
+int taos_stmt_add_batch(TAOS_STMT *stmt) {
+  // TODO
+  return -1;
 }
 
 TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
-    // TODO
-    return NULL;
+  // TODO
+  return NULL;
 }
 
-int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) {
-    // TODO
-    return -1;
+int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
+  // TODO
+  return -1;
 }

source/client/src/clientMsgHandler.c

@@ -191,7 +191,7 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code)
   pResInfo->completed = pRetrieve->completed;
 
   pResInfo->current = 0;
-  setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows);
+//  setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows);
 
   tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows,
            pRetrieve->completed, pRequest->body.showInfo.execId);
@@ -225,7 +225,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   pResInfo->pData     = pFetchRsp->data;
 
   pResInfo->current = 0;
-  setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows);
+//  setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows);
 
   tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pFetchRsp->numOfRows,
            pFetchRsp->completed, pRequest->body.showInfo.execId);

source/client/src/tmq.c

@@ -17,25 +17,19 @@
 #include "clientLog.h"
 #include "parser.h"
 #include "planner.h"
-#include "scheduler.h"
 #include "tdatablock.h"
 #include "tdef.h"
 #include "tglobal.h"
 #include "tmsgtype.h"
-#include "tpagedbuf.h"
 #include "tqueue.h"
 #include "tref.h"
 
-typedef struct {
-  int32_t curBlock;
-  int32_t curRow;
-  void**  uData;
-} SMqRowIter;
-
 struct tmq_message_t {
   SMqPollRsp msg;
+  char*      topic;
   void*      vg;
-  SMqRowIter iter;
+  SArray*    res;  // SArray<SReqResultInfo>
+  int32_t    resIter;
 };
 
 struct tmq_list_t {
@@ -849,7 +843,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   if (msgEpoch < tmqEpoch) {
     /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
     /*tsem_post(&tmq->rspSem);*/
-    tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
+    tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch,
+            tmqEpoch);
     return 0;
   }
 
@@ -886,8 +881,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
   memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
   tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg);
-  pRsp->iter.curBlock = 0;
-  pRsp->iter.curRow = 0;
+  /*pRsp->iter.curBlock = 0;*/
+  /*pRsp->iter.curRow = 0;*/
   // TODO: alloc mem
   /*pRsp->*/
   /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
@@ -899,8 +894,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 #endif
 
-  tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, pRsp->msg.reqOffset,
-           pRsp->msg.rspOffset);
+  tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId,
+           pRsp->msg.reqOffset, pRsp->msg.rspOffset);
 
   pRsp->vg = pParam->pVg;
   taosWriteQitem(tmq->mqueue, pRsp);
@@ -921,7 +916,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
   bool    set = false;
   int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
   char    vgKey[TSDB_TOPIC_FNAME_LEN + 22];
-  tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet);
+  tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch,
+           topicNumGet);
   SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic));
   if (newTopics == NULL) {
     return false;
@@ -1289,7 +1285,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
       atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
+      tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId,
+               pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
@@ -1566,30 +1563,4 @@ const char* tmq_err2str(tmq_resp_err_t err) {
   return "fail";
 }
 
-TAOS_ROW tmq_get_row(tmq_message_t* message) {
-  SMqPollRsp* rsp = &message->msg;
-  while (1) {
-    if (message->iter.curBlock < taosArrayGetSize(rsp->pBlockData)) {
-      SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->iter.curBlock);
-      if (message->iter.curRow < pBlock->info.rows) {
-        for (int i = 0; i < pBlock->info.numOfCols; i++) {
-          SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i);
-          if (colDataIsNull_s(pData, message->iter.curRow))
-            message->iter.uData[i] = NULL;
-          else {
-            message->iter.uData[i] = colDataGetData(pData, message->iter.curRow);
-          }
-        }
-        message->iter.curRow++;
-        return message->iter.uData;
-      } else {
-        message->iter.curBlock++;
-        message->iter.curRow = 0;
-        continue;
-      }
-    }
-    return NULL;
-  }
-}
-
 char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; }

source/common/src/tdataformat.c

@@ -411,6 +411,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
 #endif
 
   pCols->numOfRows = 0;
+  pCols->bitmapMode = 0;
   pCols->numOfCols = schemaNCols(pSchema);
 
   for (i = 0; i < schemaNCols(pSchema); ++i) {

source/common/src/tglobal.c

@@ -256,7 +256,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e
   return 0;
 }
 
-static int32_t taosAddClientLogCfg(SConfig *pCfg) {
+int32_t taosAddClientLogCfg(SConfig *pCfg) {
   if (cfgAddDir(pCfg, "configDir", configDir, 1) != 0) return -1;
   if (cfgAddDir(pCfg, "scriptDir", configDir, 1) != 0) return -1;
   if (cfgAddDir(pCfg, "logDir", tsLogDir, 1) != 0) return -1;

source/common/src/tmsg.c

@@ -419,6 +419,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
       tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
       for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].index);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
@@ -489,6 +490,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
       for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
+        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].index));
         buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
@@ -2100,7 +2102,7 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) {
   taosArrayDestroy(pRsp->pArray);
 }
 
-int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq) {
+int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
@@ -2113,7 +2115,7 @@ int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq) {
   return tlen;
 }
 
-int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq) {
+int32_t tDeserializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) {
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
@@ -2125,7 +2127,7 @@ int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq) {
   return 0;
 }
 
-int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp) {
+int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
@@ -2156,7 +2158,7 @@ int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp) {
   return tlen;
 }
 
-int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp) {
+int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) {
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
@@ -2187,7 +2189,7 @@ int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp) {
   return 0;
 }
 
-int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq) {
+int32_t tSerializeSUserIndexReq(void *buf, int32_t bufLen, SUserIndexReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
@@ -2200,7 +2202,7 @@ int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq)
   return tlen;
 }
 
-int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq) {
+int32_t tDeserializeSUserIndexReq(void *buf, int32_t bufLen, SUserIndexReq *pReq) {
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
@@ -2212,7 +2214,7 @@ int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq
   return 0;
 }
 
-int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp) {
+int32_t tSerializeSUserIndexRsp(void *buf, int32_t bufLen, const SUserIndexRsp *pRsp) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
@@ -2229,7 +2231,7 @@ int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp*
   return tlen;
 }
 
-int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp) {
+int32_t tDeserializeSUserIndexRsp(void *buf, int32_t bufLen, SUserIndexRsp *pRsp) {
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
@@ -2245,7 +2247,6 @@ int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp
   return 0;
 }
 
-
 int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);

source/common/src/trow.c

@@ -385,7 +385,6 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int
     pCol->len += pCol->bytes;
   }
 #ifdef TD_SUPPORT_BITMAP
-
     tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode);
 #endif
   return 0;
@@ -486,7 +485,7 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
  * @param pCols
  * @param forceSetNull
  */
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
+int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
   if (TD_IS_TP_ROW(pRow)) {
     return tdAppendTpRowToDataCol(pRow, pSchema, pCols);
   } else if (TD_IS_KV_ROW(pRow)) {
@@ -497,7 +496,7 @@ int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCol
   return TSDB_CODE_SUCCESS;
 }
 
-int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull) {
+int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, TDRowVerT maxVer) {
   ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
   ASSERT(target->numOfCols == source->numOfCols);
   int offset = 0;
@@ -510,6 +509,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *
 
   if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) {  // No overlap
     ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
+    // TODO: filter the maxVer
     for (int i = 0; i < rowsToMerge; i++) {
       for (int j = 0; j < source->numOfCols; j++) {
         if (source->cols[j].len > 0 || target->cols[j].len > 0) {
@@ -555,9 +555,9 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
     // TKEY  tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
 
     ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
-
+    // TODO: filter the maxVer
     if (key1 < key2) {
-      for (int i = 0; i < src1->numOfCols; i++) {
+      for (int i = 0; i < src1->numOfCols; ++i) {
         ASSERT(target->cols[i].type == src1->cols[i].type);
         if (src1->cols[i].len > 0 || target->cols[i].len > 0) {
           SCellVal sVal = {0};
@@ -568,12 +568,12 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
         }
       }
 
-      target->numOfRows++;
-      (*iter1)++;
+      ++target->numOfRows;
+      ++(*iter1);
     } else if (key1 >= key2) {
-      // if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
-      if ((key1 > key2) || (key1 == key2)) {
-        for (int i = 0; i < src2->numOfCols; i++) {
+      // TODO: filter the maxVer
+      if ((key1 > key2) || ((key1 == key2) && !TKEY_IS_DELETED(key2))) {
+        for (int i = 0; i < src2->numOfCols; ++i) {
           SCellVal sVal = {0};
           ASSERT(target->cols[i].type == src2->cols[i].type);
           if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2, src2->bitmapMode) < 0) {
@@ -590,11 +590,11 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
             dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode);
           }
         }
-        target->numOfRows++;
+        ++target->numOfRows;
       }
 
-      (*iter2)++;
-      if (key1 == key2) (*iter1)++;
+      ++(*iter2);
+      if (key1 == key2) ++(*iter1);
     }
 
     ASSERT(target->numOfRows <= target->maxPoints);

source/common/src/ttszip.c

@@ -39,7 +39,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
 
   taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path);
   // pTSBuf->pFile = fopen(pTSBuf->path, "wb+");
-  pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
   if (pTSBuf->pFile == NULL) {
     taosMemoryFree(pTSBuf);
     return NULL;

source/common/src/tvariant.c

@@ -1028,13 +1028,18 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
 
 char * taosVariantGet(SVariant *pVar, int32_t type) {
   switch (type) {
-    case TSDB_DATA_TYPE_BOOL: 
+    case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
     case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_INT: 
     case TSDB_DATA_TYPE_TIMESTAMP:
       return (char *)&pVar->i;
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_UBIGINT:
+      return (char *)&pVar->u;
     case TSDB_DATA_TYPE_DOUBLE:
     case TSDB_DATA_TYPE_FLOAT:
       return (char *)&pVar->d;
@@ -1042,7 +1047,7 @@ char * taosVariantGet(SVariant *pVar, int32_t type) {
       return (char *)pVar->pz;
     case TSDB_DATA_TYPE_NCHAR:
       return (char *)pVar->ucs4;
-    default: 
+    default:
       return NULL;
   }
 

source/dnode/mgmt/bm/bmHandle.c

@@ -58,7 +58,7 @@ int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
     dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId);
     return -1;
   } else {
-    return bmOpen(pWrapper);
+    return dndOpenNode(pWrapper);
   }
 }
 
@@ -77,6 +77,7 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
     dError("failed to drop bnode since %s", terrstr());
     return -1;
   } else {
+    // dndCloseNode(pWrapper);
     return bmDrop(pWrapper);
   }
 }

source/dnode/mgmt/dm/dmFile.c

@@ -162,7 +162,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
   char file[PATH_MAX];
   snprintf(file, sizeof(file), "%s%sdnode.json.bak", pMgmt->path, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     dError("failed to write %s since %s", file, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);

source/dnode/mgmt/dm/dmMonitor.c

@@ -25,11 +25,10 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
 
 static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
   pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f);
-  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE);
-  if (pWrapper != NULL) {
-    pInfo->has_mnode = pWrapper->required;
-    dndReleaseWrapper(pWrapper);
-  }
+  pInfo->has_mnode = pDnode->wrappers[MNODE].required;
+  pInfo->has_qnode = pDnode->wrappers[QNODE].required;
+  pInfo->has_snode = pDnode->wrappers[SNODE].required;
+  pInfo->has_bnode = pDnode->wrappers[BNODE].required;
   tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
   pInfo->logdir.size = tsLogSpace.size;
   tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
@@ -65,7 +64,7 @@ void dmSendMonitorReport(SDnode *pDnode) {
   bool getFromAPI = !tsMultiProcess;
   pWrapper = &pDnode->wrappers[MNODE];
   if (getFromAPI) {
-    if (dndMarkWrapper(pWrapper) != 0) {
+    if (dndMarkWrapper(pWrapper) == 0) {
       mmGetMonitorInfo(pWrapper, &mmInfo);
       dndReleaseWrapper(pWrapper);
     }
@@ -82,7 +81,7 @@ void dmSendMonitorReport(SDnode *pDnode) {
 
   pWrapper = &pDnode->wrappers[VNODES];
   if (getFromAPI) {
-    if (dndMarkWrapper(pWrapper) != 0) {
+    if (dndMarkWrapper(pWrapper) == 0) {
       vmGetMonitorInfo(pWrapper, &vmInfo);
       dndReleaseWrapper(pWrapper);
     }
@@ -99,7 +98,7 @@ void dmSendMonitorReport(SDnode *pDnode) {
 
   pWrapper = &pDnode->wrappers[QNODE];
   if (getFromAPI) {
-    if (dndMarkWrapper(pWrapper) != 0) {
+    if (dndMarkWrapper(pWrapper) == 0) {
       qmGetMonitorInfo(pWrapper, &qmInfo);
       dndReleaseWrapper(pWrapper);
     }
@@ -116,7 +115,7 @@ void dmSendMonitorReport(SDnode *pDnode) {
 
   pWrapper = &pDnode->wrappers[SNODE];
   if (getFromAPI) {
-    if (dndMarkWrapper(pWrapper) != 0) {
+    if (dndMarkWrapper(pWrapper) == 0) {
       smGetMonitorInfo(pWrapper, &smInfo);
       dndReleaseWrapper(pWrapper);
     }
@@ -133,7 +132,7 @@ void dmSendMonitorReport(SDnode *pDnode) {
 
   pWrapper = &pDnode->wrappers[BNODE];
   if (getFromAPI) {
-    if (dndMarkWrapper(pWrapper) != 0) {
+    if (dndMarkWrapper(pWrapper) == 0) {
       bmGetMonitorInfo(pWrapper, &bmInfo);
       dndReleaseWrapper(pWrapper);
     }

source/dnode/mgmt/inc/bmInt.h

@@ -27,7 +27,7 @@ extern "C" {
 typedef struct SBnodeMgmt {
   SBnode       *pBnode;
   SDnode       *pDnode;
-  SMgmtWrapper *pWrapper;
+  SMgmtWrapper *pWrapper;  
   const char   *path;
   SMultiWorker  writeWorker;
   SSingleWorker monitorWorker;

source/dnode/mgmt/inc/dndInt.h

@@ -126,6 +126,7 @@ typedef struct SDnode {
   int32_t      numOfDisks;
   uint16_t     serverPort;
   bool         dropped;
+  EProcType    procType;
   EDndType     ntype;
   EDndStatus   status;
   EDndEvent    event;

source/dnode/mgmt/main/dndExec.c

@@ -27,7 +27,81 @@ static bool dndRequireNode(SMgmtWrapper *pWrapper) {
   return required;
 }
 
-int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
+static int32_t dndInitNodeProc(SMgmtWrapper *pWrapper) {
+  int32_t shmsize = tsMnodeShmSize;
+  if (pWrapper->ntype == VNODES) {
+    shmsize = tsVnodeShmSize;
+  } else if (pWrapper->ntype == QNODE) {
+    shmsize = tsQnodeShmSize;
+  } else if (pWrapper->ntype == SNODE) {
+    shmsize = tsSnodeShmSize;
+  } else if (pWrapper->ntype == MNODE) {
+    shmsize = tsMnodeShmSize;
+  } else if (pWrapper->ntype == BNODE) {
+    shmsize = tsBnodeShmSize;
+  } else {
+    return -1;
+  }
+
+  if (taosCreateShm(&pWrapper->shm, pWrapper->ntype, shmsize) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(terrno);
+    dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
+    return -1;
+  }
+  dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->shm.id, shmsize);
+
+  SProcCfg cfg = dndGenProcCfg(pWrapper);
+  cfg.isChild = false;
+  pWrapper->procType = PROC_PARENT;
+  pWrapper->pProc = taosProcInit(&cfg);
+  if (pWrapper->pProc == NULL) {
+    dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t dndNewNodeProc(SMgmtWrapper *pWrapper, EDndType n) {
+  char  tstr[8] = {0};
+  char *args[6] = {0};
+  snprintf(tstr, sizeof(tstr), "%d", n);
+  args[1] = "-c";
+  args[2] = configDir;
+  args[3] = "-n";
+  args[4] = tstr;
+  args[5] = NULL;
+
+  int32_t pid = taosNewProc(args);
+  if (pid <= 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("node:%s, failed to exec in new process since %s", pWrapper->name, terrstr());
+    return -1;
+  }
+
+  pWrapper->procId = pid;
+  dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid);
+  return 0;
+}
+
+static int32_t dndRunNodeProc(SMgmtWrapper *pWrapper) {
+  if (pWrapper->pDnode->ntype == NODE_MAX) {
+    dInfo("node:%s, should be started manually", pWrapper->name);
+  } else {
+    if (dndNewNodeProc(pWrapper, pWrapper->ntype) != 0) {
+      return -1;
+    }
+  }
+
+  if (taosProcRun(pWrapper->pProc) != 0) {
+    dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t dndOpenNodeImp(SMgmtWrapper *pWrapper) {
   if (taosMkDir(pWrapper->path) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr());
@@ -44,7 +118,19 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
   return 0;
 }
 
-void dndCloseNode(SMgmtWrapper *pWrapper) {
+int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
+  SDnode *pDnode = pWrapper->pDnode;
+  if (pDnode->procType == PROC_SINGLE) {
+    return dndOpenNodeImp(pWrapper);
+  } else if (pDnode->procType == PROC_PARENT) {
+    if (dndInitNodeProc(pWrapper) != 0) return -1;
+    if (dndWriteShmFile(pDnode) != 0) return -1;
+    if (dndRunNodeProc(pWrapper) != 0) return -1;
+  }
+  return 0;
+}
+
+static void dndCloseNodeImp(SMgmtWrapper *pWrapper) {
   dDebug("node:%s, mgmt start to close", pWrapper->name);
   pWrapper->required = false;
   taosWLockLatch(&pWrapper->latch);
@@ -65,27 +151,17 @@ void dndCloseNode(SMgmtWrapper *pWrapper) {
   dDebug("node:%s, mgmt has been closed", pWrapper->name);
 }
 
-
-static int32_t dndNewProc(SMgmtWrapper *pWrapper, EDndType n) {
-  char  tstr[8] = {0};
-  char *args[6] = {0};
-  snprintf(tstr, sizeof(tstr), "%d", n);
-  args[1] = "-c";
-  args[2] = configDir;
-  args[3] = "-n";
-  args[4] = tstr;
-  args[5] = NULL;
-
-  int32_t pid = taosNewProc(args);
-  if (pid <= 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("node:%s, failed to exec in new process since %s", pWrapper->name, terrstr());
-    return -1;
+void dndCloseNode(SMgmtWrapper *pWrapper) {
+  if (pWrapper->pDnode->procType == PROC_PARENT) {
+    if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
+      dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId);
+      taosKillProc(pWrapper->procId);
+      dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pWrapper->procId);
+      taosWaitProc(pWrapper->procId);
+      dInfo("node:%s, child process:%d is stopped", pWrapper->name, pWrapper->procId);
+    }
   }
-
-  pWrapper->procId = pid;
-  dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid);
-  return 0;
+  dndCloseNodeImp(pWrapper);
 }
 
 static void dndProcessProcHandle(void *handle) {
@@ -96,13 +172,14 @@ static void dndProcessProcHandle(void *handle) {
 
 static int32_t dndRunInSingleProcess(SDnode *pDnode) {
   dInfo("dnode run in single process");
+  pDnode->procType = PROC_SINGLE;
 
   for (EDndType n = DNODE; n < NODE_MAX; ++n) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
     pWrapper->required = dndRequireNode(pWrapper);
     if (!pWrapper->required) continue;
 
-    if (dndOpenNode(pWrapper) != 0) {
+    if (dndOpenNodeImp(pWrapper) != 0) {
       dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
       return -1;
     }
@@ -136,8 +213,10 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) {
 
 static int32_t dndRunInParentProcess(SDnode *pDnode) {
   dInfo("dnode run in parent process");
+  pDnode->procType = PROC_PARENT;
+
   SMgmtWrapper *pDWrapper = &pDnode->wrappers[DNODE];
-  if (dndOpenNode(pDWrapper) != 0) {
+  if (dndOpenNodeImp(pDWrapper) != 0) {
     dError("node:%s, failed to start since %s", pDWrapper->name, terrstr());
     return -1;
   }
@@ -146,36 +225,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
     pWrapper->required = dndRequireNode(pWrapper);
     if (!pWrapper->required) continue;
-
-    int32_t shmsize = tsMnodeShmSize;
-    if (n == VNODES) {
-      shmsize = tsVnodeShmSize;
-    } else if (n == QNODE) {
-      shmsize = tsQnodeShmSize;
-    } else if (n == SNODE) {
-      shmsize = tsSnodeShmSize;
-    } else if (n == MNODE) {
-      shmsize = tsMnodeShmSize;
-    } else if (n == BNODE) {
-      shmsize = tsBnodeShmSize;
-    } else {
-    }
-
-    if (taosCreateShm(&pWrapper->shm, n, shmsize) != 0) {
-      terrno = TAOS_SYSTEM_ERROR(terrno);
-      dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
-      return -1;
-    }
-    dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->shm.id, shmsize);
-
-    SProcCfg cfg = dndGenProcCfg(pWrapper);
-    cfg.isChild = false;
-    pWrapper->procType = PROC_PARENT;
-    pWrapper->pProc = taosProcInit(&cfg);
-    if (pWrapper->pProc == NULL) {
-      dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr());
-      return -1;
-    }
+    if (dndInitNodeProc(pWrapper) != 0) return -1;
   }
 
   if (dndWriteShmFile(pDnode) != 0) {
@@ -186,19 +236,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
   for (EDndType n = DNODE + 1; n < NODE_MAX; ++n) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
     if (!pWrapper->required) continue;
-
-    if (pDnode->ntype == NODE_MAX) {
-      dInfo("node:%s, should be started manually", pWrapper->name);
-    } else {
-      if (dndNewProc(pWrapper, n) != 0) {
-        return -1;
-      }
-    }
-
-    if (taosProcRun(pWrapper->pProc) != 0) {
-      dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
-      return -1;
-    }
+    if (dndRunNodeProc(pWrapper) != 0) return -1;
   }
 
   dndSetStatus(pDnode, DND_STAT_RUNNING);
@@ -239,7 +277,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
         if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) {
           dWarn("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
           taosProcCloseHandles(pWrapper->pProc, dndProcessProcHandle);
-          dndNewProc(pWrapper, n);
+          dndNewNodeProc(pWrapper, n);
         }
       }
     }
@@ -253,6 +291,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
 static int32_t dndRunInChildProcess(SDnode *pDnode) {
   SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
   dInfo("%s run in child process", pWrapper->name);
+  pDnode->procType = PROC_CHILD;
 
   pWrapper->required = dndRequireNode(pWrapper);
   if (!pWrapper->required) {
@@ -264,7 +303,7 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) {
   tmsgSetDefaultMsgCb(&msgCb);
   pWrapper->procType = PROC_CHILD;
 
-  if (dndOpenNode(pWrapper) != 0) {
+  if (dndOpenNodeImp(pWrapper) != 0) {
     dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
     return -1;
   }

source/dnode/mgmt/main/dndFile.c

@@ -75,7 +75,7 @@ int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) {
   snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
   snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
 
-  pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());
@@ -121,7 +121,7 @@ TdFilePtr dndCheckRunning(const char *dataDir) {
   char filepath[PATH_MAX] = {0};
   snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to lock file:%s since %s", filepath, terrstr());
@@ -218,7 +218,7 @@ int32_t dndWriteShmFile(SDnode *pDnode) {
   snprintf(file, sizeof(file), "%s%s.shmfile.bak", pDnode->dataDir, TD_DIRSEP);
   snprintf(realfile, sizeof(realfile), "%s%s.shmfile", pDnode->dataDir, TD_DIRSEP);
 
-  pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to open file:%s since %s", file, terrstr());

source/dnode/mgmt/mm/mmFile.c

@@ -109,7 +109,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) {
   char file[PATH_MAX];
   snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());

source/dnode/mgmt/mm/mmHandle.c

@@ -80,6 +80,7 @@ int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
     dError("failed to drop mnode since %s", terrstr());
     return -1;
   } else {
+    // dndCloseNode(pWrapper);
     return mmDrop(pWrapper);
   }
 }

source/dnode/mgmt/qm/qmHandle.c

@@ -58,7 +58,7 @@ int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
     dError("failed to create qnode since %s", terrstr());
     return -1;
   } else {
-    return qmOpen(pWrapper);
+    return dndOpenNode(pWrapper);
   }
 }
 
@@ -77,6 +77,7 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
     dError("failed to drop qnode since %s", terrstr());
     return -1;
   } else {
+    // dndCloseNode(pWrapper);
     return qmDrop(pWrapper);
   }
 }

source/dnode/mgmt/qm/qmWorker.c

@@ -32,7 +32,7 @@ static void qmProcessMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   SRpcMsg *pRpc = &pMsg->rpcMsg;
   int32_t  code = -1;
 
-  if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) {
+  if (pMsg->rpcMsg.msgType == TDMT_MON_QM_INFO) {
     code = qmProcessGetMonQmInfoReq(pMgmt->pWrapper, pMsg);
   } else {
     terrno = TSDB_CODE_MSG_NOT_PROCESSED;

source/dnode/mgmt/sm/smHandle.c

@@ -58,7 +58,7 @@ int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
     dError("failed to create snode since %s", terrstr());
     return -1;
   } else {
-    return smOpen(pWrapper);
+    return dndOpenNode(pWrapper);
   }
 }
 
@@ -78,6 +78,7 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
     return -1;
   } else {
     return smDrop(pWrapper);
+    // return dndCloseNode(pWrapper);
   }
 }
 

source/dnode/mgmt/vm/vmFile.c

@@ -154,7 +154,7 @@ int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) {
   snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP);
   snprintf(realfile, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());

source/dnode/mnode/impl/test/trans/trans.cpp

@@ -38,7 +38,7 @@ class MndTestTrans : public ::testing::Test {
 
     test.ServerStop();
 
-    pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+    pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
     int32_t writeLen = taosWriteFile(pFile, buffer, readLen);
     if (writeLen < 0 || writeLen == readLen) {
       ASSERT(1);

source/dnode/mnode/sdb/src/sdbFile.c

@@ -232,7 +232,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
   mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer,
          pSdb->lastCommitVer);
 
-  TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to open file:%s for write since %s", tmpfile, terrstr());

source/dnode/vnode/CMakeLists.txt

@@ -55,6 +55,8 @@ target_include_directories(
     vnode
     PUBLIC "inc"
     PRIVATE "src/inc"
+    PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
+    
 )
 target_link_libraries(
     vnode
@@ -69,6 +71,7 @@ target_link_libraries(
     PUBLIC scheduler
     PUBLIC tdb
     #PUBLIC bdb
+    #PUBLIC scalar
     PUBLIC transport
     PUBLIC stream
 )

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -95,7 +95,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
     tqError("failed to create dir:%s since %s ", name, terrstr());
   }
   strcat(name, "/" TQ_IDX_NAME);
-  TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
+  TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ);
   if (pIdxFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tqError("failed to open file:%s since %s ", name, terrstr());
@@ -113,7 +113,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
 
   strcpy(name, path);
   strcat(name, "/" TQ_META_NAME);
-  TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
+  TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tqError("failed to open file:%s since %s", name, terrstr());

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1298,7 +1298,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
     int32_t tBitmaps = 0;
     int32_t tBitmapsLen = 0;
     if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
-      tBitmaps = sBitmaps;
+      tBitmaps = isSuper ? sBitmaps : nBitmaps;
     }
 #endif
 
@@ -1678,10 +1678,11 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
         ASSERT(pSchema != NULL);
       }
 
-      tdAppendSTSRowToDataCol(row, pSchema, pTarget, true);
+      tdAppendSTSRowToDataCol(row, pSchema, pTarget);
 
       tSkipListIterNext(pCommitIter->pIter);
     } else {
+#if 0
       if (update != TD_ROW_OVERWRITE_UPDATE) {
         // copy disk data
         for (int i = 0; i < pDataCols->numOfCols; ++i) {
@@ -1706,6 +1707,43 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
       }
       ++(*iter);
       tSkipListIterNext(pCommitIter->pIter);
+#endif
+      // copy disk data
+      for (int i = 0; i < pDataCols->numOfCols; ++i) {
+        SCellVal sVal = {0};
+        if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
+          TASSERT(0);
+        }
+        // TODO: tdAppendValToDataCol may fail
+        tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints,
+                             pTarget->bitmapMode);
+      }
+
+      if (TD_SUPPORT_UPDATE(update)) {
+        // copy mem data(Multi-Version)
+        if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
+          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row));
+          ASSERT(pSchema != NULL);
+        }
+
+        // TODO: merge with Multi-Version
+        STSRow *curRow = row;
+
+        ++(*iter);
+        tSkipListIterNext(pCommitIter->pIter);
+        STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter);
+
+        if (key2 < TD_ROW_KEY(nextRow)) {
+          tdAppendSTSRowToDataCol(row, pSchema, pTarget);
+        } else {
+          tdAppendSTSRowToDataCol(row, pSchema, pTarget);
+        }
+        // TODO: merge with Multi-Version
+      } else {
+        ++pTarget->numOfRows;
+        ++(*iter);
+        tSkipListIterNext(pCommitIter->pIter);
+      }
     }
 
     if (pTarget->numOfRows >= maxRows) break;

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -416,7 +416,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
   tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname);
   tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname);
 
-  TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -360,7 +360,7 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
 int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType) {
   ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
 
-  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pDFile->pFile == NULL) {
     if (errno == ENOENT) {
       // Try to create directory recursively
@@ -371,7 +371,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T
       }
       taosMemoryFreeClear(s);
 
-      pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+      pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
       if (pDFile->pFile == NULL) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -475,7 +475,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema *
       }
     }
 
-    tdAppendSTSRowToDataCol(row, *ppSchema, pCols, true);
+    tdAppendSTSRowToDataCol(row, *ppSchema, pCols);
   }
 
   return 0;

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -263,8 +263,9 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
   for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
     iBlock++;
     if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
+    // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
     if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
-                        update != TD_ROW_PARTIAL_UPDATE) < 0)
+                        update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0)
       return -1;
   }
 
@@ -293,8 +294,9 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
   for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
     iBlock++;
     if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
+    // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
     if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
-                        update != TD_ROW_PARTIAL_UPDATE) < 0)
+                        update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0)
       return -1;
   }
 
@@ -304,6 +306,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
       if (pDataCol->bitmap) {
         ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID);
         tdMergeBitmap(pDataCol->pBitmap, TD_BITMAP_BYTES(pReadh->pDCols[0]->numOfRows), pDataCol->pBitmap);
+        tdDataColsSetBitmapI(pReadh->pDCols[0]);
       }
     }
   }

source/libs/command/src/explain.c

@@ -639,7 +639,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
     }
     case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{
       SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
-      EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->pTspk));
+      EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk));
       EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
       if (pResNode->pExecInfo) {
         QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));

source/libs/executor/inc/executorimpl.h

@@ -232,9 +232,11 @@ typedef struct STaskAttr {
 } STaskAttr;
 
 struct SOperatorInfo;
+struct SAggSupporter;
+struct SOptrBasicInfo;
 
-typedef void (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, char **result, int32_t *length);
-typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char *result, int32_t length);
+typedef void (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, struct SAggSupporter *pSup, struct SOptrBasicInfo *pInfo, char **result, int32_t *length);
+typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, struct SAggSupporter *pSup, struct SOptrBasicInfo *pInfo, char *result, int32_t length);
 
 typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr);
 typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr, bool* newgroup);
@@ -409,7 +411,7 @@ typedef struct STableScanInfo {
   SResultRowInfo* pResultRowInfo;
   int32_t*        rowCellInfoOffset;
   SExprInfo*      pExpr;
-  SSDataBlock     block;
+  SSDataBlock*    pResBlock;
   SArray*         pColMatchInfo;
   int32_t         numOfOutput;
   int64_t         elapsedTime;
@@ -567,6 +569,9 @@ typedef struct SGroupbyOperatorInfo {
   int32_t        groupKeyLen;   // total group by column width
   SGroupResInfo  groupResInfo;
   SAggSupporter  aggSup;
+  SExprInfo*     pScalarExprInfo;
+  int32_t        numOfScalarExpr;// the number of scalar expression in group operator
+  SqlFunctionCtx*pScalarFuncCtx;
 } SGroupbyOperatorInfo;
 
 typedef struct SDataGroupInfo {
@@ -672,7 +677,7 @@ void operatorDummyCloseFn(void* param, int32_t numOfCols);
 int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num);
 int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols,
                     int32_t numOfRows, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey);
-void toSDatablock(SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset);
+void toSDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, int32_t* rowCellOffset);
 void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
 void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order);
 int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type,
@@ -683,12 +688,11 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
                                          uint64_t* total, SArray* pColList);
 void doSetOperatorCompleted(SOperatorInfo* pOperator);
 void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock);
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity);
-SSDataBlock* loadNextDataBlock(void* param);
+SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset);
 
 SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t repeatTime,
-                                           int32_t reverseTime, SArray* pColMatchInfo, SNode* pCondition, SExecTaskInfo* pTaskInfo);
+                                           int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
                                            SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
 SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
@@ -702,8 +706,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
 SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlot,
                                           const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
-                                       SArray* pGroupColList, SNode* pCondition, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
+SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList,
+                                       SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
 SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo);
 
@@ -727,6 +731,8 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOf
                                       int32_t numOfOutput);
 #endif
 
+void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList);
+
 void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order);
 
 void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput);
@@ -753,6 +759,9 @@ void    setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
 int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, EOPTR_EXEC_MODEL model);
 int32_t getOperatorExplainExecInfo(SOperatorInfo *operatorInfo, SExplainExecInfo **pRes, int32_t *capacity, int32_t *resNum);
 
+bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasicInfo *pInfo, char* result, int32_t length);
+void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasicInfo *pInfo, char **result, int32_t *length);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/executor/src/dataDispatcher.c

@@ -92,9 +92,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput,
     return false;
   }
 
-  // NOTE: there are four bytes of an integer more than the required buffer space.
-  // struct size + data payload + length for each column + bitmap length
-  pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockDataGetSerialMetaSize(pInput->pData) + blockDataGetSize(pInput->pData);
+  pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockEstimateEncodeSize(pInput->pData);
 
   pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
   if (pBuf->pData == NULL) {

source/libs/executor/src/groupoperator.c

@@ -166,7 +166,7 @@ static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) {
 // assign the group keys or user input constant values if required
 static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) {
   for (int32_t i = 0; i < numOfOutput; ++i) {
-    if (pCtx[i].functionId == -1) {
+    if (pCtx[i].functionId == -1) {       // select count(*),key from t group by key.
       SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]);
 
       SColumnInfoData* pColInfoData = pCtx[i].input.pData[0];
@@ -265,7 +265,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
   SSDataBlock* pRes = pInfo->binfo.pRes;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset);
+    toSDatablock(pRes, pInfo->binfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset);
     if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       pOperator->status = OP_EXEC_DONE;
     }
@@ -285,6 +285,12 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
 
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order);
+
+    // there is an scalar expression that needs to be calculated before apply the group aggregation.
+    if (pInfo->pScalarExprInfo != NULL) {
+      projectApplyFunctions(pInfo->pScalarExprInfo, pBlock, pBlock, pInfo->pScalarFuncCtx, pInfo->numOfScalarExpr, NULL);
+    }
+
     //    setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput);
     doHashGroupbyAgg(pOperator, pBlock);
   }
@@ -305,7 +311,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
   initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
 
   while(1) {
-    toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset);
+    toSDatablock(pRes, pInfo->binfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset);
     doFilter(pInfo->pCondition, pRes);
 
     bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo);
@@ -322,16 +328,21 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
   return (pRes->info.rows == 0)? NULL:pRes;
 }
 
-SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExecTaskInfo* pTaskInfo,
-                                       const STableGroupInfo* pTableGroupInfo) {
+SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList,
+    SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) {
   SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo));
   SOperatorInfo*        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
     goto _error;
   }
 
-  pInfo->pGroupCols = pGroupColList;
-  pInfo->pCondition = pCondition;
+  pInfo->pGroupCols      = pGroupColList;
+  pInfo->pCondition      = pCondition;
+
+  pInfo->pScalarExprInfo = pScalarExprInfo;
+  pInfo->numOfScalarExpr = numOfScalarExpr;
+  pInfo->pScalarFuncCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset);
+
 
   int32_t code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList);
   if (code != TSDB_CODE_SUCCESS) {
@@ -344,7 +355,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
   pOperator->name         = "GroupbyAggOperator";
   pOperator->blockingOptr = true;
   pOperator->status       = OP_NOT_OPENED;
-  //  pOperator->operatorType = OP_Groupby;
+  // pOperator->operatorType = OP_Groupby;
   pOperator->pExpr        = pExprInfo;
   pOperator->numOfOutput  = numOfCols;
   pOperator->info         = pInfo;
@@ -352,6 +363,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
   pOperator->_openFn      = operatorDummyOpenFn;
   pOperator->getNextFn    = hashGroupbyAggregate;
   pOperator->closeFn      = destroyGroupOperatorInfo;
+  pOperator->encodeResultRow = aggEncodeResultRow;
+  pOperator->decodeResultRow = aggDecodeResultRow;
 
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;

source/libs/executor/src/scanoperator.c

@@ -113,7 +113,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
   STableScanInfo* pTableScanInfo = pOperator->info;
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
 
-  SSDataBlock*     pBlock = &pTableScanInfo->block;
+  SSDataBlock*     pBlock = pTableScanInfo->pResBlock;
   STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo;
 
   *newgroup = false;
@@ -218,7 +218,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
 }
 
 SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput,
-                                           int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo,
+                                           int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock,
                                            SNode* pCondition, SExecTaskInfo* pTaskInfo) {
   assert(repeatTime > 0);
 
@@ -232,12 +232,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
     return NULL;
   }
 
-  pInfo->block.pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData));
-  for (int32_t i = 0; i < numOfOutput; ++i) {
-    SColumnInfoData idata = {0};
-    taosArrayPush(pInfo->block.pDataBlock, &idata);
-  }
-
+  pInfo->pResBlock        = pResBlock;
   pInfo->pFilterNode      = pCondition;
   pInfo->dataReader       = pTsdbReadHandle;
   pInfo->times            = repeatTime;
@@ -312,7 +307,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) {
   tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist);
   tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader);
 
-  SSDataBlock* pBlock = &pTableScanInfo->block;
+  SSDataBlock* pBlock = pTableScanInfo->pResBlock;
   pBlock->info.rows   = 1;
   pBlock->info.numOfCols = 1;
 
@@ -343,13 +338,13 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo*
   }
 
   pInfo->dataReader       = dataReader;
-  pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+//  pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
 
   SColumnInfoData infoData = {0};
   infoData.info.type  = TSDB_DATA_TYPE_BINARY;
   infoData.info.bytes = 1024;
   infoData.info.colId = 0;
-  taosArrayPush(pInfo->block.pDataBlock, &infoData);
+//  taosArrayPush(pInfo->block.pDataBlock, &infoData);
 
   pOperator->name          = "DataBlockInfoScanOperator";
   //  pOperator->operatorType = OP_TableBlockInfoScan;

source/libs/function/inc/builtins.h

@@ -26,25 +26,29 @@ extern "C" {
 
 #define FUNC_MGT_FUNC_CLASSIFICATION_MASK(n)    (1 << n)
 
-#define FUNC_MGT_AGG_FUNC             FUNC_MGT_FUNC_CLASSIFICATION_MASK(0)
-#define FUNC_MGT_SCALAR_FUNC          FUNC_MGT_FUNC_CLASSIFICATION_MASK(1)
-#define FUNC_MGT_NONSTANDARD_SQL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(2)
-#define FUNC_MGT_STRING_FUNC          FUNC_MGT_FUNC_CLASSIFICATION_MASK(3)
-#define FUNC_MGT_DATETIME_FUNC        FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
-#define FUNC_MGT_TIMELINE_FUNC        FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)
-#define FUNC_MGT_TIMEORDER_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
-#define FUNC_MGT_PSEUDO_COLUMN_FUNC   FUNC_MGT_FUNC_CLASSIFICATION_MASK(7)
-#define FUNC_MGT_WINDOW_PC_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
+#define FUNC_MGT_AGG_FUNC               FUNC_MGT_FUNC_CLASSIFICATION_MASK(0)
+#define FUNC_MGT_SCALAR_FUNC            FUNC_MGT_FUNC_CLASSIFICATION_MASK(1)
+#define FUNC_MGT_NONSTANDARD_SQL_FUNC   FUNC_MGT_FUNC_CLASSIFICATION_MASK(2)
+#define FUNC_MGT_STRING_FUNC            FUNC_MGT_FUNC_CLASSIFICATION_MASK(3)
+#define FUNC_MGT_DATETIME_FUNC          FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
+#define FUNC_MGT_TIMELINE_FUNC          FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)
+#define FUNC_MGT_TIMEORDER_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
+#define FUNC_MGT_PSEUDO_COLUMN_FUNC     FUNC_MGT_FUNC_CLASSIFICATION_MASK(7)
+#define FUNC_MGT_WINDOW_PC_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
+#define FUNC_MGT_SPECIAL_DATA_REQUIRED  FUNC_MGT_FUNC_CLASSIFICATION_MASK(9)
+#define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10)
 
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 
 typedef int32_t (*FCheckAndGetResultType)(SFunctionNode* pFunc);
+typedef EFuncDataRequired (*FFuncDataRequired)(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
 
 typedef struct SBuiltinFuncDefinition {
   char name[FUNCTION_NAME_MAX_LENGTH];
   EFunctionType type;
   uint64_t classification;
   FCheckAndGetResultType checkFunc;
+  FFuncDataRequired dataRequiredFunc;
   FExecGetEnv getEnvFunc;
   FExecInit initFunc;
   FExecProcess processFunc;

source/libs/function/inc/builtinsimpl.h

@@ -21,10 +21,12 @@ extern "C" {
 #endif
 
 #include "function.h"
+#include "functionMgt.h"
 
 bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 void functionFinalize(SqlFunctionCtx *pCtx);
 
+EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
 bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t countFunction(SqlFunctionCtx *pCtx);
 

source/libs/function/src/builtinsimpl.c

@@ -55,6 +55,14 @@ void functionFinalize(SqlFunctionCtx *pCtx) {
   pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0;
 }
 
+EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
+  SNode* pParam = nodesListGetNode(pFunc->pParameterList, 0);
+  if (QUERY_NODE_COLUMN == nodeType(pParam) && PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pParam)->colId) {
+    return FUNC_DATA_REQUIRED_NO_NEEDED;
+  }
+  return FUNC_DATA_REQUIRED_STATIS_NEEDED;
+}
+
 bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(int64_t);
   return true;
@@ -212,6 +220,9 @@ bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
     case TSDB_DATA_TYPE_UTINYINT:
       *((uint8_t *)buf) = 0;
       break;
+    case TSDB_DATA_TYPE_BOOL:
+      *((int8_t*)buf) = 0;
+      break;
     default:
       assert(0);
   }
@@ -255,6 +266,9 @@ bool minFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
     case TSDB_DATA_TYPE_DOUBLE:
       SET_DOUBLE_VAL(((double *)buf), DBL_MAX);
       break;
+    case TSDB_DATA_TYPE_BOOL:
+      *((int8_t*)buf) = 1;
+      break;
     default:
       assert(0);
   }
@@ -385,8 +399,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
   int32_t start = pInput->startRowIndex;
   int32_t numOfRows = pInput->numOfRows;
 
-  if (IS_SIGNED_NUMERIC_TYPE(type)) {
-    if (type == TSDB_DATA_TYPE_TINYINT) {
+  if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
+    if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
       LOOPCHECK_N(*(int8_t*)buf, pCol, pCtx, int8_t, numOfRows, start, isMinFunc, numOfElems);
     } else if (type == TSDB_DATA_TYPE_SMALLINT) {
       LOOPCHECK_N(*(int16_t*) buf, pCol, pCtx, int16_t, numOfRows, start, isMinFunc, numOfElems);

source/libs/function/src/functionMgt.c

@@ -76,6 +76,16 @@ int32_t fmGetFuncResultType(SFunctionNode* pFunc) {
   return funcMgtBuiltins[pFunc->funcId].checkFunc(pFunc);
 }
 
+EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
+  if (pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) {
+    return FUNC_DATA_REQUIRED_ALL_NEEDED;
+  }
+  if (NULL == funcMgtBuiltins[pFunc->funcId].dataRequiredFunc) {
+    return FUNC_DATA_REQUIRED_ALL_NEEDED;
+  }
+  return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow);
+}
+
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
   if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
     return TSDB_CODE_FAILED;
@@ -120,6 +130,13 @@ bool fmIsNonstandardSQLFunc(int32_t funcId) {
   return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC);
 }
 
+bool fmIsSpecialDataRequiredFunc(int32_t funcId) {
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED);
+}
+
+bool fmIsDynamicScanOptimizedFunc(int32_t funcId) {
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED);
+}
 
 void fmFuncMgtDestroy() {
   void* m = gFunMgtService.pFuncNameHashTable;

source/libs/function/src/tfill.c

@@ -541,7 +541,7 @@ struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, co
     pFillCol[i].col.bytes  = pExprInfo->base.resSchema.bytes;
     pFillCol[i].col.type   = (int8_t)pExprInfo->base.resSchema.type;
     pFillCol[i].col.offset = offset;
-    pFillCol[i].col.colId  = pExprInfo->base.resSchema.colId;
+    pFillCol[i].col.colId  = pExprInfo->base.resSchema.slotId;
     pFillCol[i].tagIndex   = -2;
     pFillCol[i].flag       = pExprInfo->base.pParam[0].pCol->flag;    // always be the normal column for table query
 //    pFillCol[i].functionId = pExprInfo->pExpr->_function.functionId;

source/libs/index/src/indexFstCountingWriter.c

@@ -92,7 +92,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
     ctx->file.readOnly = readOnly;
     if (readOnly == false) {
       // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
-      ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
+      ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
       taosFtruncateFile(ctx->file.pFile, 0);
       int64_t file_size;
       taosStatFile(path, &file_size, NULL);

source/libs/monitor/src/monMain.c

@@ -375,6 +375,9 @@ static void monGenDnodeJson(SMonInfo *pMonitor) {
   tjsonAddDoubleToObject(pJson, "vnodes_num", pStat->totalVnodes);
   tjsonAddDoubleToObject(pJson, "masters", pStat->masterNum);
   tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode);
+  tjsonAddDoubleToObject(pJson, "has_qnode", pInfo->has_qnode);
+  tjsonAddDoubleToObject(pJson, "has_snode", pInfo->has_snode);
+  tjsonAddDoubleToObject(pJson, "has_bnode", pInfo->has_bnode);
 }
 
 static void monGenDiskJson(SMonInfo *pMonitor) {
@@ -530,7 +533,7 @@ void monSendReport() {
   if (pCont != NULL) {
     EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT;
     if (taosSendHttpReport(tsMonitor.cfg.server, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) {
-      uError("failed to send monitor msg since %s", terrstr());
+      uError("failed to send monitor msg");
     }
     taosMemoryFree(pCont);
   }

source/libs/monitor/src/monMsg.c

@@ -194,9 +194,9 @@ int32_t tDecodeSMonVgroupInfo(SCoder *decoder, SMonVgroupInfo *pInfo) {
     if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1;
     if (tDecodeCStrTo(decoder, desc.status) < 0) return -1;
     for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) {
-      SMonVnodeDesc vdesc = {0};
-      if (tDecodeI32(decoder, &vdesc.dnode_id) < 0) return -1;
-      if (tDecodeCStrTo(decoder, vdesc.vnode_role) < 0) return -1;
+      SMonVnodeDesc *pVDesc = &desc.vnodes[j];
+      if (tDecodeI32(decoder, &pVDesc->dnode_id) < 0) return -1;
+      if (tDecodeCStrTo(decoder, pVDesc->vnode_role) < 0) return -1;
     }
     taosArrayPush(pInfo->vgroups, &desc);
   }
@@ -205,15 +205,15 @@ int32_t tDecodeSMonVgroupInfo(SCoder *decoder, SMonVgroupInfo *pInfo) {
 
 int32_t tEncodeSMonGrantInfo(SCoder *encoder, const SMonGrantInfo *pInfo) {
   if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1;
-  if (tEncodeI32(encoder, pInfo->timeseries_used) < 0) return -1;
-  if (tEncodeI32(encoder, pInfo->timeseries_total) < 0) return -1;
+  if (tEncodeI64(encoder, pInfo->timeseries_used) < 0) return -1;
+  if (tEncodeI64(encoder, pInfo->timeseries_total) < 0) return -1;
   return 0;
 }
 
 int32_t tDecodeSMonGrantInfo(SCoder *decoder, SMonGrantInfo *pInfo) {
   if (tDecodeI32(decoder, &pInfo->expire_time) < 0) return -1;
-  if (tDecodeI32(decoder, &pInfo->timeseries_used) < 0) return -1;
-  if (tDecodeI32(decoder, &pInfo->timeseries_total) < 0) return -1;
+  if (tDecodeI64(decoder, &pInfo->timeseries_used) < 0) return -1;
+  if (tDecodeI64(decoder, &pInfo->timeseries_total) < 0) return -1;
   return 0;
 }
 

source/libs/nodes/src/nodesCodeFuncs.c

@@ -65,7 +65,7 @@ const char* nodesNodeName(ENodeType type) {
     case QUERY_NODE_TARGET:
       return "Target";
     case QUERY_NODE_DATABLOCK_DESC:
-      return "TupleDesc";
+      return "DataBlockDesc";
     case QUERY_NODE_SLOT_DESC:
       return "SlotDesc";
     case QUERY_NODE_COLUMN_DEF:
@@ -723,6 +723,9 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {
 static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag";
 static const char* jkTableScanPhysiPlanStartKey = "StartKey";
 static const char* jkTableScanPhysiPlanEndKey = "EndKey";
+static const char* jkTableScanPhysiPlanRatio = "Ratio";
+static const char* jkTableScanPhysiPlanDataRequired = "DataRequired";
+static const char* jkTableScanPhysiPlanDynamicScanFuncs = "DynamicScanFuncs";
 
 static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
@@ -737,6 +740,15 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanEndKey, pNode->scanRange.ekey);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanRatio, pNode->ratio);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkTableScanPhysiPlanDynamicScanFuncs, pNode->pDynamicScanFuncs);
+  }
 
   return code;
 }
@@ -754,6 +766,15 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanEndKey, &pNode->scanRange.ekey);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanRatio, &pNode->ratio);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkTableScanPhysiPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs);
+  }
 
   return code;
 }
@@ -1060,6 +1081,7 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) {
 
 static const char* jkWindowPhysiPlanExprs = "Exprs";
 static const char* jkWindowPhysiPlanFuncs = "Funcs";
+static const char* jkWindowPhysiPlanTsPk = "TsPk";
 
 static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
   const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj;
@@ -1071,6 +1093,9 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkWindowPhysiPlanFuncs, pNode->pFuncs);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkWindowPhysiPlanTsPk, nodeToJson, pNode->pTspk);
+  }
 
   return code;
 }
@@ -1085,6 +1110,9 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkWindowPhysiPlanFuncs, &pNode->pFuncs);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkWindowPhysiPlanTsPk, (SNode**)&pNode->pTspk);
+  }
 
   return code;
 }
@@ -1095,7 +1123,6 @@ static const char* jkIntervalPhysiPlanSliding = "Sliding";
 static const char* jkIntervalPhysiPlanIntervalUnit = "intervalUnit";
 static const char* jkIntervalPhysiPlanSlidingUnit  = "slidingUnit";
 static const char* jkIntervalPhysiPlanFill = "Fill";
-static const char* jkIntervalPhysiPlanTsPk = "TsPk";
 static const char* jkIntervalPhysiPlanPrecision = "Precision";
 
 static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) {
@@ -1120,9 +1147,6 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddObject(pJson, jkIntervalPhysiPlanFill, nodeToJson, pNode->pFill);
   }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddObject(pJson, jkIntervalPhysiPlanTsPk, nodeToJson, pNode->pTspk);
-  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanPrecision, pNode->precision);
   }
@@ -1152,9 +1176,6 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeObject(pJson, jkIntervalPhysiPlanFill, (SNode**)&pNode->pFill);
   }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = jsonToNodeObject(pJson, jkIntervalPhysiPlanTsPk, (SNode**)&pNode->pTspk);
-  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUTinyIntValue(pJson, jkIntervalPhysiPlanPrecision, &pNode->precision);
   }
@@ -2291,7 +2312,7 @@ static int32_t jsonToDataBlockDescNode(const SJson* pJson, void* pObj) {
     code = jsonToNodeList(pJson, jkDataBlockDescSlots, &pNode->pSlots);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetSmallIntValue(pJson, jkDataBlockPrecision, &pNode->precision);
+    code = tjsonGetUTinyIntValue(pJson, jkDataBlockPrecision, &pNode->precision);
   }
 
   return code;
@@ -2767,6 +2788,7 @@ int32_t nodesStringToList(const char* pStr, SNodeList** pList) {
     return TSDB_CODE_FAILED;
   }
   int32_t code = jsonToNodeListImpl(pJson, pList);
+  tjsonDelete(pJson);
   if (TSDB_CODE_SUCCESS != code) {
     nodesDestroyList(*pList);
     terrno = code;

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -77,9 +77,14 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker
     case QUERY_NODE_ORDER_BY_EXPR:
       res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext);
       break;
-    case QUERY_NODE_STATE_WINDOW:
-      res = walkNode(((SStateWindowNode*)pNode)->pExpr, order, walker, pContext);
+    case QUERY_NODE_STATE_WINDOW: {
+      SStateWindowNode* pState = (SStateWindowNode*)pNode;
+      res = walkNode(pState->pExpr, order, walker, pContext);
+      if (DEAL_RES_ERROR != res) {
+        res = walkNode(pState->pCol, order, walker, pContext);
+      }
       break;
+    }
     case QUERY_NODE_SESSION_WINDOW: {
       SSessionWindowNode* pSession = (SSessionWindowNode*)pNode;
       res = walkNode(pSession->pCol, order, walker, pContext);
@@ -211,12 +216,22 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit
     case QUERY_NODE_ORDER_BY_EXPR:
       res = rewriteNode(&(((SOrderByExprNode*)pNode)->pExpr), order, rewriter, pContext);
       break;
-    case QUERY_NODE_STATE_WINDOW:
-      res = rewriteNode(&(((SStateWindowNode*)pNode)->pExpr), order, rewriter, pContext);
+    case QUERY_NODE_STATE_WINDOW: {
+      SStateWindowNode* pState = (SStateWindowNode*)pNode;
+      res = rewriteNode(&pState->pExpr, order, rewriter, pContext);
+      if (DEAL_RES_ERROR != res) {
+        res = rewriteNode(&pState->pCol, order, rewriter, pContext);
+      }
       break;
-    case QUERY_NODE_SESSION_WINDOW:
-      res = rewriteNode(&(((SSessionWindowNode*)pNode)->pCol), order, rewriter, pContext);
+    }
+    case QUERY_NODE_SESSION_WINDOW: {
+      SSessionWindowNode* pSession = (SSessionWindowNode*)pNode;
+      res = rewriteNode(&pSession->pCol, order, rewriter, pContext);
+      if (DEAL_RES_ERROR != res) {
+        res = rewriteNode(&pSession->pGap, order, rewriter, pContext);
+      }
       break;
+    }
     case QUERY_NODE_INTERVAL_WINDOW: {
       SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pNode;
       res = rewriteNode(&(pInterval->pInterval), order, rewriter, pContext);

source/libs/parser/src/parCalcConst.c

@@ -45,7 +45,7 @@ static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) {
 
 static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) {
   SFunctionNode* pFunc = *pNode;
-  if (fmIsPseudoColumnFunc(pFunc->funcId)) {
+  if (!fmIsScalarFunc(pFunc->funcId)) {
     return DEAL_RES_CONTINUE;
   }
   SNode* pParam = NULL;
@@ -61,6 +61,7 @@ static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext)
   SLogicConditionNode* pCond = *pNode;
   SNode* pParam = NULL;
   FOREACH(pParam, pCond->pParameterList) {
+    // todo calc "true and c1 > 10"
     if (QUERY_NODE_VALUE != nodeType(pParam)) {
       return DEAL_RES_CONTINUE;
     }