Merge branch '3.0' into feature/3_liaohj

include/common/tcommon.h

@@ -46,6 +46,7 @@ typedef enum EStreamType {
   STREAM_INVALID,
   STREAM_GET_ALL,
   STREAM_DELETE,
+  STREAM_RETRIEVE,
 } EStreamType;
 
 typedef struct {

include/common/tmsg.h

@@ -1205,6 +1205,13 @@ typedef struct {
   char*   data;
 } STagData;
 
+typedef struct {
+  int32_t useless;  // useless
+} SShowVariablesReq;
+
+int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
+int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq);
+
 /*
  * sql: show tables like '%a_%'
  * payload is the query condition, e.g., '%a_%'

include/common/tmsgdef.h

@@ -159,6 +159,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_MERGE_VGROUP, "merge-vgroup", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_REDISTRIBUTE_VGROUP, "redistribute-vgroup", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SPLIT_VGROUP, "split-vgroup", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_SHOW_VARIABLES, "show-variables", NULL, NULL)
 
   TD_NEW_MSG_SEG(TDMT_VND_MSG)
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
@@ -200,15 +201,14 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "drop-ttl-stb", NULL, NULL)
-  TD_NEW_MSG_SEG(TDMT_QND_MSG)
 
-  //shared by snode and vnode
   TD_NEW_MSG_SEG(TDMT_STREAM_MSG)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
 
   TD_NEW_MSG_SEG(TDMT_SCH_MSG)
   TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)

include/common/ttokendef.h

@@ -49,214 +49,216 @@
 #define TK_CONNS         31
 #define TK_STATE         32
 #define TK_USER          33
-#define TK_PRIVILEGE     34
-#define TK_DROP          35
-#define TK_GRANT         36
-#define TK_ON            37
-#define TK_TO            38
-#define TK_REVOKE        39
-#define TK_FROM          40
-#define TK_NK_COMMA      41
-#define TK_READ          42
-#define TK_WRITE         43
-#define TK_NK_DOT        44
-#define TK_DNODE         45
-#define TK_PORT          46
-#define TK_NK_INTEGER    47
-#define TK_DNODES        48
-#define TK_NK_IPTOKEN    49
-#define TK_LOCAL         50
-#define TK_QNODE         51
-#define TK_BNODE         52
-#define TK_SNODE         53
-#define TK_MNODE         54
-#define TK_DATABASE      55
-#define TK_USE           56
-#define TK_IF            57
-#define TK_NOT           58
-#define TK_EXISTS        59
-#define TK_BUFFER        60
-#define TK_CACHELAST     61
-#define TK_COMP          62
-#define TK_DURATION      63
-#define TK_NK_VARIABLE   64
-#define TK_FSYNC         65
-#define TK_MAXROWS       66
-#define TK_MINROWS       67
-#define TK_KEEP          68
-#define TK_PAGES         69
-#define TK_PAGESIZE      70
-#define TK_PRECISION     71
-#define TK_REPLICA       72
-#define TK_STRICT        73
-#define TK_WAL           74
-#define TK_VGROUPS       75
-#define TK_SINGLE_STABLE 76
-#define TK_RETENTIONS    77
-#define TK_SCHEMALESS    78
-#define TK_NK_COLON      79
-#define TK_TABLE         80
-#define TK_NK_LP         81
-#define TK_NK_RP         82
-#define TK_STABLE        83
-#define TK_ADD           84
-#define TK_COLUMN        85
-#define TK_MODIFY        86
-#define TK_RENAME        87
-#define TK_TAG           88
-#define TK_SET           89
-#define TK_NK_EQ         90
-#define TK_USING         91
-#define TK_TAGS          92
-#define TK_COMMENT       93
-#define TK_BOOL          94
-#define TK_TINYINT       95
-#define TK_SMALLINT      96
-#define TK_INT           97
-#define TK_INTEGER       98
-#define TK_BIGINT        99
-#define TK_FLOAT         100
-#define TK_DOUBLE        101
-#define TK_BINARY        102
-#define TK_TIMESTAMP     103
-#define TK_NCHAR         104
-#define TK_UNSIGNED      105
-#define TK_JSON          106
-#define TK_VARCHAR       107
-#define TK_MEDIUMBLOB    108
-#define TK_BLOB          109
-#define TK_VARBINARY     110
-#define TK_DECIMAL       111
-#define TK_MAX_DELAY     112
-#define TK_WATERMARK     113
-#define TK_ROLLUP        114
-#define TK_TTL           115
-#define TK_SMA           116
-#define TK_FIRST         117
-#define TK_LAST          118
-#define TK_SHOW          119
-#define TK_DATABASES     120
-#define TK_TABLES        121
-#define TK_STABLES       122
-#define TK_MNODES        123
-#define TK_MODULES       124
-#define TK_QNODES        125
-#define TK_FUNCTIONS     126
-#define TK_INDEXES       127
-#define TK_ACCOUNTS      128
-#define TK_APPS          129
-#define TK_CONNECTIONS   130
-#define TK_LICENCE       131
-#define TK_GRANTS        132
-#define TK_QUERIES       133
-#define TK_SCORES        134
-#define TK_TOPICS        135
-#define TK_VARIABLES     136
-#define TK_BNODES        137
-#define TK_SNODES        138
-#define TK_CLUSTER       139
-#define TK_TRANSACTIONS  140
-#define TK_DISTRIBUTED   141
-#define TK_LIKE          142
-#define TK_INDEX         143
-#define TK_FULLTEXT      144
+#define TK_ENABLE        34
+#define TK_NK_INTEGER    35
+#define TK_SYSINFO       36
+#define TK_DROP          37
+#define TK_GRANT         38
+#define TK_ON            39
+#define TK_TO            40
+#define TK_REVOKE        41
+#define TK_FROM          42
+#define TK_NK_COMMA      43
+#define TK_READ          44
+#define TK_WRITE         45
+#define TK_NK_DOT        46
+#define TK_DNODE         47
+#define TK_PORT          48
+#define TK_DNODES        49
+#define TK_NK_IPTOKEN    50
+#define TK_LOCAL         51
+#define TK_QNODE         52
+#define TK_BNODE         53
+#define TK_SNODE         54
+#define TK_MNODE         55
+#define TK_DATABASE      56
+#define TK_USE           57
+#define TK_IF            58
+#define TK_NOT           59
+#define TK_EXISTS        60
+#define TK_BUFFER        61
+#define TK_CACHELAST     62
+#define TK_COMP          63
+#define TK_DURATION      64
+#define TK_NK_VARIABLE   65
+#define TK_FSYNC         66
+#define TK_MAXROWS       67
+#define TK_MINROWS       68
+#define TK_KEEP          69
+#define TK_PAGES         70
+#define TK_PAGESIZE      71
+#define TK_PRECISION     72
+#define TK_REPLICA       73
+#define TK_STRICT        74
+#define TK_WAL           75
+#define TK_VGROUPS       76
+#define TK_SINGLE_STABLE 77
+#define TK_RETENTIONS    78
+#define TK_SCHEMALESS    79
+#define TK_NK_COLON      80
+#define TK_TABLE         81
+#define TK_NK_LP         82
+#define TK_NK_RP         83
+#define TK_STABLE        84
+#define TK_ADD           85
+#define TK_COLUMN        86
+#define TK_MODIFY        87
+#define TK_RENAME        88
+#define TK_TAG           89
+#define TK_SET           90
+#define TK_NK_EQ         91
+#define TK_USING         92
+#define TK_TAGS          93
+#define TK_COMMENT       94
+#define TK_BOOL          95
+#define TK_TINYINT       96
+#define TK_SMALLINT      97
+#define TK_INT           98
+#define TK_INTEGER       99
+#define TK_BIGINT        100
+#define TK_FLOAT         101
+#define TK_DOUBLE        102
+#define TK_BINARY        103
+#define TK_TIMESTAMP     104
+#define TK_NCHAR         105
+#define TK_UNSIGNED      106
+#define TK_JSON          107
+#define TK_VARCHAR       108
+#define TK_MEDIUMBLOB    109
+#define TK_BLOB          110
+#define TK_VARBINARY     111
+#define TK_DECIMAL       112
+#define TK_MAX_DELAY     113
+#define TK_WATERMARK     114
+#define TK_ROLLUP        115
+#define TK_TTL           116
+#define TK_SMA           117
+#define TK_FIRST         118
+#define TK_LAST          119
+#define TK_SHOW          120
+#define TK_DATABASES     121
+#define TK_TABLES        122
+#define TK_STABLES       123
+#define TK_MNODES        124
+#define TK_MODULES       125
+#define TK_QNODES        126
+#define TK_FUNCTIONS     127
+#define TK_INDEXES       128
+#define TK_ACCOUNTS      129
+#define TK_APPS          130
+#define TK_CONNECTIONS   131
+#define TK_LICENCE       132
+#define TK_GRANTS        133
+#define TK_QUERIES       134
+#define TK_SCORES        135
+#define TK_TOPICS        136
+#define TK_VARIABLES     137
+#define TK_BNODES        138
+#define TK_SNODES        139
+#define TK_CLUSTER       140
+#define TK_TRANSACTIONS  141
+#define TK_DISTRIBUTED   142
+#define TK_LIKE          143
+#define TK_INDEX         144
 #define TK_FUNCTION      145
 #define TK_INTERVAL      146
 #define TK_TOPIC         147
 #define TK_AS            148
-#define TK_CONSUMER      149
-#define TK_GROUP         150
-#define TK_DESC          151
-#define TK_DESCRIBE      152
-#define TK_RESET         153
-#define TK_QUERY         154
-#define TK_CACHE         155
-#define TK_EXPLAIN       156
-#define TK_ANALYZE       157
-#define TK_VERBOSE       158
-#define TK_NK_BOOL       159
-#define TK_RATIO         160
-#define TK_NK_FLOAT      161
-#define TK_COMPACT       162
-#define TK_VNODES        163
-#define TK_IN            164
-#define TK_OUTPUTTYPE    165
-#define TK_AGGREGATE     166
-#define TK_BUFSIZE       167
-#define TK_STREAM        168
-#define TK_INTO          169
-#define TK_TRIGGER       170
-#define TK_AT_ONCE       171
-#define TK_WINDOW_CLOSE  172
-#define TK_KILL          173
-#define TK_CONNECTION    174
-#define TK_TRANSACTION   175
-#define TK_BALANCE       176
-#define TK_VGROUP        177
-#define TK_MERGE         178
-#define TK_REDISTRIBUTE  179
-#define TK_SPLIT         180
-#define TK_SYNCDB        181
-#define TK_DELETE        182
-#define TK_NULL          183
-#define TK_NK_QUESTION   184
-#define TK_NK_ARROW      185
-#define TK_ROWTS         186
-#define TK_TBNAME        187
-#define TK_QSTARTTS      188
-#define TK_QENDTS        189
-#define TK_WSTARTTS      190
-#define TK_WENDTS        191
-#define TK_WDURATION     192
-#define TK_CAST          193
-#define TK_NOW           194
-#define TK_TODAY         195
-#define TK_TIMEZONE      196
-#define TK_COUNT         197
-#define TK_LAST_ROW      198
-#define TK_BETWEEN       199
-#define TK_IS            200
-#define TK_NK_LT         201
-#define TK_NK_GT         202
-#define TK_NK_LE         203
-#define TK_NK_GE         204
-#define TK_NK_NE         205
-#define TK_MATCH         206
-#define TK_NMATCH        207
-#define TK_CONTAINS      208
-#define TK_JOIN          209
-#define TK_INNER         210
-#define TK_SELECT        211
-#define TK_DISTINCT      212
-#define TK_WHERE         213
-#define TK_PARTITION     214
-#define TK_BY            215
-#define TK_SESSION       216
-#define TK_STATE_WINDOW  217
-#define TK_SLIDING       218
-#define TK_FILL          219
-#define TK_VALUE         220
-#define TK_NONE          221
-#define TK_PREV          222
-#define TK_LINEAR        223
-#define TK_NEXT          224
-#define TK_HAVING        225
-#define TK_RANGE         226
-#define TK_EVERY         227
-#define TK_ORDER         228
-#define TK_SLIMIT        229
-#define TK_SOFFSET       230
-#define TK_LIMIT         231
-#define TK_OFFSET        232
-#define TK_ASC           233
-#define TK_NULLS         234
-#define TK_ID            235
-#define TK_NK_BITNOT     236
-#define TK_INSERT        237
-#define TK_VALUES        238
-#define TK_IMPORT        239
-#define TK_NK_SEMI       240
-#define TK_FILE          241
+#define TK_WITH          149
+#define TK_META          150
+#define TK_CONSUMER      151
+#define TK_GROUP         152
+#define TK_DESC          153
+#define TK_DESCRIBE      154
+#define TK_RESET         155
+#define TK_QUERY         156
+#define TK_CACHE         157
+#define TK_EXPLAIN       158
+#define TK_ANALYZE       159
+#define TK_VERBOSE       160
+#define TK_NK_BOOL       161
+#define TK_RATIO         162
+#define TK_NK_FLOAT      163
+#define TK_COMPACT       164
+#define TK_VNODES        165
+#define TK_IN            166
+#define TK_OUTPUTTYPE    167
+#define TK_AGGREGATE     168
+#define TK_BUFSIZE       169
+#define TK_STREAM        170
+#define TK_INTO          171
+#define TK_TRIGGER       172
+#define TK_AT_ONCE       173
+#define TK_WINDOW_CLOSE  174
+#define TK_KILL          175
+#define TK_CONNECTION    176
+#define TK_TRANSACTION   177
+#define TK_BALANCE       178
+#define TK_VGROUP        179
+#define TK_MERGE         180
+#define TK_REDISTRIBUTE  181
+#define TK_SPLIT         182
+#define TK_SYNCDB        183
+#define TK_DELETE        184
+#define TK_NULL          185
+#define TK_NK_QUESTION   186
+#define TK_NK_ARROW      187
+#define TK_ROWTS         188
+#define TK_TBNAME        189
+#define TK_QSTARTTS      190
+#define TK_QENDTS        191
+#define TK_WSTARTTS      192
+#define TK_WENDTS        193
+#define TK_WDURATION     194
+#define TK_CAST          195
+#define TK_NOW           196
+#define TK_TODAY         197
+#define TK_TIMEZONE      198
+#define TK_COUNT         199
+#define TK_LAST_ROW      200
+#define TK_BETWEEN       201
+#define TK_IS            202
+#define TK_NK_LT         203
+#define TK_NK_GT         204
+#define TK_NK_LE         205
+#define TK_NK_GE         206
+#define TK_NK_NE         207
+#define TK_MATCH         208
+#define TK_NMATCH        209
+#define TK_CONTAINS      210
+#define TK_JOIN          211
+#define TK_INNER         212
+#define TK_SELECT        213
+#define TK_DISTINCT      214
+#define TK_WHERE         215
+#define TK_PARTITION     216
+#define TK_BY            217
+#define TK_SESSION       218
+#define TK_STATE_WINDOW  219
+#define TK_SLIDING       220
+#define TK_FILL          221
+#define TK_VALUE         222
+#define TK_NONE          223
+#define TK_PREV          224
+#define TK_LINEAR        225
+#define TK_NEXT          226
+#define TK_HAVING        227
+#define TK_RANGE         228
+#define TK_EVERY         229
+#define TK_ORDER         230
+#define TK_SLIMIT        231
+#define TK_SOFFSET       232
+#define TK_LIMIT         233
+#define TK_OFFSET        234
+#define TK_ASC           235
+#define TK_NULLS         236
+#define TK_ID            237
+#define TK_NK_BITNOT     238
+#define TK_INSERT        239
+#define TK_VALUES        240
+#define TK_IMPORT        241
+#define TK_NK_SEMI       242
+#define TK_FILE          243
 
 #define TK_NK_SPACE   300
 #define TK_NK_COMMENT 301

include/libs/nodes/cmdnodes.h

@@ -179,13 +179,16 @@ typedef struct SCreateUserStmt {
   ENodeType type;
   char      useName[TSDB_USER_LEN];
   char      password[TSDB_USET_PASSWORD_LEN];
+  int8_t    sysinfo;
 } SCreateUserStmt;
 
 typedef struct SAlterUserStmt {
   ENodeType type;
   char      useName[TSDB_USER_LEN];
-  char      password[TSDB_USET_PASSWORD_LEN];
   int8_t    alterType;
+  char      password[TSDB_USET_PASSWORD_LEN];
+  int8_t    enable;
+  int8_t    sysinfo;
 } SAlterUserStmt;
 
 typedef struct SDropUserStmt {
@@ -268,7 +271,6 @@ typedef struct SDropIndexStmt {
   ENodeType type;
   bool      ignoreNotExists;
   char      indexName[TSDB_INDEX_NAME_LEN];
-  char      tableName[TSDB_TABLE_NAME_LEN];
 } SDropIndexStmt;
 
 typedef struct SCreateComponentNodeStmt {
@@ -287,6 +289,7 @@ typedef struct SCreateTopicStmt {
   char      subDbName[TSDB_DB_NAME_LEN];
   char      subSTbName[TSDB_TABLE_NAME_LEN];
   bool      ignoreExists;
+  bool      withMeta;
   SNode*    pQuery;
 } SCreateTopicStmt;
 

include/libs/nodes/querynodes.h

@@ -385,6 +385,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp);
 bool nodesIsComparisonOp(const SOperatorNode* pOp);
 bool nodesIsJsonOp(const SOperatorNode* pOp);
 bool nodesIsRegularOp(const SOperatorNode* pOp);
+bool nodesIsBitwiseOp(const SOperatorNode* pOp);
 
 bool nodesExprHasColumn(SNode* pNode);
 bool nodesExprsHasColumn(SNodeList* pList);

include/libs/stream/tstream.h

@@ -77,7 +77,7 @@ typedef struct {
 typedef struct {
   int8_t type;
 
-  int32_t sourceVg;
+  int32_t srcVgId;
   int64_t sourceVer;
 
   SArray* blocks;  // SArray<SSDataBlock*>
@@ -145,11 +145,6 @@ void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit);
 
 SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit);
 
-#if 0
-int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
-void*   streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
-#endif
-
 typedef struct {
   char* qmsg;
   // followings are not applicable to encoder and decoder
@@ -234,6 +229,13 @@ enum {
   TASK_TRIGGER_STATUS__ACTIVE,
 };
 
+typedef struct {
+  int32_t nodeId;
+  int32_t childId;
+  int32_t taskId;
+  SEpSet  epSet;
+} SStreamChildEpInfo;
+
 struct SStreamTask {
   int64_t streamId;
   int32_t taskId;
@@ -247,13 +249,16 @@ struct SStreamTask {
   int8_t  dispatchType;
   int16_t dispatchMsgType;
 
-  int8_t dataScan;
+  int8_t isDataScan;
 
   // node info
-  int32_t childId;
+  int32_t selfChildId;
   int32_t nodeId;
   SEpSet  epSet;
 
+  // children info
+  SArray* childEpInfo;  // SArray<SStreamChildEpInfo*>
+
   // exec
   STaskExec exec;
 
@@ -291,6 +296,9 @@ struct SStreamTask {
   SMsgCb* pMsgCb;
 };
 
+int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
+int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
+
 SStreamTask* tNewSStreamTask(int64_t streamId);
 int32_t      tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
 int32_t      tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
@@ -369,9 +377,9 @@ typedef struct {
 typedef struct {
   int64_t streamId;
   int32_t taskId;
-  int32_t sourceTaskId;
-  int32_t sourceVg;
-  int32_t sourceChildId;
+  int32_t dataSrcVgId;
+  int32_t upstreamTaskId;
+  int32_t upstreamChildId;
   int32_t upstreamNodeId;
 #if 0
   int64_t sourceVer;
@@ -387,6 +395,23 @@ typedef struct {
   int8_t  inputStatus;
 } SStreamDispatchRsp;
 
+typedef struct {
+  int64_t            streamId;
+  int32_t            srcTaskId;
+  int32_t            srcNodeId;
+  int32_t            dstTaskId;
+  int32_t            dstNodeId;
+  int32_t            retrieveLen;
+  SRetrieveTableRsp* pRetrieve;
+} SStreamRetrieveReq;
+
+typedef struct {
+  int64_t streamId;
+  int32_t childId;
+  int32_t rspFromTaskId;
+  int32_t rspToTaskId;
+} SStreamRetrieveRsp;
+
 typedef struct {
   int64_t streamId;
   int32_t taskId;
@@ -401,6 +426,7 @@ typedef struct {
 } SStreamTaskRecoverRsp;
 
 int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq);
+int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq);
 
 int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId);
 int32_t streamSetupTrigger(SStreamTask* pTask);
@@ -411,6 +437,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp);
 int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
 int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
 
+int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
+int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/wal/wal.h

@@ -195,7 +195,6 @@ void            walCloseReadHandle(SWalReadHandle *);
 int32_t         walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
 
 // only for tq usage
-// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
 void    walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
 int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead);
 int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead);
@@ -211,13 +210,8 @@ void     walCloseRef(SWalRef *);
 int32_t  walRefVer(SWalRef *, int64_t ver);
 int32_t  walUnrefVer(SWal *);
 
-// deprecated
-#if 0
-int32_t walRead(SWal *, SWalHead **, int64_t ver);
-int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
-#endif
-
 // lifecycle check
+bool    walIsEmpty(SWal *);
 int64_t walGetFirstVer(SWal *);
 int64_t walGetSnapshotVer(SWal *);
 int64_t walGetLastVer(SWal *);

include/util/tdef.h

@@ -134,7 +134,7 @@ typedef enum EOperatorType {
   OP_TYPE_MINUS,
   OP_TYPE_ASSIGN,
 
-  // bit operator
+  // bitwise operator
   OP_TYPE_BIT_AND,
   OP_TYPE_BIT_OR,
 

source/common/src/tmsg.c

@@ -2733,6 +2733,31 @@ void tFreeSTableIndexInfo(void *info) {
   taosMemoryFree(pInfo->expr);
 }
 
+int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->useless) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+}
+
 int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);

source/dnode/mgmt/mgmt_snode/src/smHandle.c

@@ -101,6 +101,8 @@ SArray *smGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
 
   code = 0;
 _OVER:

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -359,6 +359,8 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mnode/impl/src/mndScheduler.c

@@ -37,7 +37,7 @@ extern bool tsSchedStreamToSnode;
 
 static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) {
   int32_t childId = taosArrayGetSize(pArray);
-  pTask->childId = childId;
+  pTask->selfChildId = childId;
   taosArrayPush(pArray, &pTask);
   return 0;
 }
@@ -270,6 +270,8 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb
     pTask->nodeId = pVgroup->vgId;
     pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
+    pTask->isDataScan = 0;
+
     // source
     pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
 
@@ -306,6 +308,8 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj*
   }
   mndAddTaskToTaskSet(tasks, pTask);
 
+  ASSERT(pStream->fixedSinkVg.vgId == pStream->fixedSinkVgId);
+
   pTask->nodeId = pStream->fixedSinkVgId;
 #if 0
   SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId);
@@ -315,6 +319,9 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj*
   pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
 #endif
   pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg);
+
+  pTask->isDataScan = 0;
+
   // source
   pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
 
@@ -384,6 +391,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
 
       pInnerTask = tNewSStreamTask(pStream->uid);
       mndAddTaskToTaskSet(taskInnerLevel, pInnerTask);
+
+      pInnerTask->isDataScan = 0;
+
+      pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*));
+
       // input
       pInnerTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
 
@@ -446,7 +458,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       SStreamTask* pTask = tNewSStreamTask(pStream->uid);
       mndAddTaskToTaskSet(taskSourceLevel, pTask);
 
-      pTask->dataScan = 1;
+      pTask->isDataScan = 1;
 
       // input
       pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
@@ -467,6 +479,20 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
         qDestroyQueryPlan(pPlan);
         return -1;
       }
+
+      SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo));
+      if (pEpInfo == NULL) {
+        ASSERT(0);
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        sdbRelease(pSdb, pVgroup);
+        qDestroyQueryPlan(pPlan);
+        return -1;
+      }
+      pEpInfo->childId = pTask->selfChildId;
+      pEpInfo->epSet = pTask->epSet;
+      pEpInfo->nodeId = pTask->nodeId;
+      pEpInfo->taskId = pTask->taskId;
+      taosArrayPush(pInnerTask->childEpInfo, &pEpInfo);
     }
   }
 
@@ -491,7 +517,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       SStreamTask* pTask = tNewSStreamTask(pStream->uid);
       mndAddTaskToTaskSet(taskOneLevel, pTask);
 
-      pTask->dataScan = 1;
+      pTask->isDataScan = 1;
 
       // input
       pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;

source/dnode/snode/src/snode.c

@@ -105,13 +105,14 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) {
 
   ASSERT(pTask->execType != TASK_EXEC__NONE);
 
-  ASSERT(pTask->dataScan == 0);
+  ASSERT(pTask->isDataScan == 0);
   pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
   ASSERT(pTask->exec.executor);
 
   streamSetupTrigger(pTask);
 
-  qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId, pTask->childId);
+  qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId,
+        pTask->selfChildId);
 
   taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void *));
 
@@ -198,6 +199,34 @@ static int32_t sndProcessTaskDropReq(SSnode *pNode, SRpcMsg *pMsg) {
   return code;
 }
 
+static int32_t sndProcessTaskRetrieveReq(SSnode *pNode, SRpcMsg *pMsg) {
+  SStreamMeta *pMeta = pNode->pMeta;
+
+  char              *msgStr = pMsg->pCont;
+  char              *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
+  int32_t            msgLen = pMsg->contLen - sizeof(SMsgHead);
+  SStreamRetrieveReq req;
+  SDecoder           decoder;
+  tDecoderInit(&decoder, msgBody, msgLen);
+  tDecodeStreamRetrieveReq(&decoder, &req);
+  int32_t      taskId = req.dstTaskId;
+  SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t));
+  if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
+    return 0;
+  }
+  SRpcMsg rsp = {
+      .info = pMsg->info,
+      .code = 0,
+  };
+  streamProcessRetrieveReq(pTask, &req, &rsp);
+  return 0;
+}
+
+static int32_t sndProcessTaskRetrieveRsp(SSnode *pNode, SRpcMsg *pMsg) {
+  //
+  return 0;
+}
+
 int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) {
   // stream deploy
   // stream stop/resume
@@ -221,10 +250,14 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
       return sndProcessTaskDispatchReq(pSnode, pMsg);
     case TDMT_STREAM_TASK_RECOVER:
       return sndProcessTaskRecoverReq(pSnode, pMsg);
+    case TDMT_STREAM_RETRIEVE:
+      return sndProcessTaskRecoverReq(pSnode, pMsg);
     case TDMT_STREAM_TASK_DISPATCH_RSP:
       return sndProcessTaskDispatchRsp(pSnode, pMsg);
     case TDMT_STREAM_TASK_RECOVER_RSP:
       return sndProcessTaskRecoverRsp(pSnode, pMsg);
+    case TDMT_STREAM_RETRIEVE_RSP:
+      return sndProcessTaskRecoverRsp(pSnode, pMsg);
     default:
       ASSERT(0);
   }

source/dnode/vnode/inc/vnode.h

@@ -195,6 +195,7 @@ struct SVnodeCfg {
 typedef struct {
   TSKEY    lastKey;
   uint64_t uid;
+  uint64_t groupId;
 } STableKeyInfo;
 
 struct SMetaEntry {

source/dnode/vnode/src/inc/vnodeInt.h

@@ -149,6 +149,8 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
 
 SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid,
                             const char* stbFullName, int32_t vgId);

source/dnode/vnode/src/tq/tq.c

@@ -441,7 +441,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
   // exec
   if (pTask->execType != TASK_EXEC__NONE) {
     // expand runners
-    if (pTask->dataScan) {
+    if (pTask->isDataScan) {
       STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
       SReadHandle    handle = {
              .reader = pStreamReader,
@@ -476,7 +476,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
 
   streamSetupTrigger(pTask);
 
-  tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->childId, TD_VID(pTq->pVnode));
+  tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode));
 
   taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*));
 
@@ -616,3 +616,29 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
   return code;
 #endif
 }
+
+int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) {
+  char*              msgStr = pMsg->pCont;
+  char*              msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
+  int32_t            msgLen = pMsg->contLen - sizeof(SMsgHead);
+  SStreamRetrieveReq req;
+  SDecoder           decoder;
+  tDecoderInit(&decoder, msgBody, msgLen);
+  tDecodeStreamRetrieveReq(&decoder, &req);
+  int32_t      taskId = req.dstTaskId;
+  SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
+  if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
+    return 0;
+  }
+  SRpcMsg rsp = {
+      .info = pMsg->info,
+      .code = 0,
+  };
+  streamProcessRetrieveReq(pTask, &req, &rsp);
+  return 0;
+}
+
+int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) {
+  //
+  return 0;
+}

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2842,7 +2842,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
       break;
     }
 
-    STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id};
+    STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0};
     taosArrayPush(list, &info);
   }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -265,10 +265,14 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
       return tqProcessTaskDispatchReq(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_RECOVER:
       return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);
+    case TDMT_STREAM_RETRIEVE:
+      return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_DISPATCH_RSP:
       return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_RECOVER_RSP:
       return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);
+    case TDMT_STREAM_RETRIEVE_RSP:
+      return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg);
     default:
       vError("unknown msg type:%d in fetch queue", pMsg->msgType);
       return TSDB_CODE_VND_APP_ERROR;

source/libs/executor/inc/executorimpl.h

@@ -838,14 +838,19 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi
                                        SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset,
                                        SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
 
-int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
+int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
+                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId,
+                                SNode* pTagCond);
+int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
                                   STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
-                                  uint64_t taskId, SNode* pTagCond);
+                                  uint64_t taskId);
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SArray* dataReaders,
                                                 SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
 
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
 
+int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/executor/src/executil.c

@@ -229,7 +229,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
       }
 
       for (int i = 0; i < taosArrayGetSize(res); i++) {
-        STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)};
+        STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
         taosArrayPush(pListInfo->pTableList, &info);
       }
       taosArrayDestroy(res);
@@ -237,7 +237,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
       code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
     }
   } else {  // Create one table group.
-    STableKeyInfo info = {.lastKey = 0, .uid = tableUid};
+    STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
     taosArrayPush(pListInfo->pTableList, &info);
   }
 

source/libs/executor/src/executorimpl.c

@@ -3955,14 +3955,16 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
         }
       }
     }
-
     int32_t   len = (int32_t)(pStart - (char*)keyBuf);
-    uint64_t* groupId = taosHashGet(pTableListInfo->map, keyBuf, len);
-    if (groupId) {
-      taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), groupId, sizeof(uint64_t));
-    } else {
+
+    uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len);
+
+    if (!pGroupId) {
       uint64_t tmpId = calcGroupId(keyBuf, len);
+      info->groupId = tmpId;
       taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t));
+    } else {
+      info->groupId = *pGroupId;
     }
 
     metaReaderClear(&mr);
@@ -4010,11 +4012,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
 
       SArray* dataReaders = taosArrayInit(8, POINTER_BYTES);
-      createMultipleDataReaders(pTableScanNode, pHandle, pTableListInfo, dataReaders, queryId, taskId, pTagCond);
+      createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId, pTagCond);
+      doCreateMultipleDataReaders(pTableScanNode, pHandle, pTableListInfo, dataReaders, queryId, taskId);
+
       extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
-      SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
-      generateGroupIdMap(pTableListInfo, pHandle, groupKeys);  // todo for json
-      taosArrayDestroy(groupKeys);
+
       SOperatorInfo*  pOperator = createTableMergeScanOperatorInfo(pTableScanNode, dataReaders, pHandle, pTaskInfo);
       STableScanInfo* pScanInfo = pOperator->info;
       pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
@@ -4084,7 +4086,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
           return NULL;
         }
       } else {  // Create one table group.
-        STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid};
+        STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0};
         taosArrayPush(pTableListInfo->pTableList, &info);
       }
 

source/libs/executor/src/scanoperator.c

@@ -595,12 +595,12 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo*
   pInfo->dataReader = pReadHandle;
   //  pInfo->prevGroupId       = -1;
 
-  pOperator->name         = "TableSeqScanOperator";
+  pOperator->name = "TableSeqScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN;
-  pOperator->blocking     = false;
-  pOperator->status       = OP_NOT_OPENED;
-  pOperator->info         = pInfo;
-  pOperator->pTaskInfo    = pTaskInfo;
+  pOperator->blocking = false;
+  pOperator->status = OP_NOT_OPENED;
+  pOperator->info = pInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
   pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL, NULL, NULL);
   return pOperator;
@@ -614,7 +614,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid) {
   metaGetTableEntryByUid(&mr, uid);
   if (mr.me.type == TSDB_SUPER_TABLE) {
     int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols;
-    for(int32_t i = 0; i < numOfCols; ++i) {
+    for (int32_t i = 0; i < numOfCols; ++i) {
       rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes;
     }
   } else if (mr.me.type == TSDB_CHILD_TABLE) {
@@ -622,12 +622,12 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid) {
     metaGetTableEntryByUid(&mr, suid);
     int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols;
 
-    for(int32_t i = 0; i < numOfCols; ++i) {
+    for (int32_t i = 0; i < numOfCols; ++i) {
       rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes;
     }
   } else if (mr.me.type == TSDB_NORMAL_TABLE) {
     int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols;
-    for(int32_t i = 0; i < numOfCols; ++i) {
+    for (int32_t i = 0; i < numOfCols; ++i) {
       rowLen += mr.me.ntbEntry.schemaRow.pSchema[i].bytes;
     }
   }
@@ -651,7 +651,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
 
   SSDataBlock* pBlock = pBlockScanInfo->pResBlock;
 
-  int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId;
+  int32_t          slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId;
   SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId);
 
   int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo);
@@ -683,23 +683,23 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* re
     goto _error;
   }
 
-  pInfo->pHandle    = dataReader;
+  pInfo->pHandle = dataReader;
   pInfo->readHandle = *readHandle;
-  pInfo->uid        = uid;
-  pInfo->pResBlock  = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc);
+  pInfo->uid = uid;
+  pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc);
 
-  int32_t numOfCols = 0;
+  int32_t    numOfCols = 0;
   SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
-  int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+  int32_t    code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
-  pOperator->name      = "DataBlockDistScanOperator";
+  pOperator->name = "DataBlockDistScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN;
-  pOperator->blocking  = false;
-  pOperator->status    = OP_NOT_OPENED;
-  pOperator->info      = pInfo;
+  pOperator->blocking = false;
+  pOperator->status = OP_NOT_OPENED;
+  pOperator->info = pInfo;
   pOperator->pTaskInfo = pTaskInfo;
 
   pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL,
@@ -1866,26 +1866,25 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs);
   SArray*    colList = extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
 
-
   int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
-  pInfo->pTableList       = pTableListInfo;
-  pInfo->pColMatchInfo    = colList;
-  pInfo->pRes             = createResDataBlock(pDescNode);
-  pInfo->readHandle       = *pReadHandle;
-  pInfo->curPos           = 0;
-  pInfo->pFilterNode      = pPhyNode->node.pConditions;
+  pInfo->pTableList = pTableListInfo;
+  pInfo->pColMatchInfo = colList;
+  pInfo->pRes = createResDataBlock(pDescNode);
+  pInfo->readHandle = *pReadHandle;
+  pInfo->curPos = 0;
+  pInfo->pFilterNode = pPhyNode->node.pConditions;
 
   pOperator->name = "TagScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
 
-  pOperator->blocking     = false;
-  pOperator->status       = OP_NOT_OPENED;
-  pOperator->info         = pInfo;
-  pOperator->pTaskInfo    = pTaskInfo;
+  pOperator->blocking = false;
+  pOperator->status = OP_NOT_OPENED;
+  pOperator->info = pInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
   initResultSizeInfo(pOperator, 4096);
   blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
@@ -1951,25 +1950,42 @@ typedef struct STableMergeScanInfo {
 
 } STableMergeScanInfo;
 
-int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                  STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
-                                  uint64_t taskId, SNode* pTagCond) {
+int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
+  const STableKeyInfo* info1 = p1;
+  const STableKeyInfo* info2 = p2;
+  return info1->groupId - info2->groupId;
+}
+
+int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
+                                STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId,
+                                SNode* pTagCond) {
   int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo, pTagCond);
   if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
+    return code;
   }
 
   if (taosArrayGetSize(pTableListInfo->pTableList) == 0) {
     qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
-    goto _error;
+    return TSDB_CODE_SUCCESS;
+  }
+  SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
+  generateGroupIdMap(pTableListInfo, pHandle, groupKeys);  // todo for json
+  if (groupKeys) {
+    taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
   }
+  taosArrayDestroy(groupKeys);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
+                                  STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
+                                  uint64_t taskId) {
 
   SQueryTableDataCond cond = {0};
-  code = initQueryTableDataCond(&cond, pTableScanNode);
+  int32_t code = initQueryTableDataCond(&cond, pTableScanNode);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
-  // TODO: free the sublist info and the table list in it
   for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); ++i) {
     STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo));
     subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
@@ -1983,7 +1999,7 @@ int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHand
   }
   cleanupQueryTableDataCond(&cond);
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 
 _error:
   return code;

source/libs/nodes/src/nodesUtilFuncs.c

@@ -1357,6 +1357,17 @@ bool nodesIsRegularOp(const SOperatorNode* pOp) {
   return false;
 }
 
+bool nodesIsBitwiseOp(const SOperatorNode* pOp) {
+  switch (pOp->opType) {
+    case OP_TYPE_BIT_AND:
+    case OP_TYPE_BIT_OR:
+      return true;
+    default:
+      break;
+  }
+  return false;
+}
+
 typedef struct SCollectColumnsCxt {
   int32_t         errCode;
   const char*     pTableAlias;

source/libs/parser/inc/parAst.h

@@ -162,7 +162,7 @@ SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
 SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);
 SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable);
 SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId);
-SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword);
+SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo);
 SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal);
 SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
 SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
@@ -171,12 +171,15 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
 SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName,
                              SToken* pTableName, SNodeList* pCols, SNode* pOptions);
 SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding);
-SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName);
+SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName);
 SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId);
 SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId);
-SNode* createTopicOptions(SAstCreateContext* pCxt);
-SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery,
-                             const SToken* pSubDbName, SNode* pRealTable);
+SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
+                                     SNode* pQuery);
+SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
+                                  const SToken* pSubDbName, bool withMeta);
+SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
+                                     SNode* pRealTable, bool withMeta);
 SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName);
 SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId,
                             const SToken* pTopicName);

source/libs/parser/inc/sql.y

@@ -80,11 +80,18 @@ alter_account_option ::= CONNS literal.
 alter_account_option ::= STATE literal.                                           { }
 
 /************************************************ create/alter/drop user **********************************************/
-cmd ::= CREATE USER user_name(A) PASS NK_STRING(B).                               { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); }
+cmd ::= CREATE USER user_name(A) PASS NK_STRING(B) sysinfo_opt(C).                { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B, C); }
 cmd ::= ALTER USER user_name(A) PASS NK_STRING(B).                                { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); }
-cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B).                           { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); }
+//cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B).                           { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); }
+cmd ::= ALTER USER user_name(A) ENABLE NK_INTEGER(B).                             { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_ENABLE, &B); }
+cmd ::= ALTER USER user_name(A) SYSINFO NK_INTEGER(B).                            { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_SYSINFO, &B); }
 cmd ::= DROP USER user_name(A).                                                   { pCxt->pRootNode = createDropUserStmt(pCxt, &A); }
 
+%type sysinfo_opt                                                                 { int8_t }
+%destructor sysinfo_opt                                                           { }
+sysinfo_opt(A) ::= .                                                              { A = 1; }
+sysinfo_opt(A) ::= SYSINFO NK_INTEGER(B).                                         { A = taosStr2Int8(B.z, NULL, 10); }
+
 /************************************************ grant/revoke ********************************************************/
 cmd ::= GRANT privileges(A) ON priv_level(B) TO user_name(C).                     { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C); }
 cmd ::= REVOKE privileges(A) ON priv_level(B) FROM user_name(C).                  { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C); }
@@ -396,9 +403,9 @@ from_db_opt(A) ::= FROM db_name(B).
 /************************************************ create index ********************************************************/
 cmd ::= CREATE SMA INDEX not_exists_opt(D) 
   index_name(A) ON table_name(B) index_options(C).                                { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, &A, &B, NULL, C); }
-cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D)
-  index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP.                    { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); }
-cmd ::= DROP INDEX exists_opt(C) index_name(A) ON table_name(B).                  { pCxt->pRootNode = createDropIndexStmt(pCxt, C, &A, &B); }
+//cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D)
+//  index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP.                    { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); }
+cmd ::= DROP INDEX exists_opt(B) index_name(A).                                   { pCxt->pRootNode = createDropIndexStmt(pCxt, B, &A); }
 
 index_options(A) ::= .                                                            { A = NULL; }
 index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL 
@@ -414,10 +421,14 @@ func_list(A) ::= func_list(B) NK_COMMA func(C).
 func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP.                      { A = createFunctionNode(pCxt, &B, C); }
 
 /************************************************ create/drop topic ***************************************************/
-cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C).      { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, C, NULL, NULL); }
-cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C).      { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C, NULL); }
-cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B)
-  AS STABLE full_table_name(C).                                                   { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, NULL, C); }
+cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C).      { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, A, &B, C); }
+cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C).      { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, false); }
+cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) 
+  WITH META AS DATABASE db_name(C).                                               { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, true); }
+cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) 
+  AS STABLE full_table_name(C).                                                   { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false); }
+cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) 
+  WITH META AS STABLE full_table_name(C).                                         { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true); }
 cmd ::= DROP TOPIC exists_opt(A) topic_name(B).                                   { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); }
 cmd ::= DROP CONSUMER GROUP exists_opt(A) cgroup_name(B) ON topic_name(C).        { pCxt->pRootNode = createDropCGroupStmt(pCxt, A, &B, &C); }
 
@@ -627,6 +638,16 @@ expression(A) ::= column_reference(B) NK_ARROW NK_STRING(C).
                                                                                     SToken s = getTokenFromRawExprNode(pCxt, B);
                                                                                     A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, B), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &C))); 
                                                                                   }
+expression(A) ::= expression(B) NK_BITAND expression(C).                          {
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
+                                                                                  }
+expression(A) ::= expression(B) NK_BITOR expression(C).                           {
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C)));
+                                                                                  }
 
 %type expression_list                                                             { SNodeList* }
 %destructor expression_list                                                       { nodesDestroyList($$); }

source/libs/parser/src/parAstCreater.c

@@ -1205,7 +1205,7 @@ SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId) {
   return (SNode*)pStmt;
 }
 
-SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword) {
+SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo) {
   CHECK_PARSER_STATUS(pCxt);
   char password[TSDB_USET_PASSWORD_LEN] = {0};
   if (!checkUserName(pCxt, pUserName) || !checkPassword(pCxt, pPassword, password)) {
@@ -1215,6 +1215,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST
   CHECK_OUT_OF_MEM(pStmt);
   COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName);
   strcpy(pStmt->password, password);
+  pStmt->sysinfo = sysinfo;
   return (SNode*)pStmt;
 }
 
@@ -1226,15 +1227,26 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al
   SAlterUserStmt* pStmt = (SAlterUserStmt*)nodesMakeNode(QUERY_NODE_ALTER_USER_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName);
-  if (TSDB_ALTER_USER_PASSWD == alterType) {
-    char password[TSDB_USET_PASSWORD_LEN] = {0};
-    if (!checkPassword(pCxt, pVal, password)) {
-      nodesDestroyNode((SNode*)pStmt);
-      return NULL;
+  pStmt->alterType = alterType;
+  switch (alterType) {
+    case TSDB_ALTER_USER_PASSWD: {
+      char password[TSDB_USET_PASSWORD_LEN] = {0};
+      if (!checkPassword(pCxt, pVal, password)) {
+        nodesDestroyNode((SNode*)pStmt);
+        return NULL;
+      }
+      strcpy(pStmt->password, password);
+      break;
     }
-    strcpy(pStmt->password, password);
+    case TSDB_ALTER_USER_ENABLE:
+      pStmt->enable = taosStr2Int8(pVal->z, NULL, 10);
+      break;
+    case TSDB_ALTER_USER_SYSINFO:
+      pStmt->sysinfo = taosStr2Int8(pVal->z, NULL, 10);
+      break;
+    default:
+      break;
   }
-  pStmt->alterType = alterType;
   return (SNode*)pStmt;
 }
 
@@ -1317,16 +1329,15 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt
   return (SNode*)pOptions;
 }
 
-SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName) {
+SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName) {
   CHECK_PARSER_STATUS(pCxt);
-  if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) {
+  if (!checkDbName(pCxt, NULL, true) || !checkIndexName(pCxt, pIndexName)) {
     return NULL;
   }
   SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   pStmt->ignoreNotExists = ignoreNotExists;
   COPY_STRING_FORM_ID_TOKEN(pStmt->indexName, pIndexName);
-  COPY_STRING_FORM_ID_TOKEN(pStmt->tableName, pTableName);
   return (SNode*)pStmt;
 }
 
@@ -1346,22 +1357,40 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons
   return (SNode*)pStmt;
 }
 
-SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery,
-                             const SToken* pSubDbName, SNode* pRealTable) {
+SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
+                                     SNode* pQuery) {
   CHECK_PARSER_STATUS(pCxt);
   SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
   pStmt->ignoreExists = ignoreExists;
-  if (NULL != pRealTable) {
-    strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName);
-    strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName);
-    nodesDestroyNode(pRealTable);
-  } else if (NULL != pSubDbName) {
-    COPY_STRING_FORM_ID_TOKEN(pStmt->subDbName, pSubDbName);
-  } else {
-    pStmt->pQuery = pQuery;
-  }
+  pStmt->pQuery = pQuery;
+  return (SNode*)pStmt;
+}
+
+SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
+                                  const SToken* pSubDbName, bool withMeta) {
+  CHECK_PARSER_STATUS(pCxt);
+  SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
+  pStmt->ignoreExists = ignoreExists;
+  COPY_STRING_FORM_ID_TOKEN(pStmt->subDbName, pSubDbName);
+  pStmt->withMeta = withMeta;
+  return (SNode*)pStmt;
+}
+
+SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
+                                     SNode* pRealTable, bool withMeta) {
+  CHECK_PARSER_STATUS(pCxt);
+  SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
+  pStmt->ignoreExists = ignoreExists;
+  pStmt->withMeta = withMeta;
+  strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName);
+  strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName);
+  nodesDestroyNode(pRealTable);
   return (SNode*)pStmt;
 }
 

source/libs/parser/src/parTokenizer.c

@@ -79,6 +79,7 @@ static SKeyword keywordTable[] = {
     {"DOUBLE",        TK_DOUBLE},
     {"DROP",          TK_DROP},
     {"DURATION",      TK_DURATION},
+    {"ENABLE",        TK_ENABLE},
     {"EXISTS",        TK_EXISTS},
     {"EXPLAIN",       TK_EXPLAIN},
     {"EVERY",         TK_EVERY},
@@ -120,6 +121,7 @@ static SKeyword keywordTable[] = {
     {"MAXROWS",       TK_MAXROWS},
     {"MAX_DELAY",     TK_MAX_DELAY},
     {"MERGE",         TK_MERGE},
+    {"META",          TK_META},
     {"MINROWS",       TK_MINROWS},
     {"MINUS",         TK_MINUS},
     {"MNODE",         TK_MNODE},
@@ -146,7 +148,7 @@ static SKeyword keywordTable[] = {
     {"PORT",          TK_PORT},
     {"PPS",           TK_PPS},
     {"PRECISION",     TK_PRECISION},
-    {"PRIVILEGE",     TK_PRIVILEGE},
+    // {"PRIVILEGE",     TK_PRIVILEGE},
     {"PREV",          TK_PREV},
     {"QNODE",         TK_QNODE},
     {"QNODES",        TK_QNODES},
@@ -187,6 +189,7 @@ static SKeyword keywordTable[] = {
     {"STREAMS",       TK_STREAMS},
     {"STRICT",        TK_STRICT},
     {"SYNCDB",        TK_SYNCDB},
+    {"SYSINFO",       TK_SYSINFO},
     {"TABLE",         TK_TABLE},
     {"TABLES",        TK_TABLES},
     {"TAG",           TK_TAG},
@@ -222,6 +225,7 @@ static SKeyword keywordTable[] = {
     {"WATERMARK",     TK_WATERMARK},
     {"WHERE",         TK_WHERE},
     {"WINDOW_CLOSE",  TK_WINDOW_CLOSE},
+    {"WITH",          TK_WITH},
     {"WRITE",         TK_WRITE},
     {"_C0",           TK_ROWTS},
     {"_QENDTS",       TK_QENDTS},

source/libs/parser/src/parTranslater.c

@@ -1012,6 +1012,17 @@ static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pO
   return DEAL_RES_CONTINUE;
 }
 
+static EDealRes translateBitwiseOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
+  SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
+  SDataType rdt = ((SExprNode*)(pOp->pRight))->resType;
+  if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
+    return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
+  }
+  pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT;
+  pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
+  return DEAL_RES_CONTINUE;
+}
+
 static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode** pOpRef) {
   SOperatorNode* pOp = *pOpRef;
 
@@ -1030,6 +1041,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode** pOpRe
     return translateComparisonOperator(pCxt, pOp);
   } else if (nodesIsJsonOp(pOp)) {
     return translateJsonOperator(pCxt, pOp);
+  } else if (nodesIsBitwiseOp(pOp)) {
+    return translateBitwiseOperator(pCxt, pOp);
   }
   return DEAL_RES_CONTINUE;
 }
@@ -2381,6 +2394,8 @@ static SNode* createGroupingSet(SNode* pExpr) {
   return (SNode*)pGroupingSet;
 }
 
+// from: select unique(expr), col1 + col2 from t where_clause partition_by_clause order_by_clause ...
+// to: select expr, first(col1) + first(col2) from t where_clause partition_by_clause group by expr order_by_clause ...
 static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (!pSelect->hasUniqueFunc) {
     return TSDB_CODE_SUCCESS;
@@ -2446,6 +2461,8 @@ static SNode* createOrderByExpr(STranslateContext* pCxt) {
   return (SNode*)pOrder;
 }
 
+// from: select tail(expr, k, f) from t where_clause partition_by_clause order_by_clause ...
+// to: select expr from t where_clause order by _rowts desc limit k offset f
 static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (!pSelect->hasTailFunc) {
     return TSDB_CODE_SUCCESS;
@@ -3740,7 +3757,7 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt
   strcpy(createReq.user, pStmt->useName);
   createReq.createType = 0;
   createReq.superUser = 0;
-  createReq.sysInfo = 1;
+  createReq.sysInfo = pStmt->sysinfo;
   createReq.enable = 1;
   strcpy(createReq.pass, pStmt->password);
 
@@ -3752,6 +3769,8 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt
   strcpy(alterReq.user, pStmt->useName);
   alterReq.alterType = pStmt->alterType;
   alterReq.superUser = 0;
+  alterReq.enable = pStmt->enable;
+  alterReq.sysInfo = pStmt->sysinfo;
   strcpy(alterReq.pass, pStmt->password);
   if (NULL != pCxt->pParseCxt->db) {
     strcpy(alterReq.dbname, pCxt->pParseCxt->db);
@@ -3906,36 +3925,11 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p
 }
 
 static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) {
-  SEncoder      encoder = {0};
-  int32_t       contLen = 0;
-  SVDropTSmaReq dropSmaReq = {0};
-  strcpy(dropSmaReq.indexName, pStmt->indexName);
-
-  pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
-  if (NULL == pCxt->pCmdMsg) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  int32_t ret = 0;
-  tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret);
-  if (ret < 0) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
-  pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA;
-  pCxt->pCmdMsg->msgLen = contLen;
-  pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen);
-  if (NULL == pCxt->pCmdMsg->pMsg) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  void* pBuf = pCxt->pCmdMsg->pMsg;
-  if (tEncodeSVDropTSmaReq(&encoder, &dropSmaReq) < 0) {
-    tEncoderClear(&encoder);
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  tEncoderClear(&encoder);
-  return TSDB_CODE_SUCCESS;
+  SMDropSmaReq dropSmaReq = {0};
+  SName        name;
+  tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->indexName, &name), dropSmaReq.name);
+  dropSmaReq.igNotExists = pStmt->ignoreNotExists;
+  return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq);
 }
 
 static int16_t getCreateComponentNodeMsgType(ENodeType type) {
@@ -3987,6 +3981,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS
   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName));
   tNameGetFullDbName(&name, pReq->name);
   pReq->igExists = pStmt->ignoreExists;
+  pReq->withMeta = pStmt->withMeta;
 
   pReq->sql = strdup(pCxt->pParseCxt->pSql);
   if (NULL == pReq->sql) {
@@ -4342,6 +4337,11 @@ static int32_t translateSplitVgroup(STranslateContext* pCxt, SSplitVgroupStmt* p
   return buildCmdMsg(pCxt, TDMT_MND_SPLIT_VGROUP, (FSerializeFunc)tSerializeSSplitVgroupReq, &req);
 }
 
+static int32_t translateShowVariables(STranslateContext* pCxt, SShowStmt* pStmt) {
+  SShowVariablesReq req = {0};
+  return buildCmdMsg(pCxt, TDMT_MND_SHOW_VARIABLES, (FSerializeFunc)tSerializeSShowVariablesReq, &req);
+}
+
 static int32_t translateShowCreateDatabase(STranslateContext* pCxt, SShowCreateDatabaseStmt* pStmt) {
   pStmt->pCfg = taosMemoryCalloc(1, sizeof(SDbCfgInfo));
   if (NULL == pStmt->pCfg) {
@@ -4486,6 +4486,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
     case QUERY_NODE_SPLIT_VGROUP_STMT:
       code = translateSplitVgroup(pCxt, (SSplitVgroupStmt*)pNode);
       break;
+    case QUERY_NODE_SHOW_VARIABLES_STMT:
+      code = translateShowVariables(pCxt, (SShowStmt*)pNode);
+      break;
     case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
       code = translateShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pNode);
       break;

source/libs/parser/test/parInitialATest.cpp

@@ -121,7 +121,7 @@ TEST_F(ParserInitialATest, alterSTable) {
     int32_t len = snprintf(expect.name, sizeof(expect.name), "0.test.%s", pTbname);
     expect.name[len] = '\0';
     expect.alterType = alterType;
-//    expect.ttl = ttl;
+    //    expect.ttl = ttl;
     if (nullptr != pComment) {
       expect.comment = strdup(pComment);
       expect.commentLen = strlen(pComment);
@@ -180,9 +180,9 @@ TEST_F(ParserInitialATest, alterSTable) {
     tFreeSMAltertbReq(&req);
   });
 
-//  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
-//  run("ALTER TABLE st1 TTL 10");
-//  clearAlterStbReq();
+  //  setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10);
+  //  run("ALTER TABLE st1 TTL 10");
+  //  clearAlterStbReq();
 
   setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test");
   run("ALTER TABLE st1 COMMENT 'test'");
@@ -381,9 +381,48 @@ TEST_F(ParserInitialATest, alterTableSemanticCheck) {
 TEST_F(ParserInitialATest, alterUser) {
   useDb("root", "test");
 
-  run("ALTER user wxy PASS '123456'");
+  SAlterUserReq expect = {0};
 
-  run("ALTER user wxy privilege 'write'");
+  auto clearAlterUserReq = [&]() { memset(&expect, 0, sizeof(SAlterUserReq)); };
+
+  auto setAlterUserReq = [&](const char* pUser, int8_t alterType, const char* pPass = nullptr, int8_t sysInfo = 0,
+                             int8_t enable = 0) {
+    strcpy(expect.user, pUser);
+    expect.alterType = alterType;
+    expect.superUser = 0;
+    expect.sysInfo = sysInfo;
+    expect.enable = enable;
+    if (nullptr != pPass) {
+      strcpy(expect.pass, pPass);
+    }
+    strcpy(expect.dbname, "test");
+  };
+
+  setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
+    ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_USER_STMT);
+    SAlterUserReq req = {0};
+    ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req));
+
+    ASSERT_EQ(req.alterType, expect.alterType);
+    ASSERT_EQ(req.superUser, expect.superUser);
+    ASSERT_EQ(req.sysInfo, expect.sysInfo);
+    ASSERT_EQ(req.enable, expect.enable);
+    ASSERT_EQ(std::string(req.user), std::string(expect.user));
+    ASSERT_EQ(std::string(req.pass), std::string(expect.pass));
+    ASSERT_EQ(std::string(req.dbname), std::string(expect.dbname));
+  });
+
+  setAlterUserReq("wxy", TSDB_ALTER_USER_PASSWD, "123456");
+  run("ALTER USER wxy PASS '123456'");
+  clearAlterUserReq();
+
+  setAlterUserReq("wxy", TSDB_ALTER_USER_ENABLE, nullptr, 0, 1);
+  run("ALTER USER wxy ENABLE 1");
+  clearAlterUserReq();
+
+  setAlterUserReq("wxy", TSDB_ALTER_USER_SYSINFO, nullptr, 1);
+  run("ALTER USER wxy SYSINFO 1");
+  clearAlterUserReq();
 }
 
 TEST_F(ParserInitialATest, balanceVgroup) {

source/libs/parser/test/parInitialCTest.cpp

@@ -370,7 +370,7 @@ TEST_F(ParserInitialCTest, createStable) {
     expect.delay2 = delay2;
     expect.watermark1 = watermark1;
     expect.watermark2 = watermark2;
-//    expect.ttl = ttl;
+    //    expect.ttl = ttl;
     if (nullptr != pComment) {
       expect.comment = strdup(pComment);
       expect.commentLen = strlen(pComment);
@@ -414,7 +414,7 @@ TEST_F(ParserInitialCTest, createStable) {
     ASSERT_EQ(req.ttl, expect.ttl);
     ASSERT_EQ(req.numOfColumns, expect.numOfColumns);
     ASSERT_EQ(req.numOfTags, expect.numOfTags);
-//    ASSERT_EQ(req.commentLen, expect.commentLen);
+    //    ASSERT_EQ(req.commentLen, expect.commentLen);
     ASSERT_EQ(req.ast1Len, expect.ast1Len);
     ASSERT_EQ(req.ast2Len, expect.ast2Len);
 
@@ -621,10 +621,11 @@ TEST_F(ParserInitialCTest, createTopic) {
   auto clearCreateTopicReq = [&]() { memset(&expect, 0, sizeof(SCMCreateTopicReq)); };
 
   auto setCreateTopicReqFunc = [&](const char* pTopicName, int8_t igExists, const char* pSql, const char* pAst,
-                                   const char* pDbName = nullptr, const char* pTbname = nullptr) {
+                                   const char* pDbName = nullptr, const char* pTbname = nullptr, int8_t withMeta = 0) {
     snprintf(expect.name, sizeof(expect.name), "0.%s", pTopicName);
     expect.igExists = igExists;
     expect.sql = (char*)pSql;
+    expect.withMeta = withMeta;
     if (nullptr != pTbname) {
       expect.subType = TOPIC_SUB_TYPE__TABLE;
       snprintf(expect.subStbName, sizeof(expect.subStbName), "0.%s.%s", pDbName, pTbname);
@@ -647,6 +648,7 @@ TEST_F(ParserInitialCTest, createTopic) {
     ASSERT_EQ(req.igExists, expect.igExists);
     ASSERT_EQ(req.subType, expect.subType);
     ASSERT_EQ(std::string(req.sql), std::string(expect.sql));
+    ASSERT_EQ(req.withMeta, expect.withMeta);
     switch (expect.subType) {
       case TOPIC_SUB_TYPE__DB:
         ASSERT_EQ(std::string(req.subDbName), std::string(expect.subDbName));
@@ -675,15 +677,55 @@ TEST_F(ParserInitialCTest, createTopic) {
   run("CREATE TOPIC tp1 AS DATABASE test");
   clearCreateTopicReq();
 
+  setCreateTopicReqFunc("tp1", 0, "create topic tp1 with meta as database test", nullptr, "test", nullptr, 1);
+  run("CREATE TOPIC tp1 WITH META AS DATABASE test");
+  clearCreateTopicReq();
+
   setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 as stable st1", nullptr, "test", "st1");
   run("CREATE TOPIC IF NOT EXISTS tp1 AS STABLE st1");
   clearCreateTopicReq();
+
+  setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1);
+  run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1");
+  clearCreateTopicReq();
 }
 
 TEST_F(ParserInitialCTest, createUser) {
   useDb("root", "test");
 
+  SCreateUserReq expect = {0};
+
+  auto clearCreateUserReq = [&]() { memset(&expect, 0, sizeof(SCreateUserReq)); };
+
+  auto setCreateUserReq = [&](const char* pUser, const char* pPass, int8_t sysInfo = 1) {
+    strcpy(expect.user, pUser);
+    strcpy(expect.pass, pPass);
+    expect.createType = 0;
+    expect.superUser = 0;
+    expect.sysInfo = sysInfo;
+    expect.enable = 1;
+  };
+
+  setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
+    ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_USER_STMT);
+    SCreateUserReq req = {0};
+    ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSCreateUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req));
+
+    ASSERT_EQ(req.createType, expect.createType);
+    ASSERT_EQ(req.superUser, expect.superUser);
+    ASSERT_EQ(req.sysInfo, expect.sysInfo);
+    ASSERT_EQ(req.enable, expect.enable);
+    ASSERT_EQ(std::string(req.user), std::string(expect.user));
+    ASSERT_EQ(std::string(req.pass), std::string(expect.pass));
+  });
+
+  setCreateUserReq("wxy", "123456");
   run("CREATE USER wxy PASS '123456'");
+  clearCreateUserReq();
+
+  setCreateUserReq("wxy1", "a123456", 1);
+  run("CREATE USER wxy1 PASS 'a123456' SYSINFO 1");
+  clearCreateUserReq();
 }
 
 }  // namespace ParserTest

source/libs/parser/test/parInitialDTest.cpp

@@ -132,7 +132,31 @@ TEST_F(ParserInitialDTest, dropDnode) {
 TEST_F(ParserInitialDTest, dropIndex) {
   useDb("root", "test");
 
-  run("DROP index index1 on t1");
+  SMDropSmaReq expect = {0};
+
+  auto clearDropSmaReq = [&]() { memset(&expect, 0, sizeof(SMDropSmaReq)); };
+
+  auto setDropSmaReq = [&](const char* pName, int8_t igNotExists = 0) {
+    sprintf(expect.name, "0.test.%s", pName);
+    expect.igNotExists = igNotExists;
+  };
+
+  setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
+    ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_DROP_INDEX_STMT);
+    SMDropSmaReq req = {0};
+    ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMDropSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req));
+
+    ASSERT_EQ(std::string(req.name), std::string(expect.name));
+    ASSERT_EQ(req.igNotExists, expect.igNotExists);
+  });
+
+  setDropSmaReq("index1");
+  run("DROP INDEX index1");
+  clearDropSmaReq();
+
+  setDropSmaReq("index2", 1);
+  run("DROP INDEX IF EXISTS index2");
+  clearDropSmaReq();
 }
 
 TEST_F(ParserInitialDTest, dropMnode) {

source/libs/parser/test/parSelectTest.cpp

@@ -58,6 +58,8 @@ TEST_F(ParserSelectTest, expression) {
   run("SELECT ts > 0, c1 < 20 and c2 = 'qaz' FROM t1");
 
   run("SELECT ts > 0, c1 between 10 and 20 and c2 = 'qaz' FROM t1");
+
+  run("SELECT c1 | 10, c2 & 20, c4 | c5 FROM t1");
 }
 
 TEST_F(ParserSelectTest, condition) {

source/libs/planner/src/planOptimizer.c

@@ -79,7 +79,8 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func)
 
 EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) {
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
-    *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType);
+    // *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType);
+    *((bool*)pContext) = true;
     return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD;
   }
   return DEAL_RES_CONTINUE;
@@ -98,11 +99,6 @@ static bool osdMayBeOptimized(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) {
     return false;
   }
-  // todo: release after function splitting
-  if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->tableType &&
-      SCAN_TYPE_STREAM != ((SScanLogicNode*)pNode)->scanType) {
-    return false;
-  }
   if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) &&
                                  QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) &&
                                  QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) {
@@ -775,7 +771,7 @@ static bool smaOptMayBeOptimized(SLogicNode* pNode) {
   }
 
   SScanLogicNode* pScan = (SScanLogicNode*)pNode;
-  if (0 == pScan->interval || NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) {
+  if (NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) {
     return false;
   }
 

source/libs/stream/inc/streamInc.h

@@ -33,8 +33,13 @@ static SStreamGlobalEnv streamEnv;
 int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb);
 int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb);
 int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData);
+int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData);
 int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet);
 
+int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock);
+
+int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/stream/src/stream.c

@@ -112,7 +112,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
   // enqueue
   if (pData != NULL) {
     pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
-    pData->sourceVg = pReq->sourceVg;
+    pData->srcVgId = pReq->dataSrcVgId;
     // decode
     /*pData->blocks = pReq->data;*/
     /*pBlock->sourceVer = pReq->sourceVer;*/
@@ -133,7 +133,42 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
   SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
   pCont->inputStatus = status;
   pCont->streamId = pReq->streamId;
-  pCont->taskId = pReq->sourceTaskId;
+  pCont->taskId = pReq->upstreamTaskId;
+  pRsp->pCont = buf;
+  pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
+  tmsgSendRsp(pRsp);
+  return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
+}
+
+int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
+  SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
+  int8_t            status = TASK_INPUT_STATUS__NORMAL;
+
+  // enqueue
+  if (pData != NULL) {
+    pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
+    pData->srcVgId = 0;
+    // decode
+    /*pData->blocks = pReq->data;*/
+    /*pBlock->sourceVer = pReq->sourceVer;*/
+    streamRetrieveReqToData(pReq, pData);
+    if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) {
+      status = TASK_INPUT_STATUS__NORMAL;
+    } else {
+      status = TASK_INPUT_STATUS__FAILED;
+    }
+  } else {
+    /*streamTaskInputFail(pTask);*/
+    /*status = TASK_INPUT_STATUS__FAILED;*/
+  }
+
+  // rsp by input status
+  void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp));
+  ((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId);
+  SStreamRetrieveRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  pCont->streamId = pReq->streamId;
+  pCont->rspToTaskId = pReq->srcTaskId;
+  pCont->rspFromTaskId = pReq->dstTaskId;
   pRsp->pCont = buf;
   pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
   tmsgSendRsp(pRsp);
@@ -141,7 +176,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
 }
 
 int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
-  qInfo("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, pReq->sourceTaskId);
+  qInfo("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, pReq->upstreamTaskId);
 
   // 1. handle input
   streamTaskEnqueue(pTask, pReq, pRsp);
@@ -208,3 +243,22 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp)
   //
   return 0;
 }
+
+int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
+  qInfo("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId);
+
+  streamTaskEnqueueRetrieve(pTask, pReq, pRsp);
+
+  ASSERT(pTask->execType != TASK_EXEC__NONE);
+  streamExec(pTask, pTask->pMsgCb);
+
+  ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE);
+  streamDispatch(pTask, pTask->pMsgCb);
+
+  return 0;
+}
+
+int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) {
+  //
+  return 0;
+}

source/libs/stream/src/streamData.c

@@ -15,27 +15,6 @@
 
 #include "streamInc.h"
 
-#if 0
-int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI8(buf, pOutput->type);
-  tlen += taosEncodeFixedI32(buf, pOutput->sourceVg);
-  tlen += taosEncodeFixedI64(buf, pOutput->sourceVer);
-  ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK);
-  tlen += tEncodeDataBlocks(buf, pOutput->blocks);
-  return tlen;
-}
-
-void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
-  buf = taosDecodeFixedI8(buf, &pInput->type);
-  buf = taosDecodeFixedI32(buf, &pInput->sourceVg);
-  buf = taosDecodeFixedI64(buf, &pInput->sourceVer);
-  ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK);
-  buf = tDecodeDataBlocks(buf, &pInput->blocks);
-  return (void*)buf;
-}
-#endif
-
 int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData) {
   int32_t blockNum = pReq->blockNum;
   SArray* pArray = taosArrayInit(blockNum, sizeof(SSDataBlock));
@@ -54,8 +33,21 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock
     blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
     // TODO: refactor
     pDataBlock->info.type = pRetrieve->streamBlockType;
-    pDataBlock->info.childId = pReq->sourceChildId;
+    pDataBlock->info.childId = pReq->upstreamChildId;
+  }
+  pData->blocks = pArray;
+  return 0;
+}
+
+int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData) {
+  SArray* pArray = taosArrayInit(1, sizeof(SSDataBlock));
+  if (pArray == NULL) {
+    return -1;
   }
+  taosArraySetSize(pArray, 1);
+  SRetrieveTableRsp* pRetrieve = pReq->pRetrieve;
+  SSDataBlock*       pBlock = taosArrayGet(pArray, 0);
+  blockCompressDecode(pBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data);
   pData->blocks = pArray;
   return 0;
 }

source/libs/stream/src/streamDispatch.c

@@ -19,9 +19,9 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p
   if (tStartEncode(pEncoder) < 0) return -1;
   if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->sourceChildId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->dataSrcVgId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->upstreamChildId) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
   if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1;
   ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum);
@@ -40,9 +40,9 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
   if (tStartDecode(pDecoder) < 0) return -1;
   if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->sourceChildId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->dataSrcVgId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->upstreamChildId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1;
   ASSERT(pReq->blockNum > 0);
@@ -62,6 +62,102 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
   return 0;
 }
 
+int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) {
+  //
+  if (tStartEncode(pEncoder) < 0) return -1;
+  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->dstTaskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->srcNodeId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->srcTaskId) < 0) return -1;
+  if (tEncodeBinary(pEncoder, (const uint8_t*)&pReq->pRetrieve, pReq->retrieveLen) < 0) return -1;
+  tEndEncode(pEncoder);
+  return pEncoder->pos;
+}
+
+int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) {
+  int32_t tlen = 0;
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->dstNodeId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->dstTaskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->srcNodeId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->srcTaskId) < 0) return -1;
+  if (tDecodeBinary(pDecoder, (uint8_t**)&pReq->pRetrieve, &pReq->retrieveLen) < 0) return -1;
+  tEndDecode(pDecoder);
+  return 0;
+}
+
+int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) {
+  SRetrieveTableRsp* pRetrieve = NULL;
+  void*              buf = NULL;
+  int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+
+  pRetrieve = taosMemoryCalloc(1, dataStrLen);
+  if (pRetrieve == NULL) return -1;
+
+  pRetrieve->useconds = 0;
+  pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+  pRetrieve->compressed = 0;
+  pRetrieve->completed = 1;
+  pRetrieve->streamBlockType = pBlock->info.type;
+  pRetrieve->numOfRows = htonl(pBlock->info.rows);
+  pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);
+
+  int32_t actualLen = 0;
+  blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false);
+
+  SStreamRetrieveReq req = {
+      .streamId = pTask->streamId,
+      .srcNodeId = pTask->nodeId,
+      .srcTaskId = pTask->taskId,
+      .pRetrieve = pRetrieve,
+  };
+
+  int32_t sz = taosArrayGetSize(pTask->childEpInfo);
+  ASSERT(sz > 0);
+  for (int32_t i = 0; i < sz; i++) {
+    SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i);
+    req.dstNodeId = pEpInfo->nodeId;
+    req.dstTaskId = pEpInfo->taskId;
+    int32_t code;
+    int32_t len;
+    tEncodeSize(tEncodeStreamRetrieveReq, &req, len, code);
+    if (code < 0) {
+      ASSERT(0);
+      return -1;
+    }
+
+    buf = rpcMallocCont(sizeof(SMsgHead) + len);
+    if (buf == NULL) {
+      goto FAIL;
+    }
+
+    ((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId);
+    void*    abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+    SEncoder encoder;
+    tEncoderInit(&encoder, abuf, len);
+    tEncodeStreamRetrieveReq(&encoder, &req);
+
+    SRpcMsg rpcMsg = {
+        .code = 0,
+        .msgType = TDMT_STREAM_RETRIEVE,
+        .pCont = buf,
+        .contLen = len,
+    };
+
+    if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  }
+  return 0;
+FAIL:
+  if (pRetrieve) taosMemoryFree(pRetrieve);
+  if (buf) taosMemoryFree(buf);
+  return -1;
+}
+
 static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) {
   int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
   void*   buf = taosMemoryCalloc(1, dataStrLen);
@@ -96,9 +192,9 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
 
   SStreamDispatchReq req = {
       .streamId = pTask->streamId,
-      .sourceTaskId = pTask->taskId,
-      .sourceVg = data->sourceVg,
-      .sourceChildId = pTask->childId,
+      .dataSrcVgId = data->srcVgId,
+      .upstreamTaskId = pTask->taskId,
+      .upstreamChildId = pTask->selfChildId,
       .upstreamNodeId = pTask->nodeId,
       .blockNum = blockNum,
   };
@@ -149,7 +245,7 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
   ASSERT(vgId > 0 || vgId == SNODE_HANDLE);
   req.taskId = downstreamTaskId;
 
-  qInfo("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->childId,
+  qInfo("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId,
         downstreamTaskId, vgId);
 
   // serialize

source/libs/stream/src/streamExec.c

@@ -13,8 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "executor.h"
-#include "tstream.h"
+#include "streamInc.h"
 
 static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
   void* exec = pTask->exec.executor;
@@ -46,9 +45,17 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
       ASSERT(false);
     }
     if (output == NULL) break;
+
+    if (output->info.type == STREAM_RETRIEVE) {
+      if (streamBroadcastToChildren(pTask, output) < 0) {
+        // TODO
+      }
+      continue;
+    }
+
     // TODO: do we need free memory?
     SSDataBlock* outputCopy = createOneDataBlock(output, true);
-    outputCopy->info.childId = pTask->childId;
+    outputCopy->info.childId = pTask->selfChildId;
     taosArrayPush(pRes, outputCopy);
   }
   return 0;

source/libs/stream/src/streamTask.c

@@ -30,6 +30,22 @@ SStreamTask* tNewSStreamTask(int64_t streamId) {
   return pTask;
 }
 
+int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) {
+  if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1;
+  return 0;
+}
+
+int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) {
+  if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1;
+  return 0;
+}
+
 int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   /*if (tStartEncode(pEncoder) < 0) return -1;*/
   if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
@@ -41,12 +57,19 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
   if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1;
   if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->dataScan) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->isDataScan) < 0) return -1;
 
-  if (tEncodeI32(pEncoder, pTask->childId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1;
   if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1;
 
+  int32_t epSz = taosArrayGetSize(pTask->childEpInfo);
+  if (tEncodeI32(pEncoder, epSz) < 0) return -1;
+  for (int32_t i = 0; i < epSz; i++) {
+    SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->childEpInfo, i);
+    if (tEncodeStreamEpInfo(pEncoder, pInfo) < 0) return -1;
+  }
+
   if (pTask->execType != TASK_EXEC__NONE) {
     if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1;
   }
@@ -90,12 +113,22 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
   if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1;
   if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->dataScan) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->isDataScan) < 0) return -1;
 
-  if (tDecodeI32(pDecoder, &pTask->childId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1;
   if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1;
 
+  int32_t epSz;
+  if (tDecodeI32(pDecoder, &epSz) < 0) return -1;
+  pTask->childEpInfo = taosArrayInit(epSz, sizeof(void*));
+  for (int32_t i = 0; i < epSz; i++) {
+    SStreamChildEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamChildEpInfo));
+    if (pInfo == NULL) return -1;
+    if (tDecodeStreamEpInfo(pDecoder, pInfo) < 0) return -1;
+    taosArrayPush(pTask->childEpInfo, &pInfo);
+  }
+
   if (pTask->execType != TASK_EXEC__NONE) {
     if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1;
   }

source/libs/wal/src/walMeta.c

@@ -19,6 +19,8 @@
 #include "tref.h"
 #include "walInt.h"
 
+bool FORCE_INLINE walIsEmpty(SWal* pWal) { return pWal->vers.firstVer == -1; }
+
 int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }
 
 int64_t FORCE_INLINE walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; }

source/libs/wal/src/walWrite.c

@@ -141,7 +141,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
   // validate offset
   SWalHead head;
   ASSERT(taosValidFile(pLogTFile));
-  int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead));
+  int64_t size = taosReadFile(pLogTFile, &head, sizeof(SWalHead));
   if (size != sizeof(SWalHead)) {
     return -1;
   }
@@ -149,22 +149,33 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
 
   ASSERT(code == 0);
   if (code != 0) {
+    terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
   if (head.head.version != ver) {
-    // TODO
+    ASSERT(0);
+    terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
+
   // truncate old files
   code = taosFtruncateFile(pLogTFile, entry.offset);
   if (code < 0) {
+    ASSERT(0);
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
   code = taosFtruncateFile(pIdxTFile, idxOff);
   if (code < 0) {
+    ASSERT(0);
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
   pWal->vers.lastVer = ver - 1;
+  if (pWal->vers.lastVer < pWal->vers.firstVer) {
+    ASSERT(pWal->vers.lastVer == pWal->vers.firstVer - 1);
+    pWal->vers.firstVer = -1;
+  }
   ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
   ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
   taosCloseFile(&pIdxTFile);

tests/pytest/util/cluster.py

+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os 
+import  socket
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+
+class ClusterDnodes(TDDnodes):
+    """rewrite TDDnodes and make MyDdnodes as TDDnodes child class"""
+    def __init__(self ,dnodes_lists):
+
+        super(ClusterDnodes,self).__init__()
+        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
+        self.simDeployed = False
+        self.testCluster = False
+        self.valgrind = 0
+        self.killValgrind = 1
+        self.independent = True
+        self.dnodeNums = 5
+
+    # def getTDDnodes(dnodeNums):
+
+    #     return 
+
+
+class ConfigureyCluster:
+    """configure dnodes and return TDDnodes list, it can """
+
+    def __init__(self):
+        self.dnodes = None      
+        self.dnodes_nums = 5
+        self.independent = True
+        self.start_port = 6030
+        self.portStep = 100
+    hostname1= socket.gethostname()
+
+    def configure_cluster(self ,dnodes_nums=5,independent=True,start_port=6030,portStep=100,hostname="%s"%hostname1): 
+        self.start_port=int(start_port)
+        self.portStep=int(portStep)
+        self.hostname=hostname
+        self.dnodes_nums = int(dnodes_nums)
+        self.independent = independent
+        self.dnodes = []
+        start_port_sec = 6130
+        for num in range(1, (self.dnodes_nums+1)):
+            dnode = TDDnode(num)
+            dnode.addExtraCfg("firstEp", f"{hostname}:{self.start_port}")
+            dnode.addExtraCfg("fqdn", f"{hostname}")
+            dnode.addExtraCfg("serverPort", f"{self.start_port + (num-1)*self.portStep}")
+            # dnode.addExtraCfg("monitorFqdn", hostname)
+            # dnode.addExtraCfg("monitorPort", 7043)
+            dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
+            # configure three dnoe don't support vnodes
+            if self.dnodes_nums > 4 :
+                if self.independent and (num < 4):
+                    dnode.addExtraCfg("supportVnodes", 0)
+            # print(dnode)
+            self.dnodes.append(dnode)
+        return self.dnodes
+        
+    def create_dnode(self,conn):
+        tdSql.init(conn.cursor())
+        for dnode in self.dnodes[1:]:
+            # print(dnode.cfgDict)
+            dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
+            print(dnode_id)
+            tdSql.execute(" create dnode '%s';"%dnode_id)
+        # count=0
+        # while count < 10:
+        #     time.sleep(1)
+        #     tdSql.query("show dnodes;")
+        #     if tdSql.checkRows(self.dnodes_nums) :
+        #         print("mnode is  three nodes")
+        #     if  tdSql.queryResult[0][4]=='leader' :
+        #         if  tdSql.queryResult[2][4]=='offline':
+        #             if  tdSql.queryResult[1][2]=='follower':
+        #                 print("stop mnodes  on dnode 3 successfully in 10s")
+        #                 break
+        #     count+=1
+        # else:
+        #     print("stop mnodes  on dnode 3 failed in 10s")
+        #     return -1
+        checkstatus=False
+
+        
+        
+    def check_dnode(self,conn):
+        tdSql.init(conn.cursor())
+        count=0
+        while count < 5:
+            tdSql.query("show dnodes")
+            # tdLog.debug(tdSql.queryResult)
+            status=0
+            for i in range(self.dnodes_nums):
+                if tdSql.queryResult[i][4] == "ready":
+                    status+=1
+            tdLog.debug(status)
+            
+            if status == self.dnodes_nums:
+                tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodes_nums)
+                break 
+            count+=1
+            time.sleep(1)
+        else:
+            tdLog.debug("create cluster with %d dnode but  check dnode not ready within 5s ! "%self.dnodes_nums)
+            return -1
+
+
+cluster = ConfigureyCluster()
+
+    # def start(self ,dnodes_nums): 
+
+    #     self.TDDnodes = ClusterDnodes(dnodes)
+    #     self.TDDnodes.init("")
+    #     self.TDDnodes.setTestCluster(testCluster)
+    #     self.TDDnodes.setValgrind(valgrind)
+    #     self.TDDnodes.stopAll()
+    #     for dnode in self.TDDnodes.dnodes:
+    #         self.TDDnodes.deploy(dnode.index,{})
+            
+    #     for dnode in self.TDDnodes.dnodes:
+    #         self.TDDnodes.starttaosd(dnode.index)
+
+    #     # create cluster 
+    #     for dnode in self.TDDnodes.dnodes[1:]:
+    #         # print(dnode.cfgDict)
+    #         dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
+    #         dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
+    #         dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
+    #         cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+    #         print(cmd)
+    #         os.system(cmd)
+        
+    #     time.sleep(2)
+    #     tdLog.info(" create cluster with %d dnode  done! " %dnodes_nums)
+
+    # def buildcluster(self,dnodenumber):
+    #     self.depoly_cluster(dnodenumber)
+    #     self.master_dnode = self.TDDnodes.dnodes[0]
+    #     self.host=self.master_dnode.cfgDict["fqdn"]
+    #     conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
+    #     tdSql.init(conn1.cursor())
+        
+    # def checkdnodes(self,dnodenumber):
+    #     count=0
+    #     while count < 10:
+    #         time.sleep(1)
+    #         statusReadyBumber=0
+    #         tdSql.query("show dnodes;")
+    #         if tdSql.checkRows(dnodenumber) :     
+    #             print("dnode is %d nodes"%dnodenumber)   
+    #         for i in range(dnodenumber):
+    #             if tdSql.queryResult[i][4] !='ready'  :
+    #                 status=tdSql.queryResult[i][4]
+    #                 print("dnode:%d status is %s "%(i,status))
+    #                 break
+    #             else:
+    #                 statusReadyBumber+=1
+    #         print(statusReadyBumber)
+    #         if statusReadyBumber == dnodenumber :
+    #             print("all of %d mnodes is ready in 10s "%dnodenumber)
+    #             return True
+    #             break
+    #         count+=1
+    #     else:
+    #         print("%d mnodes is not ready in 10s "%dnodenumber)
+    #         return False
+           
+
+    # def check3mnode(self):
+    #     count=0
+    #     while count < 10:
+    #         time.sleep(1)
+    #         tdSql.query("show mnodes;")
+    #         if tdSql.checkRows(3) :     
+    #             print("mnode is  three nodes")           
+    #         if  tdSql.queryResult[0][2]=='leader' :
+    #             if  tdSql.queryResult[1][2]=='follower':
+    #                 if  tdSql.queryResult[2][2]=='follower':
+    #                     print("three mnodes is ready in 10s")
+    #                     break
+    #         elif tdSql.queryResult[0][2]=='follower' :
+    #             if  tdSql.queryResult[1][2]=='leader':
+    #                 if  tdSql.queryResult[2][2]=='follower':
+    #                     print("three mnodes is ready in 10s")
+    #                     break      
+    #         elif tdSql.queryResult[0][2]=='follower' :
+    #             if  tdSql.queryResult[1][2]=='follower':
+    #                 if  tdSql.queryResult[2][2]=='leader':
+    #                     print("three mnodes is ready in 10s")
+    #                     break                   
+    #         count+=1
+    #     else:
+    #         print("three mnodes is not ready in 10s ")
+    #         return -1
+
+    #     tdSql.query("show mnodes;")       
+    #     tdSql.checkRows(3) 
+    #     tdSql.checkData(0,1,'%s:6030'%self.host)
+    #     tdSql.checkData(0,3,'ready')
+    #     tdSql.checkData(1,1,'%s:6130'%self.host)
+    #     tdSql.checkData(1,3,'ready')
+    #     tdSql.checkData(2,1,'%s:6230'%self.host)
+    #     tdSql.checkData(2,3,'ready')
+
+    # def check3mnode1off(self):
+    #     count=0
+    #     while count < 10:
+    #         time.sleep(1)
+    #         tdSql.query("show mnodes;")
+    #         if tdSql.checkRows(3) :
+    #             print("mnode is  three nodes")
+    #         if  tdSql.queryResult[0][2]=='offline' :
+    #             if  tdSql.queryResult[1][2]=='leader':
+    #                 if  tdSql.queryResult[2][2]=='follower':
+    #                     print("stop mnodes  on dnode 2 successfully in 10s")
+    #                     break
+    #             elif tdSql.queryResult[1][2]=='follower':
+    #                 if  tdSql.queryResult[2][2]=='leader':
+    #                     print("stop mnodes  on dnode 2 successfully in 10s")
+    #                     break
+    #         count+=1
+    #     else:
+    #         print("stop mnodes  on dnode 2 failed in 10s ")
+    #         return -1
+    #     tdSql.error("drop mnode on dnode 1;")
+
+    #     tdSql.query("show mnodes;")       
+    #     tdSql.checkRows(3) 
+    #     tdSql.checkData(0,1,'%s:6030'%self.host)
+    #     tdSql.checkData(0,2,'offline')
+    #     tdSql.checkData(0,3,'ready')
+    #     tdSql.checkData(1,1,'%s:6130'%self.host)
+    #     tdSql.checkData(1,3,'ready')
+    #     tdSql.checkData(2,1,'%s:6230'%self.host)
+    #     tdSql.checkData(2,3,'ready')
+
+    # def check3mnode2off(self):
+    #     count=0
+    #     while count < 40:
+    #         time.sleep(1)
+    #         tdSql.query("show mnodes;")
+    #         if tdSql.checkRows(3) :
+    #             print("mnode is  three nodes")
+    #         if  tdSql.queryResult[0][2]=='leader' :
+    #             if  tdSql.queryResult[1][2]=='offline':
+    #                 if  tdSql.queryResult[2][2]=='follower':
+    #                     print("stop mnodes  on dnode 2 successfully in 10s")
+    #                     break
+    #         count+=1
+    #     else:
+    #         print("stop mnodes  on dnode 2 failed in 10s ")
+    #         return -1
+    #     tdSql.error("drop mnode on dnode 2;")
+
+    #     tdSql.query("show mnodes;")       
+    #     tdSql.checkRows(3) 
+    #     tdSql.checkData(0,1,'%s:6030'%self.host)
+    #     tdSql.checkData(0,2,'leader')
+    #     tdSql.checkData(0,3,'ready')
+    #     tdSql.checkData(1,1,'%s:6130'%self.host)
+    #     tdSql.checkData(1,2,'offline')
+    #     tdSql.checkData(1,3,'ready')
+    #     tdSql.checkData(2,1,'%s:6230'%self.host)
+    #     tdSql.checkData(2,2,'follower')
+    #     tdSql.checkData(2,3,'ready')
+
+    # def check3mnode3off(self):
+    #     count=0
+    #     while count < 10:
+    #         time.sleep(1)
+    #         tdSql.query("show mnodes;")
+    #         if tdSql.checkRows(3) :
+    #             print("mnode is  three nodes")
+    #         if  tdSql.queryResult[0][2]=='leader' :
+    #             if  tdSql.queryResult[2][2]=='offline':
+    #                 if  tdSql.queryResult[1][2]=='follower':
+    #                     print("stop mnodes  on dnode 3 successfully in 10s")
+    #                     break
+    #         count+=1
+    #     else:
+    #         print("stop mnodes  on dnode 3 failed in 10s")
+    #         return -1
+    #     tdSql.error("drop mnode on dnode 3;")
+    #     tdSql.query("show mnodes;")       
+    #     tdSql.checkRows(3) 
+    #     tdSql.checkData(0,1,'%s:6030'%self.host)
+    #     tdSql.checkData(0,2,'leader')
+    #     tdSql.checkData(0,3,'ready')
+    #     tdSql.checkData(1,1,'%s:6130'%self.host)
+    #     tdSql.checkData(1,2,'follower')
+    #     tdSql.checkData(1,3,'ready')
+    #     tdSql.checkData(2,1,'%s:6230'%self.host)
+    #     tdSql.checkData(2,2,'offline')
+    #     tdSql.checkData(2,3,'ready')
+

tests/system-test/6-cluster/5dnode3mnodeStop.py

@@ -7,31 +7,25 @@ import os
 from util.log import *
 from util.sql import *
 from util.cases import *
+from util.dnodes import *
 from util.dnodes import TDDnodes
 from util.dnodes import TDDnode
+from util.cluster import *
+from test import tdDnodes
+
 import time
 import socket
 import subprocess
 from multiprocessing import Process
-class MyDnodes(TDDnodes):
-    def __init__(self ,dnodes_lists):
-        super(MyDnodes,self).__init__()
-        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
-        self.simDeployed = False
+
         
 class TDTestCase:
 
     def init(self,conn ,logSql):
         tdLog.debug(f"start to excute {__file__}")
-        self.TDDnodes = None
-
-    def buildcluster(self,dnodenumber):
-        self.depoly_cluster(dnodenumber)
-        self.master_dnode = self.TDDnodes.dnodes[0]
-        self.host=self.master_dnode.cfgDict["fqdn"]
-        conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
-        tdSql.init(conn1.cursor())
-        
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+
 
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -70,72 +64,34 @@ class TDTestCase:
             for i in range(4):
                 tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
 
-    def depoly_cluster(self ,dnodes_nums): 
-
-        testCluster = False
-        valgrind = 0  
-        hostname = socket.gethostname()
-        dnodes = []
-        start_port = 6030
-        for num in range(1, dnodes_nums+1):
-            dnode = TDDnode(num)
-            dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
-            dnode.addExtraCfg("fqdn", f"{hostname}")
-            dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
-            dnode.addExtraCfg("monitorFqdn", hostname)
-            dnode.addExtraCfg("monitorPort", 7043)
-            dnodes.append(dnode)
-        
-        self.TDDnodes = MyDnodes(dnodes)
-        self.TDDnodes.init("")
-        self.TDDnodes.setTestCluster(testCluster)
-        self.TDDnodes.setValgrind(valgrind)
-        self.TDDnodes.stopAll()
-        for dnode in self.TDDnodes.dnodes:
-            self.TDDnodes.deploy(dnode.index,{})
-            
-        for dnode in self.TDDnodes.dnodes:
-            self.TDDnodes.starttaosd(dnode.index)
-
-        # create cluster 
-        for dnode in self.TDDnodes.dnodes[1:]:
-            # tdLog.debug(dnode.cfgDict)
-            dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
-            dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
-            dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
-            cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
-            tdLog.debug(cmd)
-            os.system(cmd)
-        
-        time.sleep(2)
-        tdLog.info(" create cluster with %d dnode  done! " %dnodes_nums)
-
+    
     def check3mnode(self):
         count=0
         while count < 10:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :     
-                tdLog.debug("mnode is  three nodes")           
+                tdLog.debug("mnode is  three nodes")
+            else:
+                tdLog.exit("mnode number is correct")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='follower':
                     if  tdSql.queryResult[2][2]=='follower':
                         tdLog.debug("three mnodes is ready in 10s")
                         break
-            elif tdSql.queryResult[0][2]=='follower' :
-                if  tdSql.queryResult[1][2]=='leader':
+            elif tdSql.queryResult[1][2]=='leader' :
+                if  tdSql.queryResult[0][2]=='follower':
                     if  tdSql.queryResult[2][2]=='follower':
                         tdLog.debug("three mnodes is ready in 10s")
                         break      
-            elif tdSql.queryResult[0][2]=='follower' :
+            elif tdSql.queryResult[2][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='follower':
-                    if  tdSql.queryResult[2][2]=='leader':
+                    if  tdSql.queryResult[0][2]=='follower':
                         tdLog.debug("three mnodes is ready in 10s")
                         break                   
             count+=1
         else:
-            tdLog.debug("three mnodes is not ready in 10s ")
-            return -1
+            tdLog.exit("three mnodes is not ready in 10s ")
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -153,6 +109,8 @@ class TDTestCase:
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
                 tdLog.debug("mnode is  three nodes")
+            else:
+                tdLog.exit("mnode number is correct")
             if  tdSql.queryResult[0][2]=='offline' :
                 if  tdSql.queryResult[1][2]=='leader':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -164,8 +122,8 @@ class TDTestCase:
                         break
             count+=1
         else:
-            tdLog.debug("stop mnodes  on dnode 2 failed in 10s ")
-            return -1
+            tdLog.exit("stop mnodes  on dnode 2 failed in 10s ")
+            
         tdSql.error("drop mnode on dnode 1;")
 
         tdSql.query("show mnodes;")       
@@ -185,6 +143,8 @@ class TDTestCase:
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
                 tdLog.debug("mnode is  three nodes")
+            else:
+                tdLog.exit("mnode number is correct")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='offline':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -192,8 +152,8 @@ class TDTestCase:
                         break
             count+=1
         else:
-            tdLog.debug("stop mnodes  on dnode 2 failed in 10s ")
-            return -1
+            tdLog.exit("stop mnodes  on dnode 2 failed in 10s ")
+
         tdSql.error("drop mnode on dnode 2;")
 
         tdSql.query("show mnodes;")       
@@ -215,6 +175,8 @@ class TDTestCase:
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
                 tdLog.debug("mnode is  three nodes")
+            else:
+                tdLog.exit("mnode number is correct")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[2][2]=='offline':
                     if  tdSql.queryResult[1][2]=='follower':
@@ -222,8 +184,8 @@ class TDTestCase:
                         break
             count+=1
         else:
-            tdLog.debug("stop mnodes  on dnode 3 failed in 10s")
-            return -1
+            tdLog.exit("stop mnodes  on dnode 3 failed in 10s")
+
         tdSql.error("drop mnode on dnode 3;")
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -237,15 +199,30 @@ class TDTestCase:
         tdSql.checkData(2,2,'offline')
         tdSql.checkData(2,3,'ready')
 
-
+    
+    def check_dnodes_status(self,dnodeNumbers):
+        count=0
+        while count < 5:
+            tdSql.query("show dnodes")
+            # tdLog.debug(tdSql.queryResult)
+            status=0
+            for i in range(dnodeNumbers):
+                if tdSql.queryResult[i][4] == "ready":
+                    status+=1
+            tdLog.debug(status)
+            
+            if status == dnodeNumbers:
+                tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %dnodeNumbers)
+                break 
+            count+=1
+            time.sleep(1)
+        else:
+            tdLog.exit("create cluster with %d dnode but  check dnode not ready within 5s ! "%dnodeNumbers)
 
     def five_dnode_three_mnode(self,dnodenumber):
-        tdSql.query("show dnodes;")
-        tdSql.checkData(0,1,'%s:6030'%self.host)
-        tdSql.checkData(4,1,'%s:6430'%self.host)
-        tdSql.checkData(0,4,'ready')
-        tdSql.checkData(4,4,'ready')
+        self.check_dnodes_status(5)
         tdSql.query("show mnodes;")   
+        tdLog.debug(self.host)
         tdSql.checkRows(1)    
         tdSql.checkData(0,1,'%s:6030'%self.host)
         tdSql.checkData(0,2,'leader')
@@ -262,26 +239,32 @@ class TDTestCase:
         tdSql.error("create mnode on dnode 2")
 
         tdSql.query("show dnodes;")
-        tdLog.debug(tdSql.queryResult)
+        # tdLog.debug(tdSql.queryResult)
 
         tdLog.debug("stop and follower of mnode") 
-        self.TDDnodes.stoptaosd(2)
+        tdDnodes=cluster.dnodes
+        # tdLog.debug(tdDnodes[0])
+
+        tdDnodes[1].stoptaosd()
         self.check3mnode2off()
-        self.TDDnodes.starttaosd(2)
+        tdDnodes[1].starttaosd()
+        self.check3mnode()
 
-        self.TDDnodes.stoptaosd(3)
+        tdDnodes[2].stoptaosd()
         self.check3mnode3off()
-        self.TDDnodes.starttaosd(3)
+        tdDnodes[2].starttaosd()
+        self.check3mnode()
 
-        self.TDDnodes.stoptaosd(1)
+        tdDnodes[0].stoptaosd()
         self.check3mnode1off()
-        self.TDDnodes.starttaosd(1)
+        tdDnodes[0].starttaosd()
+        self.check3mnode()
 
         self.check3mnode()
         stopcount =0 
         while stopcount <= 2:
             for i in range(dnodenumber):
-                self.TDDnodes.stoptaosd(i+1)
+                tdDnodes[i].stoptaosd()
                 # if i == 1 :
                 #     self.check3mnode2off()
                 # elif i == 2 :
@@ -289,24 +272,14 @@ class TDTestCase:
                 # elif i == 0:
                 #     self.check3mnode1off()
 
-                self.TDDnodes.starttaosd(i+1)
+                tdDnodes[i].starttaosd()
                 # self.check3mnode()
             stopcount+=1
         self.check3mnode()
 
-
-    def getConnection(self, dnode):
-        host = dnode.cfgDict["fqdn"]
-        port = dnode.cfgDict["serverPort"]
-        config_dir = dnode.cfgDir
-        return taos.connect(host=host, port=int(port), config=config_dir)
-
-
     def run(self): 
-        # tdLog.debug(self.master_dnode.cfgDict)
-        self.buildcluster(5)
         self.five_dnode_three_mnode(5)
-
+ 
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

tests/system-test/fulltest.sh

@@ -110,11 +110,11 @@ python3 ./test.py -f 2-query/distribute_agg_avg.py
 python3 ./test.py -f 2-query/distribute_agg_stddev.py
 python3 ./test.py -f 2-query/twa.py
 
-python3 ./test.py -f 6-cluster/5dnode1mnode.py
-python3 ./test.py -f 6-cluster/5dnode2mnode.py
-#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py
-#python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py
-# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py
+python3 ./test.py -f 6-cluster/5dnode1mnode.py 
+python3 ./test.py -f 6-cluster/5dnode2mnode.py 
+python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5
+# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
+# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py 
 
 python3 ./test.py -f 7-tmq/basic5.py
 python3 ./test.py -f 7-tmq/subscribeDb.py

tests/system-test/test.py

@@ -28,6 +28,7 @@ sys.path.append("../pytest")
 from util.log import *
 from util.dnodes import *
 from util.cases import *
+from util.cluster import *
 
 import taos
 
@@ -58,10 +59,11 @@ if __name__ == "__main__":
     logSql = True
     stop = 0
     restart = False
+    dnodeNums = 1
     updateCfgDict = {}
     execCmd = ""
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:', [
-        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd'])
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums'])
     for key, value in opts:
         if key in ['-h', '--help']:
             tdLog.printNoPrefix(
@@ -77,6 +79,8 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-d update cfg dict, base64 json str')
             tdLog.printNoPrefix('-k not kill valgrind processer')
             tdLog.printNoPrefix('-e eval str to run')
+            tdLog.printNoPrefix('-N create dnodes numbers clusters')
+
             sys.exit(0)
 
         if key in ['-r', '--restart']: 
@@ -126,6 +130,9 @@ if __name__ == "__main__":
                 print('updateCfgDict convert fail.')
                 sys.exit(0)
 
+        if key in ['-N', '--dnodeNums']:
+            dnodeNums = value
+
     if not execCmd == "":
         tdDnodes.init(deployPath)
         print(execCmd)
@@ -232,11 +239,36 @@ if __name__ == "__main__":
                     updateCfgDict = ucase.updatecfgDict
             except:
                 pass
-        tdDnodes.deploy(1,updateCfgDict)
-        tdDnodes.start(1)
-
-        tdCases.logSql(logSql)
-
+        if dnodeNums == 1 :
+            tdDnodes.deploy(1,updateCfgDict)
+            tdDnodes.start(1)
+            tdCases.logSql(logSql)
+        else :
+            print("start cluster and  dnodes number")
+            
+            dnodeslist = cluster.configure_cluster(dnodes_nums=dnodeNums,independent=True)
+            tdDnodes = ClusterDnodes(dnodeslist)
+            tdDnodes.init(deployPath, masterIp)
+            tdDnodes.setTestCluster(testCluster)
+            tdDnodes.setValgrind(valgrind)
+            tdDnodes.stopAll()
+            for dnode in tdDnodes.dnodes:
+                tdDnodes.deploy(dnode.index,{})
+            for dnode in tdDnodes.dnodes:
+                tdDnodes.starttaosd(dnode.index)
+            tdCases.logSql(logSql)
+            conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+            print(tdDnodes.getSimCfgPath(),host)
+            cluster.create_dnode(conn)
+            try:
+                if cluster.check_dnode(conn) :
+                    print("check dnode ready")
+            except Exception as r:
+                print(r)
+            
+            
         if testCluster:
             tdLog.info("Procedures for testing cluster")
             if fileName == "all":
@@ -248,10 +280,12 @@ if __name__ == "__main__":
             conn = taos.connect(
                 host,
                 config=tdDnodes.getSimCfgPath())
+                
             if fileName == "all":
                 tdCases.runAllLinux(conn)
             else:
                 tdCases.runOneLinux(conn, fileName)
+            
         if restart:
             if fileName == "all":
                 tdLog.info("not need to query ")

taos-tools @ 28a49b44

-Subproject commit d3c29fb492514cbaf08cb533976121bff5d94dea
+Subproject commit 28a49b447f71c4f014ebbac858b7215b897d57fd