Merge branch '3.0' into enh/stopquery

Jenkinsfile2

@@ -4,11 +4,6 @@ import jenkins.model.CauseOfInterruption
 node {
 }
 
-win_test_stage = 0
-linux_ready = 0
-linux_node_ip = ""
-linux_node_pass = ""
-
 def abortPreviousBuilds() {
     def currentJobName = env.JOB_NAME
     def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@@ -43,10 +38,8 @@ def pre_test(){
     sh '''
         cd ${WK}
         git reset --hard
-        git fetch || git fetch
         cd ${WKC}
         git reset --hard
-        git fetch || git fetch
     '''
     script {
         if (env.CHANGE_TARGET == 'master') {
@@ -82,9 +75,11 @@ def pre_test(){
     if (env.CHANGE_URL =~ /\/TDengine\//) {
         sh '''
             cd ${WKC}
+            git remote prune origin
             git pull >/dev/null
             git log -5
             echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
+            echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log
             echo "community log: `git log -5`" >>${WKDIR}/jenkins.log
             git fetch origin +refs/pull/${CHANGE_ID}/merge
             git checkout -qf FETCH_HEAD
@@ -101,12 +96,14 @@ def pre_test(){
             git pull >/dev/null
             git log -5
             echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log
+            echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log
             echo "tdinternal log: `git log -5`" >>${WKDIR}/jenkins.log
             git fetch origin +refs/pull/${CHANGE_ID}/merge
             git checkout -qf FETCH_HEAD
             git log -5
             echo "tdinternal log merged: `git log -5`" >>${WKDIR}/jenkins.log
             cd ${WKC}
+            git remote prune origin
             git pull >/dev/null
             git log -5
             echo "community log: `git log -5`" >>${WKDIR}/jenkins.log
@@ -137,53 +134,51 @@ def pre_test_win(){
         set
         date /t
         time /t
-        rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug || exit 0
+        rd /s /Q %WIN_INTERNAL_ROOT%\\debug || exit 0
     '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+        cd %WIN_INTERNAL_ROOT%
         git reset --hard
-        git fetch || git fetch
     '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+        cd %WIN_COMMUNITY_ROOT%
         git reset --hard
-        git fetch || git fetch
     '''
     script {
         if (env.CHANGE_TARGET == 'master') {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                cd %WIN_INTERNAL_ROOT%
                 git checkout master
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                cd %WIN_COMMUNITY_ROOT%
                 git checkout master
             '''
         } else if(env.CHANGE_TARGET == '2.0') {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                cd %WIN_INTERNAL_ROOT%
                 git checkout 2.0
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                cd %WIN_COMMUNITY_ROOT%
                 git checkout 2.0
             '''
         } else if(env.CHANGE_TARGET == '3.0') {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                cd %WIN_INTERNAL_ROOT%
                 git checkout 3.0
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                cd %WIN_COMMUNITY_ROOT%
                 git checkout 3.0
             '''
         } else {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                cd %WIN_INTERNAL_ROOT%
                 git checkout develop
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                cd %WIN_COMMUNITY_ROOT%
                 git checkout develop
             '''
         }
@@ -191,36 +186,38 @@ def pre_test_win(){
     script {
         if (env.CHANGE_URL =~ /\/TDengine\//) {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                cd %WIN_INTERNAL_ROOT%
                 git pull
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                cd %WIN_COMMUNITY_ROOT%
+                git remote prune origin
                 git pull
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                cd %WIN_COMMUNITY_ROOT%
                 git fetch origin +refs/pull/%CHANGE_ID%/merge
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                cd %WIN_COMMUNITY_ROOT%
                 git checkout -qf FETCH_HEAD
             '''
         } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                cd %WIN_INTERNAL_ROOT%
                 git pull
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                cd %WIN_INTERNAL_ROOT%
                 git fetch origin +refs/pull/%CHANGE_ID%/merge
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                cd %WIN_INTERNAL_ROOT%
                 git checkout -qf FETCH_HEAD
             '''
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                cd %WIN_COMMUNITY_ROOT%
+                git remote prune origin
                 git pull
             '''
         } else {
@@ -230,27 +227,27 @@ def pre_test_win(){
         }
     }
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+        cd %WIN_INTERNAL_ROOT%
         git branch
         git log -5
     '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+        cd %WIN_COMMUNITY_ROOT%
         git branch
         git log -5
     '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+        cd %WIN_COMMUNITY_ROOT%
         git submodule update --init --recursive
     '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python
+        cd %WIN_CONNECTOR_ROOT%
         git branch
         git reset --hard
         git pull
     '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python
+        cd %WIN_CONNECTOR_ROOT%
         git log -5
     '''
 }
@@ -258,7 +255,7 @@ def pre_test_build_win() {
     bat '''
         echo "building ..."
         time /t
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+        cd %WIN_INTERNAL_ROOT%
         mkdir debug
         cd debug
         time /t
@@ -273,9 +270,9 @@ def pre_test_build_win() {
         time /t
     '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python
+        cd %WIN_CONNECTOR_ROOT%
         python -m pip install .
-        xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
+        xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
     '''
     return 1
 }
@@ -283,24 +280,22 @@ def run_win_ctest() {
     bat '''
         echo "windows ctest ..."
         time /t
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug
+        cd %WIN_INTERNAL_ROOT%\\debug
         ctest -j 1 || exit 7
         time /t
     '''
 }
 def run_win_test() {
-    echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}"
     bat '''
         echo "windows test ..."
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python
+        cd %WIN_CONNECTOR_ROOT%
         python -m pip install .
-        xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
+        xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
         ls -l C:\\Windows\\System32\\taos.dll
         time /t
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community\\tests\\system-test
-        echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''"
+        cd %WIN_SYSTEM_TEST_ROOT%
         echo "testing ..."
-        test-all.bat  "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}"
+        test-all.bat ci
         time /t
     '''
 }
@@ -319,23 +314,21 @@ pipeline {
             parallel {
                 stage('windows test') {
                     agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "}
+                    environment{
+                        WIN_INTERNAL_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal"
+                        WIN_COMMUNITY_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community"
+                        WIN_SYSTEM_TEST_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community\\tests\\system-test"
+                        WIN_CONNECTOR_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\taos-connector-python"
+                    }
                     steps {
                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                             timeout(time: 55, unit: 'MINUTES'){
                                 pre_test_win()
                                 pre_test_build_win()
                                 run_win_ctest()
-                                script {
-                                    while(linux_ready == 0) {
-                                        sleep(8)
-                                    }
-                                }
                                 run_win_test()
                             }
                         }
-                        script {
-                            win_test_stage = 1
-                        }
                     }
                 }
                 stage('linux test') {
@@ -345,19 +338,8 @@ pipeline {
                         changeRequest()
                     }
                     steps {
-                        script {
-                            linux_node_ip = sh (
-                                script: 'jq .ip /home/node_info.json | sed "s/\\\"//g"',
-                                returnStdout: true
-                            ).trim()
-                            linux_node_pass = sh (
-                                script: 'jq .password /home/node_info.json | sed "s/\\\"//g" |sed "s/\\!/^^^^^^^^\\!/g"',
-                                returnStdout: true
-                            ).trim()
-                            echo "${linux_node_ip}:${linux_node_pass}"
-                        }
                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                            timeout(time: 40, unit: 'MINUTES'){
+                            timeout(time: 120, unit: 'MINUTES'){
                                 pre_test()
                                 script {
                                     sh '''
@@ -368,11 +350,36 @@ pipeline {
                                         rm -f /tmp/cases.task
                                         ./collect_cases.sh -e
                                     '''
+                                    def extra_param = ""
+                                    def log_server_file = "/home/log_server.json"
+                                    def timeout_cmd = ""
+                                    if (fileExists(log_server_file)) {
+                                        def log_server_enabled = sh (
+                                            script: 'jq .enabled ' + log_server_file,
+                                            returnStdout: true
+                                        ).trim()
+                                        def timeout_param = sh (
+                                            script: 'jq .timeout ' + log_server_file,
+                                            returnStdout: true
+                                        ).trim()
+                                        if (timeout_param != "null" &&  timeout_param != "0") {
+                                            timeout_cmd = "timeout " + timeout_param
+                                        }
+                                        if (log_server_enabled == "1") {
+                                            def log_server = sh (
+                                                script: 'jq .server ' + log_server_file + ' | sed "s/\\\"//g"',
+                                                returnStdout: true
+                                            ).trim()
+                                            if (log_server != "null" && log_server != "") { 
+                                                extra_param = "-w " + log_server
+                                            }
+                                        }
+                                    }
                                     sh '''
                                         cd ${WKC}/tests/parallel_test
                                         export DEFAULT_RETRY_TIME=2
                                         date
-                                        timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480
+                                        ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + '''
                                     '''
                                 }
                             }
@@ -387,37 +394,8 @@ pipeline {
                                         cd ${WKC}/packaging
                                         ./release.sh -v cluster -n 3.0.0.100 -s static
                                     '''
-                                    sh '''
-                                        echo "install ..."
-                                        cd ${WKC}/release
-                                        tar xzf TDengine-enterprise-server-3.0.0.100-Linux-x64.tar.gz
-                                        cd TDengine-enterprise-server-3.0.0.100
-                                        service taosd stop || :
-                                        rm -rf /var/lib/taos
-                                        ./install.sh -e no
-                                    '''
-                                    sh '''
-                                        echo "checking ..."
-                                        which taos
-                                        which taosd
-                                        rm -rf ${WK}/debug
-                                        mv ${WKC}/debug ${WK}/
-                                    '''
-                                    sh '''
-                                        echo "install taospy ..."
-                                        cd ${WKPY}
-                                        pip3 install .
-                                    '''
-                                }
                                 }
                             }
-                        script {
-                            linux_ready = 1
-                        }
-                        script {
-                            while(win_test_stage == 0){
-                                sleep(12)
-                            }
                         }
                     }
                 }

examples/c/stream_demo.c

@@ -25,7 +25,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -68,6 +68,14 @@ int32_t init_env() {
     return -1;
   }
   taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists tu3 using st1 tags(3)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
   return 0;
 }
 
@@ -90,10 +98,9 @@ int32_t create_stream() {
   /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
   /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
-  pRes = taos_query(
-      pConn,
-      "create stream stream1 trigger window_close watermark 10s into outstb as select _wstartts, sum(k) from st1 "
-      "interval(10s) ");
+  pRes = taos_query(pConn,
+                    "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 partition "
+                    "by tbname interval(10s) ");
   if (taos_errno(pRes) != 0) {
     printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;

include/common/tcommon.h

@@ -42,16 +42,20 @@ enum {
 typedef enum EStreamType {
   STREAM_NORMAL = 1,
   STREAM_INVERT,
-  STREAM_REPROCESS,
+  STREAM_CLEAR,
   STREAM_INVALID,
   STREAM_GET_ALL,
   STREAM_DELETE,
   STREAM_RETRIEVE,
+  STREAM_PUSH_DATA,
+  STREAM_PUSH_EMPTY,
 } EStreamType;
 
 typedef struct {
+  SArray*   pGroupList;
   SArray*   pTableList;
   SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
+  bool      needSortTableByGroupId;
   void*     pTagCond;
   void*     pTagIndexCond;
   uint64_t  suid;

include/common/tdatablock.h

@@ -224,6 +224,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
 
 int32_t      assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
+int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src);
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
 SSDataBlock* createDataBlock();
 int32_t      blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
@@ -236,6 +237,8 @@ void        blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t*
 const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
 
 void blockDebugShowData(const SArray* dataBlocks, const char* flag);
+// for debug
+char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf);
 
 int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
                                     tb_uid_t suid);

include/common/tmsg.h

@@ -851,7 +851,6 @@ typedef struct {
 int32_t tSerializeSServerVerRsp(void* buf, int32_t bufLen, SServerVerRsp* pRsp);
 int32_t tDeserializeSServerVerRsp(void* buf, int32_t bufLen, SServerVerRsp* pRsp);
 
-
 typedef struct SQueryNodeAddr {
   int32_t nodeId;  // vgId or qnodeId
   SEpSet  epSet;
@@ -878,7 +877,6 @@ int32_t tSerializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp);
 int32_t tDeserializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp);
 void    tFreeSDnodeListRsp(SDnodeListRsp* pRsp);
 
-
 typedef struct {
   SArray* pArray;  // Array of SUseDbRsp
 } SUseDbBatchRsp;
@@ -1250,7 +1248,7 @@ typedef struct {
 } SVariablesInfo;
 
 typedef struct {
-  SArray *variables;  //SArray<SVariablesInfo>
+  SArray* variables;  // SArray<SVariablesInfo>
 } SShowVariablesRsp;
 
 int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pReq);
@@ -1258,7 +1256,6 @@ int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesR
 
 void tFreeSShowVariablesRsp(SShowVariablesRsp* pRsp);
 
-
 /*
  * sql: show tables like '%a_%'
  * payload is the query condition, e.g., '%a_%'
@@ -1287,6 +1284,7 @@ void    tFreeSShowRsp(SShowRsp* pRsp);
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
   char    tb[TSDB_TABLE_NAME_LEN];
+  char    user[TSDB_USER_LEN];
   int64_t showId;
 } SRetrieveTableReq;
 
@@ -1308,6 +1306,8 @@ typedef struct {
   int32_t compLen;
   int32_t numOfRows;
   int32_t numOfCols;
+  int64_t skey;
+  int64_t ekey;
   char    data[];
 } SRetrieveTableRsp;
 

include/libs/executor/executor.h

@@ -36,7 +36,7 @@ typedef struct SReadHandle {
   void*   vnode;
   void*   mnd;
   SMsgCb* pMsgCb;
-  int8_t  initTsdbReader;
+//  int8_t  initTsdbReader;
 } SReadHandle;
 
 enum {

include/libs/function/functionMgt.h

@@ -124,6 +124,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_BLOCK_DIST,       // block distribution aggregate function
   FUNCTION_TYPE_BLOCK_DIST_INFO,  // block distribution pseudo column function
   FUNCTION_TYPE_TO_COLUMN,
+  FUNCTION_TYPE_GROUP_KEY,
 
   // distributed splitting functions
   FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000,

include/libs/nodes/plannodes.h

@@ -75,6 +75,7 @@ typedef struct SScanLogicNode {
   double        filesFactor;
   SArray*       pSmaIndexes;
   SNodeList*    pPartTags;
+  bool          partSort;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -458,6 +459,7 @@ typedef struct SSubplan {
   int32_t        msgType;  // message type for subplan, used to denote the send message type to vnode.
   int32_t        level;    // the execution level of current subplan, starting from 0 in a top-down manner.
   char           dbFName[TSDB_DB_FNAME_LEN];
+  char           user[TSDB_USER_LEN];
   SQueryNodeAddr execNode;      // for the scan/modify subplan, the optional execution node
   SQueryNodeStat execNodeStat;  // only for scan subplan
   SNodeList*     pChildren;     // the datasource subplan,from which to fetch the result

include/libs/planner/planner.h

@@ -36,6 +36,7 @@ typedef struct SPlanContext {
   int64_t     watermark;
   char*       pMsg;
   int32_t     msgLen;
+  const char* pUser;
 } SPlanContext;
 
 // Create the physical plan for the query, according to the AST.

include/libs/qcom/query.h

@@ -16,6 +16,7 @@
 #ifndef _TD_QUERY_H_
 #define _TD_QUERY_H_
 
+// clang-foramt off
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -206,13 +207,15 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl
 char*   jobTaskStatusStr(int32_t status);
 
 SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
+
 void    destroyQueryExecRes(SQueryExecRes* pRes);
-int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len);
+int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
 char*   parseTagDatatoJson(void* p);
 int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
 int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);
 
-extern int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallocFp)(int32_t));
+extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen,
+                                          void* (*mallocFp)(int32_t));
 extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
 
 #define SET_META_TYPE_NULL(t)       (t) = META_TYPE_NULL_TABLE
@@ -231,11 +234,13 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define NEED_CLIENT_HANDLE_ERROR(_code)                                          \
   (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
    NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
-#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \
-  || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB)
+#define NEED_CLIENT_RM_TBLMETA_REQ(_type)                                                                  \
+  ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
+   (_type) == TDMT_VND_DROP_STB)
 
 #define NEED_SCHEDULER_RETRY_ERROR(_code)                                           \
-  ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
+  ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
+   (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
 
 #define REQUEST_TOTAL_EXEC_TIMES 2
 
@@ -312,3 +317,4 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #endif
 
 #endif /*_TD_QUERY_H_*/
+       // clang-foramt on

include/libs/stream/tstream.h

@@ -58,6 +58,7 @@ enum {
 enum {
   STREAM_INPUT__DATA_SUBMIT = 1,
   STREAM_INPUT__DATA_BLOCK,
+  STREAM_INPUT__DATA_RETRIEVE,
   STREAM_INPUT__TRIGGER,
   STREAM_INPUT__CHECKPOINT,
   STREAM_INPUT__DROP,
@@ -318,7 +319,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem
       return -1;
     }
     taosWriteQitem(pTask->inputQueue->queue, pSubmitClone);
-  } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
+  } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) {
     taosWriteQitem(pTask->inputQueue->queue, pItem);
   } else if (pItem->type == STREAM_INPUT__CHECKPOINT) {
     taosWriteQitem(pTask->inputQueue->queue, pItem);

include/libs/stream/tstreamUpdate.h

@@ -32,12 +32,15 @@ typedef struct SUpdateInfo {
   int64_t interval;
   int64_t watermark;
   TSKEY minTS;
+  SScalableBf* pCloseWinSBF;
 } SUpdateInfo;
 
 SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
 SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
 bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
 void updateInfoDestroy(SUpdateInfo *pInfo);
+void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo);
+void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo);
 
 #ifdef __cplusplus
 }

include/libs/sync/sync.h

@@ -200,6 +200,7 @@ const char* syncGetMyRoleStr(int64_t rid);
 SyncTerm    syncGetMyTerm(int64_t rid);
 SyncGroupId syncGetVgId(int64_t rid);
 void        syncGetEpSet(int64_t rid, SEpSet* pEpSet);
+void        syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet);
 int32_t     syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak);
 bool        syncEnvIsStart();
 const char* syncStr(ESyncState state);

include/libs/transport/trpc.h

@@ -34,10 +34,8 @@ extern int32_t tsRpcHeadSize;
 typedef struct {
   uint32_t clientIp;
   uint16_t clientPort;
-  union {
-    char    user[TSDB_USER_LEN];
   int64_t  applyIndex;
-  };
+  char     user[TSDB_USER_LEN];
 } SRpcConnInfo;
 
 typedef struct SRpcHandleInfo {

include/os/os.h

@@ -52,6 +52,7 @@ extern "C" {
 #endif
 #else
 
+#include <malloc.h>
 #include <time.h>
 #ifndef TD_USE_WINSOCK
 #include <winsock2.h>

include/os/osMath.h

@@ -25,11 +25,10 @@ extern "C" {
 
 #define TSWAP(a, b)                              \
   do {                                           \
-    char *__tmp = taosMemoryMalloc(sizeof(a));   \
+    char *__tmp = alloca(sizeof(a));             \
     memcpy(__tmp, &(a), sizeof(a));              \
     memcpy(&(a), &(b), sizeof(a));               \
     memcpy(&(b), __tmp, sizeof(a));              \
-    taosMemoryFree(__tmp);                       \
   } while (0)
 
 #ifdef WINDOWS

include/os/osSocket.h

@@ -157,7 +157,10 @@ int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
 int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
 void    taosWinSocketInit();
 
-int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
+/*
+ * set timeout(ms)
+ */
+int32_t taosCreateSocketWithTimeout(uint32_t timeout);
 
 TdSocketPtr       taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
 TdSocketPtr       taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);

include/util/ttrace.h

@@ -47,7 +47,9 @@ typedef struct STraceId {
 
 #define TRACE_TO_STR(traceId, buf)                              \
   do {                                                          \
-    sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", traceId->rootId, traceId->msgId); \
+    int64_t rootId = (traceId) != NULL ? (traceId)->rootId : 0; \
+    int64_t msgId = (traceId) != NULL ? (traceId)->msgId : 0;   \
+    sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", rootId, msgId); \
   } while (0)
 
 #ifdef __cplusplus

source/client/src/clientEnv.c

@@ -13,11 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
 #include "catalog.h"
-#include "functionMgt.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "functionMgt.h"
+#include "os.h"
 #include "query.h"
 #include "scheduler.h"
 #include "tcache.h"
@@ -38,7 +38,7 @@ static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
 volatile int32_t    tscInitRes = 0;
 
 static void registerRequest(SRequestObj *pRequest) {
-  STscObj *pTscObj = acquireTscObj(*(int64_t*)pRequest->pTscObj->id);
+  STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id);
 
   assert(pTscObj != NULL);
 
@@ -54,14 +54,14 @@ static void registerRequest(SRequestObj *pRequest) {
     int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
     tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
              ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
-             pRequest->self, *(int64_t*)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
+             pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
   }
 }
 
 static void deregisterRequest(SRequestObj *pRequest) {
   assert(pRequest != NULL);
 
-  STscObj          *pTscObj = pRequest->pTscObj;
+  STscObj *           pTscObj = pRequest->pTscObj;
   SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary;
 
   int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
@@ -70,8 +70,8 @@ static void deregisterRequest(SRequestObj *pRequest) {
   int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
   tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
            " ms, current:%d, app current:%d",
-           pRequest->self, *(int64_t*)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
-  releaseTscObj(*(int64_t*)pTscObj->id);
+           pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
+  releaseTscObj(*(int64_t *)pTscObj->id);
 }
 
 // todo close the transporter properly
@@ -80,12 +80,13 @@ void closeTransporter(STscObj *pTscObj) {
     return;
   }
 
-  tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t*)pTscObj->id);
+  tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id);
   rpcClose(pTscObj->pAppInfo->pTransporter);
 }
 
 static bool clientRpcRfp(int32_t code) {
-  if (code == TSDB_CODE_RPC_REDIRECT) {
+  if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
+      code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
     return true;
   } else {
     return false;
@@ -128,16 +129,17 @@ void closeAllRequests(SHashObj *pRequests) {
 void destroyTscObj(void *pObj) {
   STscObj *pTscObj = pObj;
 
-  SClientHbKey connKey = {.tscRid = *(int64_t*)pTscObj->id, .connType = pTscObj->connType};
+  SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType};
   hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
   int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
   closeAllRequests(pTscObj->pRequests);
   schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter);
   if (0 == connNum) {
     // TODO
-    //closeTransporter(pTscObj);
+    // closeTransporter(pTscObj);
   }
-  tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t*)pTscObj->id, pTscObj->pAppInfo->numOfConns);
+  tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id,
+           pTscObj->pAppInfo->numOfConns);
   taosThreadMutexDestroy(&pTscObj->mutex);
   taosMemoryFreeClear(pTscObj);
 }
@@ -167,10 +169,10 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c
 
   taosThreadMutexInit(&pObj->mutex, NULL);
   pObj->id = taosMemoryMalloc(sizeof(int64_t));
-  *(int64_t*)pObj->id = taosAddRef(clientConnRefPool, pObj);
+  *(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj);
   pObj->schemalessType = 1;
 
-  tscDebug("connObj created, 0x%" PRIx64, *(int64_t*)pObj->id);
+  tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id);
   return pObj;
 }
 
@@ -326,7 +328,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
     return 0;
   }
 
-  SConfig     *pCfg = taosGetCfg();
+  SConfig *    pCfg = taosGetCfg();
   SConfigItem *pItem = NULL;
 
   switch (option) {

source/client/src/clientImpl.c

@@ -376,7 +376,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
                       .pAstRoot = pQuery->pRoot,
                       .showRewrite = pQuery->showRewrite,
                       .pMsg = pRequest->msgBuf,
-                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
+                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
+                      .pUser = pRequest->pTscObj->user};
 
   return qCreateQueryPlan(&cxt, pPlan, pNodeList);
 }
@@ -495,8 +496,7 @@ _return:
   return TSDB_CODE_SUCCESS;
 }
 
-
-int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData *pResultMeta) {
+int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData* pResultMeta) {
   SArray* pDbVgList = NULL;
   SArray* pQnodeList = NULL;
   int32_t code = 0;
@@ -609,7 +609,6 @@ _return:
   return code;
 }
 
-
 int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
   tsem_init(&schdRspSem, 0, 0);
 
@@ -833,8 +832,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
     }
   }
 
-  tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64,
-             pRequest->self, code, tstrerror(code), pRequest->requestId);
+  tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
+           tstrerror(code), pRequest->requestId);
 
   STscObj* pTscObj = pRequest->pTscObj;
   if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) {
@@ -939,7 +938,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool val
   return launchQueryImpl(pRequest, pQuery, false, NULL);
 }
 
-void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta) {
+void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) {
   int32_t code = 0;
 
   switch (pQuery->execMode) {
@@ -960,7 +959,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM
                           .pAstRoot = pQuery->pRoot,
                           .showRewrite = pQuery->showRewrite,
                           .pMsg = pRequest->msgBuf,
-                          .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
+                          .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
+                          .pUser = pRequest->pTscObj->user};
 
       SAppInstInfo* pAppInfo = getAppInfo(pRequest);
       SQueryPlan* pDag = NULL;
@@ -1507,7 +1507,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows){
+static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
   char* p = (char*)pResultInfo->pData;
 
   int32_t  len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
@@ -1545,7 +1545,6 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
         } else {
           ASSERT(0);
         }
-
       }
     } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
       int32_t lenTmp = numOfRows * sizeof(int32_t);
@@ -1569,13 +1568,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
       break;
     }
   }
-  if(!needConvert) return TSDB_CODE_SUCCESS;
+  if (!needConvert) return TSDB_CODE_SUCCESS;
 
   char*   p = (char*)pResultInfo->pData;
   int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows);
 
   pResultInfo->convertJson = taosMemoryCalloc(1, dataLen);
-  if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
   char* p1 = pResultInfo->convertJson;
 
   int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t));
@@ -1644,7 +1643,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
           ASSERT(0);
         }
 
-        offset1[j]= len;
+        offset1[j] = len;
         memcpy(pStart1 + len, dst, varDataTLen(dst));
         len += varDataTLen(dst);
       }
@@ -1662,7 +1661,6 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
       pStart += len;
       pStart1 += len;
       memcpy(pStart1, pStart, colLen);
-
     }
     pStart += colLen;
     pStart1 += colLen1;
@@ -1730,7 +1728,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
     pStart += colLength[i];
   }
 
-  if(convertUcs4){
+  if (convertUcs4) {
     code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength);
   }
 
@@ -1847,15 +1845,16 @@ _OVER:
   return code;
 }
 
-int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, int32_t acctId, char* db) {
+int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str,
+                      int32_t acctId, char* db) {
   SName name;
 
   if (len1 <= 0) {
     return -1;
   }
 
-  const char *dbName = db;
-  const char *tbName = NULL;
+  const char* dbName = db;
+  const char* tbName = NULL;
   int32_t     dbLen = 0;
   int32_t     tbLen = 0;
   if (len2 > 0) {
@@ -1899,7 +1898,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName,
   memset(vPos, -1, sizeof(vPos));
   memset(vLen, 0, sizeof(vLen));
 
-  for (int32_t i = 0; ; ++i) {
+  for (int32_t i = 0;; ++i) {
     if (0 == *(tbList + i)) {
       if (vPos[vIdx] >= 0 && vLen[vIdx] <= 0) {
         vLen[vIdx] = i - vPos[vIdx];
@@ -1973,8 +1972,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName,
       continue;
     }
 
-    if (('a' <= *(tbList + i) && 'z' >= *(tbList + i)) || 
-        ('A' <= *(tbList + i) && 'Z' >= *(tbList + i)) ||
+    if (('a' <= *(tbList + i) && 'z' >= *(tbList + i)) || ('A' <= *(tbList + i) && 'Z' >= *(tbList + i)) ||
         ('0' <= *(tbList + i) && '9' >= *(tbList + i))) {
       if (vLen[vIdx] > 0) {
         goto _return;
@@ -2001,27 +1999,26 @@ _return:
 }
 
 void syncCatalogFn(SMetaData* pResult, void* param, int32_t code) {
-  SSyncQueryParam *pParam = param;
+  SSyncQueryParam* pParam = param;
   pParam->pRequest->code = code;
 
   tsem_post(&pParam->sem);
 }
 
-
-void syncQueryFn(void *param, void *res, int32_t code) {
-  SSyncQueryParam *pParam = param;
+void syncQueryFn(void* param, void* res, int32_t code) {
+  SSyncQueryParam* pParam = param;
   pParam->pRequest = res;
   pParam->pRequest->code = code;
 
   tsem_post(&pParam->sem);
 }
 
-void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly) {
-  STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
+void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
+  STscObj* pTscObj = acquireTscObj(*(int64_t*)taos);
   if (pTscObj == NULL || sql == NULL || NULL == fp) {
     terrno = TSDB_CODE_INVALID_PARA;
     if (pTscObj) {
-      releaseTscObj(*(int64_t *)taos);
+      releaseTscObj(*(int64_t*)taos);
     } else {
       terrno = TSDB_CODE_TSC_DISCONNECTED;
     }
@@ -2039,7 +2036,7 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void
     return;
   }
 
-  SRequestObj *pRequest = NULL;
+  SRequestObj* pRequest = NULL;
   int32_t      code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
@@ -2055,45 +2052,41 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void
   releaseTscObj(*(int64_t *)taos);  
 }
 
-
-TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly) {
+TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   if (NULL == taos) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
   }
 
-  STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
+  STscObj* pTscObj = acquireTscObj(*(int64_t*)taos);
   if (pTscObj == NULL || sql == NULL) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
   }
 
 #if SYNC_ON_TOP_OF_ASYNC
-  SSyncQueryParam *param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
   tsem_init(&param->sem, 0, 0);
 
   taosAsyncQueryImpl(taos, sql, syncQueryFn, param, validateOnly);
   tsem_wait(&param->sem);
 
-  releaseTscObj(*(int64_t *)taos);
+  releaseTscObj(*(int64_t*)taos);
 
   return param->pRequest;
 #else
   size_t sqlLen = strlen(sql);
   if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
-    releaseTscObj(*(int64_t *)taos);
+    releaseTscObj(*(int64_t*)taos);
     tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
     terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
     return NULL;
   }
 
-  TAOS_RES *pRes = execQuery(pTscObj, sql, sqlLen, validateOnly);
+  TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly);
 
-  releaseTscObj(*(int64_t *)taos);
+  releaseTscObj(*(int64_t*)taos);
 
   return pRes;
 #endif
 }
-
-
-

source/client/src/clientMain.c

@@ -199,10 +199,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   return pResInfo->userFields;
 }
 
-
-TAOS_RES *taos_query(TAOS *taos, const char *sql) {
-  return taosQueryImpl(taos, sql, false);
-}
+TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); }
 
 TAOS_ROW taos_fetch_row(TAOS_RES *res) {
   if (res == NULL) {
@@ -594,7 +591,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
 }
 
 int taos_validate_sql(TAOS *taos, const char *sql) {
-  TAOS_RES* pObj = taosQueryImpl(taos, sql, true);
+  TAOS_RES *pObj = taosQueryImpl(taos, sql, true);
 
   int code = taos_errno(pObj);
 
@@ -911,13 +908,13 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
     goto _return;
   }
 
-  SCatalog* pCtg = NULL;
+  SCatalog *pCtg = NULL;
   code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg);
   if (code != TSDB_CODE_SUCCESS) {
     goto _return;
   }
 
-  char* sql = "taos_load_table_info";
+  char *sql = "taos_load_table_info";
   code = buildRequest(pTscObj, sql, strlen(sql), &pRequest);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
@@ -928,9 +925,8 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   tsem_init(&param.sem, 0, 0);
   param.pRequest = pRequest;
 
-  SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
-                           .requestId = pRequest->requestId,
-                           .requestObjRefId = pRequest->self};
+  SRequestConnInfo conn = {
+      .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
 
   conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
 
@@ -951,7 +947,6 @@ _return:
   return code;
 }
 
-
 TAOS_STMT *taos_stmt_init(TAOS *taos) {
   STscObj *pObj = acquireTscObj(*(int64_t *)taos);
   if (NULL == pObj) {

source/common/src/tdatablock.c

@@ -1164,7 +1164,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows)
 
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
   int32_t code = 0;
-  ASSERT(numOfRows > 0);
+  // ASSERT(numOfRows > 0);
 
   if (numOfRows == 0) {
     return TSDB_CODE_SUCCESS;
@@ -1230,6 +1230,32 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
   return 0;
 }
 
+int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
+  ASSERT(src != NULL && dst != NULL);
+
+  blockDataCleanup(dst);
+  int32_t code = blockDataEnsureCapacity(dst, src->info.rows);
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    return code;
+  }
+
+  size_t numOfCols = taosArrayGetSize(src->pDataBlock);
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
+    SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
+    if (pSrc->pData == NULL) {
+      continue;
+    }
+
+    colDataAssign(pDst, pSrc, src->info.rows, &src->info);
+  }
+
+  dst->info.rows = src->info.rows;
+  dst->info.window = src->info.window;
+  return TSDB_CODE_SUCCESS;
+}
+
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
   if (pDataBlock == NULL) {
     return NULL;
@@ -1627,6 +1653,57 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
   }
 }
 
+// for debug
+char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) {
+  int32_t size = 2048;
+  *pDataBuf = taosMemoryCalloc(size, 1);
+  char*   dumpBuf = *pDataBuf;
+  char    pBuf[128] = {0};
+  int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
+  int32_t rows = pDataBlock->info.rows;
+  int32_t len = 0;
+  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag,
+                  (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
+  for (int32_t j = 0; j < rows; j++) {
+    len += snprintf(dumpBuf + len, size - len, "%s |", flag);
+    for (int32_t k = 0; k < colNum; k++) {
+      SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
+      void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+      if (colDataIsNull(pColInfoData, rows, j, NULL)) {
+        len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL");
+        continue;
+      }
+      switch (pColInfoData->info.type) {
+        case TSDB_DATA_TYPE_TIMESTAMP:
+          formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
+          len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf);
+          break;
+        case TSDB_DATA_TYPE_INT:
+          len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var);
+          break;
+        case TSDB_DATA_TYPE_UINT:
+          len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var);
+          break;
+        case TSDB_DATA_TYPE_BIGINT:
+          len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var);
+          break;
+        case TSDB_DATA_TYPE_UBIGINT:
+          len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var);
+          break;
+        case TSDB_DATA_TYPE_FLOAT:
+          len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var);
+          break;
+        case TSDB_DATA_TYPE_DOUBLE:
+          len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var);
+          break;
+      }
+    }
+    len += snprintf(dumpBuf + len, size - len, "\n");
+  }
+  len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag);
+  return dumpBuf;
+}
+
 /**
  * @brief TODO: Assume that the final generated result it less than 3M
  *

source/common/src/tglobal.c

@@ -963,6 +963,10 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) {
         tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32;
       } else if (strcasecmp("transPullupInterval", name) == 0) {
         tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
+      } else if (strcasecmp("ttlUnit", name) == 0) {
+        tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
+      } else if (strcasecmp("ttlPushInterval", name) == 0) {
+        tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32;
       } else if (strcasecmp("tmrDebugFlag", name) == 0) {
         tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
       } else if (strcasecmp("tsdbDebugFlag", name) == 0) {

source/common/src/tmsg.c

@@ -3017,6 +3017,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
   if (tEncodeI64(&encoder, pReq->showId) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -3032,6 +3033,8 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
   if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -247,8 +247,6 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) {
   return TSDB_CODE_SUCCESS;
 }
 
-
-
 int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   int32_t size = 0;
   int32_t rowsRead = 0;
@@ -259,6 +257,11 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return -1;
   }
 
+  if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
+    terrno = TSDB_CODE_MND_NO_RIGHTS;
+    return -1;
+  }
+
   if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
@@ -309,7 +312,6 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 SArray *dmGetMsgHandles() {
   int32_t code = -1;
   SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle));

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -70,9 +70,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
 }
 
 static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
-  SDnodeTrans  *pTrans = &pDnode->trans;
+  SDnodeTrans * pTrans = &pDnode->trans;
   int32_t       code = -1;
-  SRpcMsg      *pMsg = NULL;
+  SRpcMsg *     pMsg = NULL;
   SMgmtWrapper *pWrapper = NULL;
   SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
 
@@ -194,11 +194,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
 
   for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
-    SArray       *pArray = (*pWrapper->func.getHandlesFp)();
+    SArray *      pArray = (*pWrapper->func.getHandlesFp)();
     if (pArray == NULL) return -1;
 
     for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
-      SMgmtHandle  *pMgmt = taosArrayGet(pArray, i);
+      SMgmtHandle * pMgmt = taosArrayGet(pArray, i);
       SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
       if (pMgmt->needCheckVgId) {
         pHandle->needCheckVgId = pMgmt->needCheckVgId;
@@ -248,7 +248,14 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
   }
 }
 
-static bool rpcRfp(int32_t code) { return code == TSDB_CODE_RPC_REDIRECT; }
+static bool rpcRfp(int32_t code) {
+  if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
+      code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
+    return true;
+  } else {
+    return false;
+  }
+}
 
 int32_t dmInitClient(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;

source/dnode/mgmt/node_util/src/dmFile.c

@@ -133,10 +133,10 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
     ret = taosLockFile(pFile);
     if (ret == 0) break;
     terrno = TAOS_SYSTEM_ERROR(errno);
-    taosMsleep(100);
+    taosMsleep(1000);
     retryTimes++;
     dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes);
-  } while (retryTimes < 120);
+  } while (retryTimes < 12);
 
   if (ret < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);

source/dnode/mnode/impl/inc/mndAuth.h → source/dnode/mnode/impl/inc/mndPrivilege.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_MND_AUTH_H_
-#define _TD_MND_AUTH_H_
+#ifndef _TD_MND_PRIVILEGE_H
+#define _TD_MND_PRIVILEGE_H
 
 #include "mndInt.h"
 
@@ -23,13 +23,18 @@ extern "C" {
 #endif
 
 typedef enum {
-  MND_OPER_CREATE_USER = 1,
+  MND_OPER_CONNECT = 1,
+  MND_OPER_CREATE_ACCT,
+  MND_OPER_DROP_ACCT,
+  MND_OPER_ALTER_ACCT,
+  MND_OPER_CREATE_USER,
   MND_OPER_DROP_USER,
   MND_OPER_ALTER_USER,
   MND_OPER_CREATE_BNODE,
   MND_OPER_DROP_BNODE,
   MND_OPER_CREATE_DNODE,
   MND_OPER_DROP_DNODE,
+  MND_OPER_CONFIG_DNODE,
   MND_OPER_CREATE_MNODE,
   MND_OPER_DROP_MNODE,
   MND_OPER_CREATE_QNODE,
@@ -37,11 +42,14 @@ typedef enum {
   MND_OPER_CREATE_SNODE,
   MND_OPER_DROP_SNODE,
   MND_OPER_REDISTRIBUTE_VGROUP,
+  MND_OPER_MERGE_VGROUP,
   MND_OPER_SPLIT_VGROUP,
   MND_OPER_BALANCE_VGROUP,
   MND_OPER_CREATE_FUNC,
   MND_OPER_DROP_FUNC,
   MND_OPER_KILL_TRANS,
+  MND_OPER_KILL_CONN,
+  MND_OPER_KILL_QUERY,
   MND_OPER_CREATE_DB,
   MND_OPER_ALTER_DB,
   MND_OPER_DROP_DB,
@@ -49,18 +57,21 @@ typedef enum {
   MND_OPER_USE_DB,
   MND_OPER_WRITE_DB,
   MND_OPER_READ_DB,
+  MND_OPER_READ_OR_WRITE_DB,
+  MND_OPER_SHOW_VARIBALES,
 } EOperType;
 
-int32_t mndInitAuth(SMnode *pMnode);
-void    mndCleanupAuth(SMnode *pMnode);
+int32_t mndInitPrivilege(SMnode *pMnode);
+void    mndCleanupPrivilege(SMnode *pMnode);
 
-int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType);
-int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
-int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType);
-int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
+int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType);
+int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
+int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
+int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);
+int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_MND_AUTH_H_*/
+#endif /*_TD_MND_PRIVILEGE_H*/

source/dnode/mnode/impl/inc/mndStream.h

@@ -33,6 +33,10 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
 
 int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
 int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+// for sma
+// TODO refactor
+int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
+int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndAcct.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndAcct.h"
+#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndTrans.h"
 
@@ -212,18 +213,30 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) {
 }
 
 static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) {
+  if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) {
+    return -1;
+  }
+
   terrno = TSDB_CODE_MSG_NOT_PROCESSED;
   mError("failed to process create acct request since %s", terrstr());
   return -1;
 }
 
 static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) {
+  if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) {
+    return -1;
+  }
+
   terrno = TSDB_CODE_MSG_NOT_PROCESSED;
   mError("failed to process create acct request since %s", terrstr());
   return -1;
 }
 
 static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) {
+  if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) {
+    return -1;
+  }
+
   terrno = TSDB_CODE_MSG_NOT_PROCESSED;
   mError("failed to process create acct request since %s", terrstr());
   return -1;

source/dnode/mnode/impl/src/mndBnode.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndBnode.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndDnode.h"
 #include "mndShow.h"
 #include "mndTrans.h"
@@ -277,6 +277,9 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) {
   }
 
   mDebug("bnode:%d, start to create", createReq.dnodeId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) {
+    goto _OVER;
+  }
 
   pObj = mndAcquireBnode(pMnode, createReq.dnodeId);
   if (pObj != NULL) {
@@ -292,10 +295,6 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndCreateBnode(pMnode, pReq, pDnode, &createReq);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -383,6 +382,9 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) {
   }
 
   mDebug("bnode:%d, start to drop", dropReq.dnodeId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) {
+    goto _OVER;
+  }
 
   if (dropReq.dnodeId <= 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -394,10 +396,6 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndDropBnode(pMnode, pReq, pObj);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 

source/dnode/mnode/impl/src/mndConsumer.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndConsumer.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
@@ -431,6 +431,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
       goto SUBSCRIBE_OVER;
     }
 
+    if (mndCheckDbPrivilegeByName(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) {
+      goto SUBSCRIBE_OVER;
+    }
+
 #if 0
     // ref topic to prevent drop
     // TODO make topic complete

source/dnode/mnode/impl/src/mndDb.c

@@ -15,9 +15,9 @@
 
 #define _DEFAULT_SOURCE
 #include "mndDb.h"
-#include "mndAuth.h"
 #include "mndDnode.h"
 #include "mndOffset.h"
+#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndSma.h"
 #include "mndStb.h"
@@ -506,6 +506,9 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
   }
 
   mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups);
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) {
+    goto _OVER;
+  }
 
   pDb = mndAcquireDb(pMnode, createReq.db);
   if (pDb != NULL) {
@@ -526,10 +529,6 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) {
-    goto _OVER;
-  }
-
   code = mndCreateDb(pMnode, pReq, &createReq, pUser);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -700,7 +699,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) {
     goto _OVER;
   }
 
@@ -980,7 +979,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) {
     }
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) {
     goto _OVER;
   }
 
@@ -1127,7 +1126,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) {
 
       mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr());
     } else {
-      if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) {
+      if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) {
         goto _OVER;
       }
 
@@ -1252,7 +1251,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) {
     goto _OVER;
   }
 
@@ -1337,7 +1336,7 @@ char *buildRetension(SArray *pRetension) {
 }
 
 static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables,
-                           bool sysDb, ESdbStatus objStatus) {
+                           bool sysDb, ESdbStatus objStatus, bool sysinfo) {
   int32_t cols = 0;
 
   int32_t     bytes = pShow->pMeta->pSchemas[cols].bytes;
@@ -1355,7 +1354,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
   char statusB[24] = {0};
   STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status));
 
-  if (sysDb) {
+  if (sysDb || !sysinfo) {
     for (int32_t i = 0; i < pShow->numOfColumns; ++i) {
       SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
       if (i == 0) {
@@ -1529,17 +1528,21 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
   SDbObj    *pDb = NULL;
   ESdbStatus objStatus = 0;
 
+  SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
+  if (pUser == NULL) return 0;
+  bool sysinfo = pUser->sysInfo;
+
   // Append the information_schema database into the result.
   if (!pShow->sysDbRsp) {
     SDbObj infoschemaDb = {0};
     setInformationSchemaDbCfg(&infoschemaDb);
-    dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0);
+    dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0, 1);
 
     numOfRows += 1;
 
     SDbObj perfschemaDb = {0};
     setPerfSchemaDbCfg(&perfschemaDb);
-    dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0);
+    dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0, 1);
 
     numOfRows += 1;
     pShow->sysDbRsp = true;
@@ -1551,16 +1554,19 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
       break;
     }
 
+    if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) {
       int32_t numOfTables = 0;
       sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL);
 
-    dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus);
+      dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus, sysinfo);
       numOfRows++;
+    }
+
     sdbRelease(pSdb, pDb);
   }
 
   pShow->numOfRows += numOfRows;
-
+  mndReleaseUser(pMnode, pUser);
   return numOfRows;
 }
 

source/dnode/mnode/impl/src/mndDnode.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndDnode.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndMnode.h"
 #include "mndQnode.h"
 #include "mndShow.h"
@@ -560,6 +560,10 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
   SShowVariablesRsp rsp = {0};
   int32_t           code = -1;
 
+  if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIBALES) != 0) {
+    goto _OVER;
+  }
+
   rsp.variables = taosArrayInit(4, sizeof(SVariablesInfo));
   if (NULL == rsp.variables) {
     mError("failed to alloc SVariablesInfo array while process show variables req");
@@ -609,7 +613,6 @@ _OVER:
   return code;
 }
 
-
 static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
   SMnode         *pMnode = pReq->info.node;
   int32_t         code = -1;
@@ -622,6 +625,9 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
   }
 
   mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) {
+    goto _OVER;
+  }
 
   if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) {
     terrno = TSDB_CODE_MND_INVALID_DNODE_EP;
@@ -635,10 +641,6 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndCreateDnode(pMnode, pReq, &createReq);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -717,6 +719,9 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
   }
 
   mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
+    goto _OVER;
+  }
 
   pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId);
   if (pDnode == NULL) {
@@ -753,10 +758,6 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
     }
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -782,6 +783,10 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
   }
 
   mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) {
+    return -1;
+  }
+
   SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId);
   if (pDnode == NULL) {
     mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr());

source/dnode/mnode/impl/src/mndFunc.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndFunc.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndSync.h"
 #include "mndTrans.h"
@@ -283,6 +283,9 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
   }
 
   mDebug("func:%s, start to create", createReq.name);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) {
+    goto _OVER;
+  }
 
   pFunc = mndAcquireFunc(pMnode, createReq.name);
   if (pFunc != NULL) {
@@ -318,10 +321,6 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) {
-    goto _OVER;
-  }
-
   code = mndCreateFunc(pMnode, pReq, &createReq);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -347,6 +346,9 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
   }
 
   mDebug("func:%s, start to drop", dropReq.name);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) {
+    goto _OVER;
+  }
 
   if (dropReq.name[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_FUNC_NAME;
@@ -365,10 +367,6 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
     }
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) {
-    goto _OVER;
-  }
-
   code = mndDropFunc(pMnode, pReq, pFunc);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 

source/dnode/mnode/impl/src/mndMain.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndAcct.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndBnode.h"
 #include "mndCluster.h"
 #include "mndConsumer.h"
@@ -100,7 +100,7 @@ static void *mndThreadFp(void *param) {
     taosMsleep(100);
     if (mndGetStop(pMnode)) break;
 
-    if (lastTime % (tsTransPullupInterval * 10) == 1) {
+    if (lastTime % (tsTtlPushInterval * 10) == 1) {
       mndTtlTimer(pMnode);
     }
 
@@ -239,7 +239,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1;
-  if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
@@ -529,16 +529,22 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
   if (!IsReq(pMsg)) return 0;
   if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
   if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
-      pMsg->msgType == TDMT_MND_TRANS_TIMER || TDMT_MND_TTL_TIMER) {
+      pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER) {
     return -1;
   }
 
-  const STraceId *trace = &pMsg->info.traceId;
-  mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
-
   SEpSet epSet = {0};
   mndGetMnodeEpSet(pMsg->info.node, &epSet);
 
+  const STraceId *trace = &pMsg->info.traceId;
+  mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(),
+         TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse);
+
+  if (epSet.numOfEps > 0) {
+    for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+      mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);
+    }
+
     int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
     pMsg->info.rsp = rpcMallocCont(contLen);
     if (pMsg->info.rsp != NULL) {
@@ -548,6 +554,9 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
     } else {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
     }
+  } else {
+    terrno = TSDB_CODE_APP_NOT_READY;
+  }
 
   return -1;
 }

source/dnode/mnode/impl/src/mndMnode.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndMnode.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndDnode.h"
 #include "mndShow.h"
 #include "mndSync.h"
@@ -389,6 +389,9 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
   }
 
   mDebug("mnode:%d, start to create", createReq.dnodeId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) {
+    goto _OVER;
+  }
 
   pObj = mndAcquireMnode(pMnode, createReq.dnodeId);
   if (pObj != NULL) {
@@ -414,10 +417,6 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndCreateMnode(pMnode, pReq, pDnode, &createReq);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -495,7 +494,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
   {
     int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead);
     void   *pReq = taosMemoryMalloc(contLen);
-    tSerializeSSetStandbyReq((char*)pReq + sizeof(SMsgHead), contLen, &standbyReq);
+    tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq);
     SMsgHead *pHead = pReq;
     pHead->contLen = htonl(contLen);
     pHead->vgId = htonl(MNODE_HANDLE);
@@ -595,6 +594,9 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
   }
 
   mDebug("mnode:%d, start to drop", dropReq.dnodeId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
+    goto _OVER;
+  }
 
   if (dropReq.dnodeId <= 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -621,10 +623,6 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndDropMnode(pMnode, pReq, pObj);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 

source/dnode/mnode/impl/src/mndOffset.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndOffset.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
@@ -36,13 +36,15 @@ static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffse
 static int32_t mndProcessCommitOffsetReq(SRpcMsg *pReq);
 
 int32_t mndInitOffset(SMnode *pMnode) {
-  SSdbTable table = {.sdbType = SDB_OFFSET,
+  SSdbTable table = {
+      .sdbType = SDB_OFFSET,
       .keyType = SDB_KEY_BINARY,
       .encodeFp = (SdbEncodeFp)mndOffsetActionEncode,
       .decodeFp = (SdbDecodeFp)mndOffsetActionDecode,
       .insertFp = (SdbInsertFp)mndOffsetActionInsert,
       .updateFp = (SdbUpdateFp)mndOffsetActionUpdate,
-                     .deleteFp = (SdbDeleteFp)mndOffsetActionDelete};
+      .deleteFp = (SdbDeleteFp)mndOffsetActionDelete,
+  };
 
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq);
 

source/dnode/mnode/impl/src/mndAuth.c → source/dnode/mnode/impl/src/mndPrivilege.c

@@ -14,66 +14,15 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "mndAuth.h"
+#include "mndPrivilege.h"
+#include "mndDb.h"
 #include "mndUser.h"
 
-static int32_t mndProcessAuthReq(SRpcMsg *pReq);
+int32_t mndInitPrivilege(SMnode *pMnode) { return 0; }
 
-int32_t mndInitAuth(SMnode *pMnode) {
-  mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq);
-  return 0;
-}
-
-void mndCleanupAuth(SMnode *pMnode) {}
-
-static int32_t mndRetriveAuth(SMnode *pMnode, SAuthRsp *pRsp) {
-  SUserObj *pUser = mndAcquireUser(pMnode, pRsp->user);
-  if (pUser == NULL) {
-    *pRsp->secret = 0;
-    mError("user:%s, failed to auth user since %s", pRsp->user, terrstr());
-    return -1;
-  }
-
-  pRsp->spi = 1;
-  pRsp->encrypt = 0;
-  *pRsp->ckey = 0;
-
-  memcpy(pRsp->secret, pUser->pass, TSDB_PASSWORD_LEN);
-  mndReleaseUser(pMnode, pUser);
-
-  mDebug("user:%s, auth info is returned", pRsp->user);
-  return 0;
-}
-
-static int32_t mndProcessAuthReq(SRpcMsg *pReq) {
-  SAuthReq authReq = {0};
-  if (tDeserializeSAuthReq(pReq->pCont, pReq->contLen, &authReq) != 0) {
-    terrno = TSDB_CODE_INVALID_MSG;
-    return -1;
-  }
-
-  SAuthReq authRsp = {0};
-  memcpy(authRsp.user, authReq.user, TSDB_USER_LEN);
-
-  int32_t code = mndRetriveAuth(pReq->info.node, &authRsp);
-  mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->info.conn.user, authRsp.spi, authRsp.encrypt,
-         authRsp.user);
-
-  int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp);
-  void   *pRsp = rpcMallocCont(contLen);
-  if (pRsp == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  tSerializeSAuthReq(pRsp, contLen, &authRsp);
-
-  pReq->info.rsp = pRsp;
-  pReq->info.rspLen = contLen;
-  return code;
-}
+void mndCleanupPrivilege(SMnode *pMnode) {}
 
-int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) {
+int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType) {
   int32_t   code = 0;
   SUserObj *pUser = mndAcquireUser(pMnode, user);
 
@@ -93,16 +42,30 @@ int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) {
     goto _OVER;
   }
 
+  switch (operType) {
+    case MND_OPER_CONNECT:
+    case MND_OPER_CREATE_FUNC:
+    case MND_OPER_DROP_FUNC:
+    case MND_OPER_SHOW_VARIBALES:
+      break;
+    default:
       terrno = TSDB_CODE_MND_NO_RIGHTS;
       code = -1;
+  }
 
 _OVER:
   mndReleaseUser(pMnode, pUser);
   return code;
 }
 
-int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) {
+int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) {
+  if (pUser->superUser && pAlter->alterType != TSDB_ALTER_USER_PASSWD) {
+    terrno = TSDB_CODE_MND_NO_RIGHTS;
+    return -1;
+  }
+
   if (pOperUser->superUser) return 0;
+
   if (!pOperUser->enable) {
     terrno = TSDB_CODE_MND_USER_DISABLED;
     return -1;
@@ -118,7 +81,7 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserRe
   return -1;
 }
 
-int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType) {
+int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname) {
   int32_t   code = 0;
   SUserObj *pUser = mndAcquireUser(pMnode, user);
 
@@ -137,21 +100,41 @@ int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType) {
     goto _OVER;
   }
 
-  if (!pUser->sysInfo) {
-    terrno = TSDB_CODE_MND_NO_RIGHTS;
-    code = -1;
+  if (pUser->sysInfo) {
     goto _OVER;
   }
 
+  switch (showType) {
+    case TSDB_MGMT_TABLE_DB:
+    case TSDB_MGMT_TABLE_STB:
+    case TSDB_MGMT_TABLE_INDEX:
+    case TSDB_MGMT_TABLE_STREAMS:
+    case TSDB_MGMT_TABLE_CONSUMERS:
+    case TSDB_MGMT_TABLE_TOPICS:
+    case TSDB_MGMT_TABLE_SUBSCRIPTIONS:
+    case TSDB_MGMT_TABLE_FUNC:
+    case TSDB_MGMT_TABLE_QUERIES:
+    case TSDB_MGMT_TABLE_CONNS:
+    case TSDB_MGMT_TABLE_APPS:
+    case TSDB_MGMT_TABLE_TRANS:
+      code = 0;
+      break;
+    default:
       terrno = TSDB_CODE_MND_NO_RIGHTS;
       code = -1;
+      goto _OVER;
+  }
+
+  if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP || showType == TSDB_MGMT_TABLE_INDEX) {
+    code = mndCheckDbPrivilegeByName(pMnode, user, MND_OPER_READ_OR_WRITE_DB, dbname);
+  }
 
 _OVER:
   mndReleaseUser(pMnode, pUser);
   return code;
 }
 
-int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) {
+int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) {
   int32_t   code = 0;
   SUserObj *pUser = mndAcquireUser(pMnode, user);
 
@@ -172,19 +155,11 @@ int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDb
     if (pUser->sysInfo) goto _OVER;
   }
 
-  if (operType == MND_OPER_ALTER_DB) {
-    if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
-  }
-
-  if (operType == MND_OPER_DROP_DB) {
+  if (operType == MND_OPER_ALTER_DB || operType == MND_OPER_DROP_DB || operType == MND_OPER_COMPACT_DB) {
     if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
   }
 
-  if (operType == MND_OPER_COMPACT_DB) {
-    if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER;
-  }
-
-  if (operType == MND_OPER_USE_DB) {
+  if (operType == MND_OPER_USE_DB || operType == MND_OPER_READ_OR_WRITE_DB) {
     if (strcmp(pUser->user, pDb->createUser) == 0) goto _OVER;
     if (taosHashGet(pUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER;
     if (taosHashGet(pUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER;
@@ -207,3 +182,12 @@ _OVER:
   mndReleaseUser(pMnode, pUser);
   return code;
 }
+
+int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) {
+  SDbObj *pDb = mndAcquireDb(pMnode, dbname);
+  if (pDb == NULL) return -1;
+
+  int32_t code = mndCheckDbPrivilege(pMnode, user, operType, pDb);
+  mndReleaseDb(pMnode, pDb);
+  return code;
+}
\ No newline at end of file

source/dnode/mnode/impl/src/mndProfile.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndProfile.h"
+#include "mndPrivilege.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
@@ -217,36 +218,45 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
   SConnObj       *pConn = NULL;
   int32_t         code = -1;
   SConnectReq     connReq = {0};
-  char            ip[30] = {0};
+  char            ip[24] = {0};
   const STraceId *trace = &pReq->info.traceId;
 
   if (tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto CONN_OVER;
+    goto _OVER;
   }
 
   taosIp2String(pReq->info.conn.clientIp, ip);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) {
+    mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr());
+    goto _OVER;
+  }
 
   pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
   if (pUser == NULL) {
-    mGError("user:%s, failed to login while acquire user since %s", pReq->info.conn.user, terrstr());
-    goto CONN_OVER;
+    mGError("user:%s, failed to login from %s while acquire user since %s", pReq->info.conn.user, ip, terrstr());
+    goto _OVER;
   }
-  if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
-    mGError("user:%s, failed to auth while acquire user, input:%s", pReq->info.conn.user, connReq.passwd);
+
+  if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0) {
+    mGError("user:%s, failed to login from %s since invalid pass, input:%s", pReq->info.conn.user, ip, connReq.passwd);
     code = TSDB_CODE_RPC_AUTH_FAILURE;
-    goto CONN_OVER;
+    goto _OVER;
   }
 
   if (connReq.db[0]) {
-    char db[TSDB_DB_FNAME_LEN];
+    char db[TSDB_DB_FNAME_LEN] = {0};
     snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db);
     pDb = mndAcquireDb(pMnode, db);
     if (pDb == NULL) {
       terrno = TSDB_CODE_MND_INVALID_DB;
       mGError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db,
               terrstr());
-      goto CONN_OVER;
+      goto _OVER;
+    }
+
+    if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) != 0) {
+      goto _OVER;
     }
   }
 
@@ -254,7 +264,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
                         pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime);
   if (pConn == NULL) {
     mGError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr());
-    goto CONN_OVER;
+    goto _OVER;
   }
 
   SConnectRsp connectRsp = {0};
@@ -271,9 +281,9 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
   mndGetMnodeEpSet(pMnode, &connectRsp.epSet);
 
   int32_t contLen = tSerializeSConnectRsp(NULL, 0, &connectRsp);
-  if (contLen < 0) goto CONN_OVER;
+  if (contLen < 0) goto _OVER;
   void *pRsp = rpcMallocCont(contLen);
-  if (pRsp == NULL) goto CONN_OVER;
+  if (pRsp == NULL) goto _OVER;
   tSerializeSConnectRsp(pRsp, contLen, &connectRsp);
 
   pReq->info.rspLen = contLen;
@@ -283,7 +293,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
 
   code = 0;
 
-CONN_OVER:
+_OVER:
 
   mndReleaseUser(pMnode, pUser);
   mndReleaseDb(pMnode, pDb);
@@ -645,15 +655,6 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
   SMnode       *pMnode = pReq->info.node;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
-  SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
-  if (pUser == NULL) return 0;
-  if (!pUser->superUser) {
-    mndReleaseUser(pMnode, pUser);
-    terrno = TSDB_CODE_MND_NO_RIGHTS;
-    return -1;
-  }
-  mndReleaseUser(pMnode, pUser);
-
   SKillQueryReq killReq = {0};
   if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -661,6 +662,10 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
   }
 
   mInfo("kill query msg is received, queryId:%s", killReq.queryStrId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) {
+    return -1;
+  }
+
   int32_t  connId = 0;
   uint64_t queryId = 0;
   char    *p = strchr(killReq.queryStrId, ':');
@@ -690,21 +695,16 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) {
   SMnode       *pMnode = pReq->info.node;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
-  SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
-  if (pUser == NULL) return 0;
-  if (!pUser->superUser) {
-    mndReleaseUser(pMnode, pUser);
-    terrno = TSDB_CODE_MND_NO_RIGHTS;
-    return -1;
-  }
-  mndReleaseUser(pMnode, pUser);
-
   SKillConnReq killReq = {0};
   if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
   }
 
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) {
+    return -1;
+  }
+
   SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t));
   if (pConn == NULL) {
     mError("connId:%u, failed to kill connection, conn not exist", killReq.connId);
@@ -739,7 +739,6 @@ _over:
   return code;
 }
 
-
 static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode   *pMnode = pReq->info.node;
   SSdb     *pSdb = pMnode->pSdb;

source/dnode/mnode/impl/src/mndQnode.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndQnode.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndDnode.h"
 #include "mndShow.h"
 #include "mndTrans.h"
@@ -279,6 +279,9 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
   }
 
   mDebug("qnode:%d, start to create", createReq.dnodeId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) {
+    goto _OVER;
+  }
 
   pObj = mndAcquireQnode(pMnode, createReq.dnodeId);
   if (pObj != NULL) {
@@ -294,10 +297,6 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndCreateQnode(pMnode, pReq, pDnode, &createReq);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -391,6 +390,9 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
   }
 
   mDebug("qnode:%d, start to drop", dropReq.dnodeId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) {
+    goto _OVER;
+  }
 
   if (dropReq.dnodeId <= 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -402,10 +404,6 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndDropQnode(pMnode, pReq, pObj);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -418,13 +416,13 @@ _OVER:
   return code;
 }
 
-int32_t mndCreateQnodeList(SMnode       *pMnode, SArray** pList, int32_t limit) {
+int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
   SSdb      *pSdb = pMnode->pSdb;
   void      *pIter = NULL;
   SQnodeObj *pObj = NULL;
   int32_t    numOfRows = 0;
 
-  SArray* qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad));
+  SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad));
   if (NULL == qnodeList) {
     mError("failed to alloc epSet while process qnode list req");
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -457,7 +455,6 @@ int32_t mndCreateQnodeList(SMnode       *pMnode, SArray** pList, int32_t limit)
   return TSDB_CODE_SUCCESS;
 }
 
-
 static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
   int32_t       code = -1;
   SMnode       *pMnode = pReq->info.node;

source/dnode/mnode/impl/src/mndShow.c

@@ -16,7 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "mndShow.h"
 #include "systable.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 
 #define SHOW_STEP_SIZE 100
 
@@ -122,6 +122,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) {
   int32_t size = sizeof(SShowObj);
 
   SShowObj showObj = {0};
+
   showObj.id = showId;
   showObj.pMnode = pMnode;
   showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb));
@@ -230,8 +231,14 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
   }
 
   mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type);
-
-  // if (mndCheckShowAuth(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1;
+  if (retrieveReq.user[0] != 0) {
+    memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN);
+  } else {
+    memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN);
+  }
+  if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) {
+    return -1;
+  }
 
   int32_t      numOfCols = pShow->pMeta->numOfColumns;
   SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));

source/dnode/mnode/impl/src/mndSma.c

@@ -15,11 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "mndSma.h"
-#include "mndAuth.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndInfoSchema.h"
 #include "mndMnode.h"
+#include "mndPrivilege.h"
 #include "mndScheduler.h"
 #include "mndShow.h"
 #include "mndStb.h"
@@ -713,7 +713,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
     goto _OVER;
   }
 
@@ -857,6 +857,23 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p
   mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name);
   mndTransSetDbName(pTrans, pDb->name, NULL);
 
+  SStreamObj *pStream = mndAcquireStream(pMnode, pSma->name);
+  if (pStream == NULL || pStream->smaId != pSma->uid) {
+    sdbRelease(pMnode->pSdb, pStream);
+    goto _OVER;
+  } else {
+    if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
+      mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
+      sdbRelease(pMnode->pSdb, pStream);
+      goto _OVER;
+    }
+
+    // drop stream
+    if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
+      sdbRelease(pMnode->pSdb, pStream);
+      goto _OVER;
+    }
+  }
   if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
   if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
   if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
@@ -974,7 +991,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndSnode.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndSnode.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndDnode.h"
 #include "mndShow.h"
 #include "mndTrans.h"
@@ -285,6 +285,9 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
   }
 
   mDebug("snode:%d, start to create", createReq.dnodeId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) {
+    goto _OVER;
+  }
 
   pObj = mndAcquireSnode(pMnode, createReq.dnodeId);
   if (pObj != NULL) {
@@ -300,10 +303,6 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) {
-    goto _OVER;
-  }
-
   code = mndCreateSnode(pMnode, pReq, pDnode, &createReq);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -398,6 +397,9 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
   }
 
   mDebug("snode:%d, start to drop", dropReq.dnodeId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) {
+    goto _OVER;
+  }
 
   if (dropReq.dnodeId <= 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -409,10 +411,6 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) {
-    goto _OVER;
-  }
-
   // check deletable
   code = mndDropSnode(pMnode, pReq, pObj);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;

source/dnode/mnode/impl/src/mndStb.c

@@ -15,12 +15,12 @@
 
 #define _DEFAULT_SOURCE
 #include "mndStb.h"
-#include "mndAuth.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndInfoSchema.h"
 #include "mndMnode.h"
 #include "mndPerfSchema.h"
+#include "mndPrivilege.h"
 #include "mndScheduler.h"
 #include "mndShow.h"
 #include "mndSma.h"
@@ -107,7 +107,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
   int32_t funcNum = taosArrayGetSize(pStb->pFuncs);
   SDB_SET_INT32(pRaw, dataPos, funcNum, _OVER)
   for (int32_t i = 0; i < funcNum; ++i) {
-    char* func = taosArrayGet(pStb->pFuncs, i);
+    char *func = taosArrayGet(pStb->pFuncs, i);
     SDB_SET_BINARY(pRaw, dataPos, func, TSDB_FUNC_NAME_LEN, _OVER)
   }
 
@@ -876,7 +876,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
     goto _OVER;
   }
 
@@ -1396,7 +1396,6 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
   return 0;
 }
 
-
 static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp,
                                  int32_t *smaVer) {
   char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
@@ -1449,8 +1448,6 @@ static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *t
   return code;
 }
 
-
-
 static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont,
                                      int32_t *pLen) {
   int32_t       ret;
@@ -1607,7 +1604,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
     goto _OVER;
   }
 
@@ -1737,7 +1734,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
     goto _OVER;
   }
 
@@ -1866,7 +1863,6 @@ _OVER:
   return code;
 }
 
-
 int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp,
                            int32_t *pRspLen) {
   SSTbHbRsp hbRsp = {0};

source/dnode/mnode/impl/src/mndStream.c

@@ -14,10 +14,10 @@
  */
 
 #include "mndStream.h"
-#include "mndAuth.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndPrivilege.h"
 #include "mndScheduler.h"
 #include "mndShow.h"
 #include "mndStb.h"
@@ -321,6 +321,10 @@ FAIL:
 }
 
 int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) {
+  ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1);
+  if (pTask->isDataScan == 0 && pTask->sinkType == TASK_SINK__NONE) {
+    ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
+  }
   SEncoder encoder;
   tEncoderInit(&encoder, NULL, 0);
   tEncodeSStreamTask(&encoder, pTask);
@@ -433,10 +437,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) {
-    goto _OVER;
-  }
-
   int32_t numOfStbs = -1;
   if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) {
     goto _OVER;
@@ -490,7 +490,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
   return 0;
 }
 
-static int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
+int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
   int32_t lv = taosArrayGetSize(pStream->tasks);
   for (int32_t i = 0; i < lv; i++) {
     SArray *pTasks = taosArrayGetP(pStream->tasks, i);
@@ -538,19 +538,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  // TODO check read auth for source and write auth for target
-#if 0
-  pDb = mndAcquireDb(pMnode, createStreamReq.sourceDB);
-  if (pDb == NULL) {
-    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    goto _OVER;
-  }
-
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
-    goto _OVER;
-  }
-#endif
-
   // build stream obj from request
   SStreamObj streamObj = {0};
   if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) {
@@ -588,6 +575,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb) != 0) {
+    mndTransDrop(pTrans);
+    goto _OVER;
+  }
+
+  if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb) != 0) {
+    mndTransDrop(pTrans);
+    goto _OVER;
+  }
+
   // execute creation
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -630,20 +627,16 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
     if (dropReq.igNotExists) {
       mDebug("stream:%s, not exist, ignore not exist is set", dropReq.name);
       sdbRelease(pMnode->pSdb, pStream);
-      return -1;
+      return 0;
     } else {
       terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
       return -1;
     }
   }
 
-#if 0
-  // todo check auth
-  pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
-  if (pUser == NULL) {
-    goto DROP_STREAM_OVER;
+  if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
+    return -1;
   }
-#endif
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq);
   if (pTrans == NULL) {

source/dnode/mnode/impl/src/mndSync.c

@@ -198,6 +198,10 @@ int32_t mndInitSync(SMnode *pMnode) {
     return -1;
   }
 
+  // decrease election timer
+  setElectTimerMS(pMgmt->sync, 600);
+  setHeartbeatTimerMS(pMgmt->sync, 300);
+
   mDebug("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
   return 0;
 }
@@ -261,7 +265,8 @@ bool mndIsMaster(SMnode *pMnode) {
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
 
   if (!syncIsReady(pMgmt->sync)) {
-    terrno = TSDB_CODE_SYN_NOT_LEADER;
+    // get terrno from syncIsReady
+    // terrno = TSDB_CODE_SYN_NOT_LEADER;
     return false;
   }
 

source/dnode/mnode/impl/src/mndTopic.c

@@ -14,12 +14,12 @@
  */
 
 #include "mndTopic.h"
-#include "mndAuth.h"
 #include "mndConsumer.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
 #include "mndOffset.h"
+#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndStb.h"
 #include "mndSubscribe.h"
@@ -401,6 +401,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
     }
   } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
     SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName);
+    if (pStb == NULL) {
+      terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+      return -1;
+    }
     topicObj.stbUid = pStb->uid;
   }
   /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/
@@ -480,7 +484,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
+  if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) {
     goto _OVER;
   }
 
@@ -571,6 +575,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
   }
 #endif
 
+  if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) {
+    return -1;
+  }
+
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq);
   mndTransSetDbName(pTrans, pTopic->db, NULL);
   if (pTrans == NULL) {

source/dnode/mnode/impl/src/mndTrans.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndTrans.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndConsumer.h"
 #include "mndDb.h"
 #include "mndShow.h"
@@ -1384,8 +1384,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) {
   }
 
   mInfo("trans:%d, start to kill", killReq.transId);
-
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) {
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) {
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndUser.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndUser.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndDb.h"
 #include "mndShow.h"
 #include "mndTrans.h"
@@ -295,7 +295,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
   tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
   userObj.createdTime = taosGetTimestampMs();
   userObj.updateTime = userObj.createdTime;
-  userObj.superUser = pCreate->superUser;
+  userObj.superUser = 0;  // pCreate->superUser;
   userObj.sysInfo = pCreate->sysInfo;
   userObj.enable = pCreate->enable;
 
@@ -337,6 +337,9 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) {
   }
 
   mDebug("user:%s, start to create", createReq.user);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) {
+    goto _OVER;
+  }
 
   if (createReq.user[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
@@ -360,10 +363,6 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) {
-    goto _OVER;
-  }
-
   code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -466,7 +465,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckAlterUserAuth(pOperUser, pUser, &alterReq) != 0) {
+  if (mndCheckAlterUserPrivilege(pOperUser, pUser, &alterReq) != 0) {
     goto _OVER;
   }
 
@@ -631,6 +630,9 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) {
   }
 
   mDebug("user:%s, start to drop", dropReq.user);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) {
+    goto _OVER;
+  }
 
   if (dropReq.user[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
@@ -643,10 +645,6 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) {
-    goto _OVER;
-  }
-
   code = mndDropUser(pMnode, pReq, pUser);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 

source/dnode/mnode/impl/src/mndVgroup.c

@@ -15,7 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndVgroup.h"
-#include "mndAuth.h"
+#include "mndPrivilege.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
@@ -1212,8 +1212,9 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
   }
 
   mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3);
-
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) goto _OVER;
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) {
+    goto _OVER;
+  }
 
   pVgroup = mndAcquireVgroup(pMnode, req.vgId);
   if (pVgroup == NULL) goto _OVER;
@@ -1506,6 +1507,9 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) {
   SDbObj *pDb = NULL;
 
   mDebug("vgId:%d, start to split", vgId);
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) {
+    goto _OVER;
+  }
 
   pVgroup = mndAcquireVgroup(pMnode, vgId);
   if (pVgroup == NULL) goto _OVER;
@@ -1513,8 +1517,6 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) {
   pDb = mndAcquireDb(pMnode, pVgroup->dbName);
   if (pDb == NULL) goto _OVER;
 
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) goto _OVER;
-
   code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -1655,8 +1657,9 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
   }
 
   mInfo("start to balance vgroup");
-
-  if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER;
+  if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) {
+    goto _OVER;
+  }
 
   while (1) {
     SDnodeObj *pDnode = NULL;

source/dnode/mnode/impl/test/user/user.cpp

@@ -33,6 +33,8 @@ TEST_F(MndTestUser, 01_Show_User) {
 TEST_F(MndTestUser, 02_Create_User) {
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "");
     strcpy(createReq.pass, "p1");
 
@@ -47,6 +49,8 @@ TEST_F(MndTestUser, 02_Create_User) {
 
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "u1");
     strcpy(createReq.pass, "");
 
@@ -61,6 +65,8 @@ TEST_F(MndTestUser, 02_Create_User) {
 
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "root");
     strcpy(createReq.pass, "1");
 
@@ -75,6 +81,8 @@ TEST_F(MndTestUser, 02_Create_User) {
 
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "u1");
     strcpy(createReq.pass, "p1");
 
@@ -108,9 +116,11 @@ TEST_F(MndTestUser, 02_Create_User) {
 
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "u2");
     strcpy(createReq.pass, "p1");
-    createReq.superUser = 1;
+    createReq.superUser = 0;
 
     int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq);
     void*   pReq = rpcMallocCont(contLen);
@@ -144,9 +154,11 @@ TEST_F(MndTestUser, 02_Create_User) {
 TEST_F(MndTestUser, 03_Alter_User) {
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "u3");
     strcpy(createReq.pass, "p1");
-    createReq.superUser = 1;
+    createReq.superUser = 0;
 
     int32_t contLen = tSerializeSCreateUserReq(NULL, 0, &createReq);
     void*   pReq = rpcMallocCont(contLen);
@@ -225,7 +237,7 @@ TEST_F(MndTestUser, 03_Alter_User) {
     alterReq.alterType = TSDB_ALTER_USER_SUPERUSER;
     strcpy(alterReq.user, "u3");
     strcpy(alterReq.pass, "1");
-    alterReq.superUser = 1;
+    alterReq.superUser = 0;
 
     int32_t contLen = tSerializeSAlterUserReq(NULL, 0, &alterReq);
     void*   pReq = rpcMallocCont(contLen);
@@ -361,7 +373,7 @@ TEST_F(MndTestUser, 03_Alter_User) {
     SGetUserAuthRsp authRsp = {0};
     tDeserializeSGetUserAuthRsp(pRsp->pCont, pRsp->contLen, &authRsp);
     EXPECT_STREQ(authRsp.user, "u3");
-    EXPECT_EQ(authRsp.superAuth, 1);
+    EXPECT_EQ(authRsp.superAuth, 0);
     int32_t numOfReadDbs = taosHashGetSize(authRsp.readDbs);
     int32_t numOfWriteDbs = taosHashGetSize(authRsp.writeDbs);
     EXPECT_EQ(numOfReadDbs, 1);
@@ -436,6 +448,8 @@ TEST_F(MndTestUser, 05_Drop_User) {
 
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "u1");
     strcpy(createReq.pass, "p1");
 
@@ -468,6 +482,8 @@ TEST_F(MndTestUser, 05_Drop_User) {
 TEST_F(MndTestUser, 06_Create_Drop_Alter_User) {
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "u1");
     strcpy(createReq.pass, "p1");
 
@@ -482,6 +498,8 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) {
 
   {
     SCreateUserReq createReq = {0};
+    createReq.enable = 1;
+    createReq.sysInfo = 1;
     strcpy(createReq.user, "u2");
     strcpy(createReq.pass, "p2");
 

source/dnode/snode/src/snode.c

@@ -257,7 +257,7 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) {
     case TDMT_STREAM_TASK_RECOVER_RSP:
       return sndProcessTaskRecoverRsp(pSnode, pMsg);
     case TDMT_STREAM_RETRIEVE_RSP:
-      return sndProcessTaskRecoverRsp(pSnode, pMsg);
+      return sndProcessTaskRetrieveRsp(pSnode, pMsg);
     default:
       ASSERT(0);
   }

source/dnode/vnode/inc/vnode.h

@@ -116,7 +116,8 @@ typedef void *tsdbReaderT;
 #define BLOCK_LOAD_TABLE_SEQ_ORDER  2
 #define BLOCK_LOAD_TABLE_RR_ORDER   3
 
-tsdbReaderT  tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *tableInfoGroup, uint64_t qId,
+int32_t     tsdbSetTableList(tsdbReaderT reader, SArray *tableList);
+tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId,
                            uint64_t taskId);
 tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId,
                                void *pMemRef);
@@ -149,8 +150,7 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL
 int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReadHandle *pHandle);
 bool    tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids);
-int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
-                            int32_t *pNumOfRows);
+int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReadHandle *pHandle);
 
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -121,7 +121,7 @@ int         tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSub
 int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
                                 SSubmitBlkRsp* pRsp);
 int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
-tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
+tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId,
                            uint64_t taskId);
 tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
                                 void* pMemRef);

source/dnode/vnode/src/meta/metaTable.c

@@ -381,13 +381,14 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
   for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
     tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i);
     metaDropTableByUid(pMeta, *uid, NULL);
+    metaDebug("ttl drop table:%"PRId64, *uid);
   }
   metaULock(pMeta);
   return 0;
 }
 
 static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
-  int32_t    ttlDays;
+  int64_t    ttlDays;
   int64_t    ctime;
   if (pME->type == TSDB_CHILD_TABLE) {
     ctime = pME->ctbEntry.ctime;
@@ -443,7 +444,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
     // drop schema.db (todo)
   }
 
-  metaError("ttl drop table:%s", e.name);
   tDecoderClear(&dc);
   tdbFree(pData);
 
@@ -976,7 +976,9 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
   SDecoder       dc = {0};
 
   // get super table
-  tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData);
+  if(tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0){
+    return -1;
+  }
   tbDbKey.uid = pCtbEntry->ctbEntry.suid;
   tbDbKey.version = *(int64_t *)pData;
   tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData);

source/dnode/vnode/src/tq/tq.c

@@ -403,7 +403,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
             .reader = pHandle->execHandle.pExecReader[i],
             .meta = pTq->pVnode->pMeta,
             .vnode = pTq->pVnode,
-            .initTsdbReader = 1,
+//            .initTsdbReader = 1,
         };
         pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
         ASSERT(pHandle->execHandle.execCol.task[i]);
@@ -455,6 +455,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
   }
   tDecoderClear(&decoder);
   ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1);
+  if (pTask->isDataScan == 0 && pTask->sinkType == TASK_SINK__NONE) {
+    ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
+  }
 
   pTask->execStatus = TASK_EXEC_STATUS__IDLE;
 
@@ -476,7 +479,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
              .reader = pStreamReader,
              .meta = pTq->pVnode->pMeta,
              .vnode = pTq->pVnode,
-             .initTsdbReader = 1,
+//             .initTsdbReader = 1,
       };
       /*pTask->exec.inputHandle = pStreamReader;*/
       pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);

source/dnode/vnode/src/tq/tqExec.c

@@ -112,7 +112,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
     tqReadHandleSetMsg(pReader, pReq, 0);
     while (tqNextDataBlock(pReader)) {
       SSDataBlock block = {0};
-      if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
+      if (tqRetrieveDataBlock(&block, pReader) < 0) {
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
         ASSERT(0);
       }
@@ -129,7 +129,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
     tqReadHandleSetMsg(pReader, pReq, 0);
     while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
       SSDataBlock block = {0};
-      if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
+      if (tqRetrieveDataBlock(&block, pReader) < 0) {
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
         ASSERT(0);
       }

source/dnode/vnode/src/tq/tqRead.c

@@ -109,11 +109,15 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t
 }
 
 bool tqNextDataBlock(STqReadHandle* pHandle) {
+  if (pHandle->pMsg == NULL) return false;
   while (1) {
     if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
       return false;
     }
-    if (pHandle->pBlock == NULL) return false;
+    if (pHandle->pBlock == NULL) {
+      pHandle->pMsg = NULL;
+      return false;
+    }
 
     if (pHandle->tbIdHash == NULL) {
       return true;
@@ -142,10 +146,7 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
   return false;
 }
 
-int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid,
-                            int32_t* pNumOfRows) {
-  *pUid = 0;
-
+int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) {
   // TODO: cache multiple schema
   int32_t sversion = htonl(pHandle->pBlock->sversion);
   if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
@@ -176,7 +177,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_
   STSchema*       pTschema = pHandle->pSchema;
   SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
 
-  *pNumOfRows = pHandle->msgIter.numOfRows;
   int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
 
   if (colNumNeed == 0) {
@@ -217,22 +217,22 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_
     }
   }
 
-  if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) {
+  if (blockDataEnsureCapacity(pBlock, pHandle->msgIter.numOfRows) < 0) {
     goto FAIL;
   }
 
   int32_t colActual = blockDataGetNumOfCols(pBlock);
 
-  // TODO in stream shuffle case, fetch groupId
-  *pGroupId = 0;
-
   STSRowIter iter = {0};
   tdSTSRowIterInit(&iter, pTschema);
   STSRow* row;
   int32_t curRow = 0;
 
   tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
-  *pUid = pHandle->msgIter.uid;  // set the uid of table for submit block
+
+  pBlock->info.groupId = 0;
+  pBlock->info.uid = pHandle->msgIter.uid;  // set the uid of table for submit block
+  pBlock->info.rows = pHandle->msgIter.numOfRows;
 
   while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
     tdSTSRowIterReset(&iter, row);
@@ -243,7 +243,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_
       if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
         break;
       }
-      if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {
+      if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) {
         goto FAIL;
       }
     }

source/dnode/vnode/src/tq/tqSink.c

@@ -18,31 +18,21 @@
 SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid,
                             const char* stbFullName, int32_t vgId) {
   SSubmitReq* ret = NULL;
+  SArray*     schemaReqs = NULL;
+  SArray*     schemaReqSz = NULL;
   SArray*     tagArray = taosArrayInit(1, sizeof(STagVal));
   if (!tagArray) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
-  // cal size
-  int32_t cap = sizeof(SSubmitReq);
   int32_t sz = taosArrayGetSize(pBlocks);
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
-    int32_t      rows = pDataBlock->info.rows;
-    // TODO min
-    int32_t rowSize = pDataBlock->info.rowSize;
-    int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
-    int32_t schemaLen = 0;
 
   if (createTb) {
-      SVCreateTbReq createTbReq = {0};
-      char*         cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
-      createTbReq.name = cname;
-      createTbReq.flags = 0;
-      createTbReq.type = TSDB_CHILD_TABLE;
-      createTbReq.ctb.suid = suid;
-
+    schemaReqs = taosArrayInit(sz, sizeof(void*));
+    schemaReqSz = taosArrayInit(sz, sizeof(int32_t));
+    for (int32_t i = 0; i < sz; i++) {
+      SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
       STagVal      tagVal = {
                .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
                .type = TSDB_DATA_TYPE_UBIGINT,
@@ -53,22 +43,62 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
       taosArrayPush(tagArray, &tagVal);
       tTagNew(tagArray, 1, false, &pTag);
       if (pTag == NULL) {
-        tdDestroySVCreateTbReq(&createTbReq);
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
         taosArrayDestroy(tagArray);
         return NULL;
       }
+
+      SVCreateTbReq createTbReq = {0};
+      createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
+      createTbReq.flags = 0;
+      createTbReq.type = TSDB_CHILD_TABLE;
+      createTbReq.ctb.suid = suid;
       createTbReq.ctb.pTag = (uint8_t*)pTag;
 
       int32_t code;
+      int32_t schemaLen;
       tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
-
-      tdDestroySVCreateTbReq(&createTbReq);
       if (code < 0) {
+        tdDestroySVCreateTbReq(&createTbReq);
         taosArrayDestroy(tagArray);
+        taosMemoryFreeClear(ret);
         return NULL;
       }
+
+      void* schemaStr = taosMemoryMalloc(schemaLen);
+      if (schemaStr == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return NULL;
       }
+      taosArrayPush(schemaReqs, &schemaStr);
+      taosArrayPush(schemaReqSz, &schemaLen);
 
+      SEncoder encoder = {0};
+      tEncoderInit(&encoder, schemaStr, schemaLen);
+      code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
+      if (code < 0) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return NULL;
+      }
+      tEncoderClear(&encoder);
+      tdDestroySVCreateTbReq(&createTbReq);
+    }
+  }
+  taosArrayDestroy(tagArray);
+
+  // cal size
+  int32_t cap = sizeof(SSubmitReq);
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i);
+    int32_t      rows = pDataBlock->info.rows;
+    // TODO min
+    int32_t rowSize = pDataBlock->info.rowSize;
+    int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);
+
+    int32_t schemaLen = 0;
+    if (createTb) {
+      schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i);
+    }
     cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen;
   }
 
@@ -99,55 +129,13 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
 
     int32_t schemaLen = 0;
     if (createTb) {
-      SVCreateTbReq createTbReq = {0};
-      char*         cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
-      createTbReq.name = cname;
-      createTbReq.flags = 0;
-      createTbReq.type = TSDB_CHILD_TABLE;
-      createTbReq.ctb.suid = suid;
-
-      STagVal tagVal = {
-          .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
-          .type = TSDB_DATA_TYPE_UBIGINT,
-          .i64 = (int64_t)pDataBlock->info.groupId,
-      };
-      taosArrayClear(tagArray);
-      taosArrayPush(tagArray, &tagVal);
-      STag* pTag = NULL;
-      tTagNew(tagArray, 1, false, &pTag);
-      if (pTag == NULL) {
-        tdDestroySVCreateTbReq(&createTbReq);
-        taosArrayDestroy(tagArray);
-        taosMemoryFreeClear(ret);
-        return NULL;
-      }
-      createTbReq.ctb.pTag = (uint8_t*)pTag;
-
-      int32_t code;
-      tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code);
-      if (code < 0) {
-        tdDestroySVCreateTbReq(&createTbReq);
-        taosArrayDestroy(tagArray);
-        taosMemoryFreeClear(ret);
-        return NULL;
-      }
-
-      SEncoder encoder = {0};
-      tEncoderInit(&encoder, blkSchema, schemaLen);
-      code = tEncodeSVCreateTbReq(&encoder, &createTbReq);
-      tEncoderClear(&encoder);
-      tdDestroySVCreateTbReq(&createTbReq);
-
-      if (code < 0) {
-        taosArrayDestroy(tagArray);
-        taosMemoryFreeClear(ret);
-        return NULL;
-      }
+      schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i);
+      void* schemaStr = taosArrayGetP(schemaReqs, i);
+      memcpy(blkSchema, schemaStr, schemaLen);
     }
     blkHead->schemaLen = htonl(schemaLen);
 
     STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen);
-
     for (int32_t j = 0; j < rows; j++) {
       SRowBuilder rb = {0};
       tdSRowInit(&rb, pTSchema->version);
@@ -175,7 +163,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
   }
 
   ret->length = htonl(ret->length);
-  taosArrayDestroy(tagArray);
+
+  if (schemaReqs) taosArrayDestroyP(schemaReqs, taosMemoryFree);
+  taosArrayDestroy(schemaReqSz);
+
   return ret;
 }
 

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -223,9 +223,8 @@ int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) {
   return rows;
 }
 
-static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableListInfo* pTableList) {
-  size_t tableSize = taosArrayGetSize(pTableList->pTableList);
-  assert(tableSize >= 1);
+static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, SArray* pTableList) {
+  size_t tableSize = taosArrayGetSize(pTableList);
 
   // allocate buffer in order to load data blocks from file
   SArray* pTableCheckInfo = taosArrayInit(tableSize, sizeof(STableCheckInfo));
@@ -235,7 +234,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
 
   // todo apply the lastkey of table check to avoid to load header file
   for (int32_t j = 0; j < tableSize; ++j) {
-    STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList->pTableList, j);
+    STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList, j);
 
     STableCheckInfo info = {.lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid};
     info.suid = pTsdbReadHandle->suid;
@@ -254,8 +253,6 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
               pTsdbReadHandle->idStr);
   }
 
-  // TODO  group table according to the tag value.
-  taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar);
   return pTableCheckInfo;
 }
 
@@ -497,8 +494,21 @@ static int32_t setCurrentSchema(SVnode* pVnode, STsdbReadHandle* pTsdbReadHandle
   return TSDB_CODE_SUCCESS;
 }
 
-tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
+int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList){
+  STsdbReadHandle* pTsdbReadHandle = reader;
+  if(pTsdbReadHandle->pTableCheckInfo) taosArrayDestroy(pTsdbReadHandle->pTableCheckInfo);
+  pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, tableList);
+  if (pTsdbReadHandle->pTableCheckInfo == NULL) {
+    return TSDB_CODE_TDB_OUT_OF_MEMORY;
+  }
+  return TDB_CODE_SUCCESS;
+}
+
+tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId,
                             uint64_t taskId) {
+  if(taosArrayGetSize(tableList) == 0){
+    return NULL;
+  }
   STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId);
   if (pTsdbReadHandle == NULL) {
     return NULL;
@@ -543,7 +553,7 @@ tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableLis
   }
 
   tsdbDebug("%p total numOfTable:%" PRIzu " in this query, table %" PRIzu " %s", pTsdbReadHandle,
-            taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList->pTableList),
+            taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList),
             pTsdbReadHandle->idStr);
 
   return (tsdbReaderT)pTsdbReadHandle;
@@ -639,7 +649,7 @@ tsdbReaderT tsdbQueryLastRow(SVnode* pVnode, SQueryTableDataCond* pCond, STableL
     return NULL;
   }
 
-  STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbReaderOpen(pVnode, pCond, pList, qId, taskId);
+  STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbReaderOpen(pVnode, pCond, pList->pTableList, qId, taskId);
   if (pTsdbReadHandle == NULL) {
     return NULL;
   }
@@ -3644,17 +3654,6 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
   }
 }
 
-static int tsdbCheckInfoCompar(const void* key1, const void* key2) {
-  if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) {
-    return -1;
-  } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) {
-    return 1;
-  } else {
-    ASSERT(false);
-    return 0;
-  }
-}
-
 static void* doFreeColumnInfoData(SArray* pColumnInfoData) {
   if (pColumnInfoData == NULL) {
     return NULL;

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -144,11 +144,15 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
       vnodeAccumBlockMsg(pVnode, pMsg->msgType);
     } else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) {
       SEpSet newEpSet = {0};
+      syncGetRetryEpSet(pVnode->sync, &newEpSet);
+
+      /*
       syncGetEpSet(pVnode->sync, &newEpSet);
       SEp *pEp = &newEpSet.eps[newEpSet.inUse];
       if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
         newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
       }
+      */
 
       vGTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps,
               newEpSet.inUse);

source/libs/executor/inc/executorimpl.h

@@ -273,6 +273,10 @@ typedef struct STableScanInfo {
 
   SSampleExecInfo sample;       // sample execution info
   int32_t         curTWinIdx;
+
+  int32_t         currentGroupId;
+  uint64_t        queryId;
+  uint64_t        taskId;
 } STableScanInfo;
 
 typedef struct STagScanInfo {
@@ -289,6 +293,7 @@ typedef enum EStreamScanMode {
   STREAM_SCAN_FROM_RES,
   STREAM_SCAN_FROM_UPDATERES,
   STREAM_SCAN_FROM_DATAREADER,
+  STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
 } EStreamScanMode;
 
 typedef struct SCatchSupporter {
@@ -344,7 +349,9 @@ typedef struct SStreamBlockScanInfo {
   SArray*        childIds;
   SessionWindowSupporter sessionSup;
   bool            assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
-  int32_t         scanWinIndex;
+  int32_t         scanWinIndex;   // for state operator
+  int32_t         pullDataResIndex;
+  SSDataBlock*    pPullDataRes;             // pull data SSDataBlock
 } SStreamBlockScanInfo;
 
 typedef struct SSysTableScanInfo {
@@ -354,6 +361,7 @@ typedef struct SSysTableScanInfo {
   tsem_t                 ready;
   SReadHandle            readHandle;
   int32_t                accountId;
+  const char*            pUser;
   bool                   showRewrite;
   SNode*                 pCondition;  // db_name filter condition, to discard data that are not in current database
   SMTbCursor*            pCur;        // cursor for iterate the local table meta store.
@@ -422,8 +430,13 @@ typedef struct SStreamFinalIntervalOperatorInfo {
   STimeWindowAggSupp twAggSup;
   SArray*            pChildren;
   SSDataBlock*       pUpdateRes;
+  bool               returnUpdate;
   SPhysiNode*        pPhyNode;           // create new child
   bool               isFinal;
+  SHashObj*          pPullDataMap;
+  SArray*            pPullWins;          // SPullWindowInfo
+  int32_t            pullIndex;
+  SSDataBlock*       pPullDataRes;
 } SStreamFinalIntervalOperatorInfo;
 
 typedef struct SAggOperatorInfo {
@@ -706,10 +719,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
 
 SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
 
-SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId);
 SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
                                          STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
                                            int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo);
@@ -749,8 +762,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
 SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode,
                                                SExecTaskInfo* pTaskInfo);
 
-SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle,
-    STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup);
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle,
+    STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId);
 
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, bool multigroupResult,
                                       SExecTaskInfo* pTaskInfo);
@@ -845,7 +858,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
 
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
 
-int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey);
+int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
+SSDataBlock* createPullDataBlock();
 
 #ifdef __cplusplus
 }

source/libs/executor/src/executil.c

@@ -287,6 +287,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){
 int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) {
   int32_t code = TSDB_CODE_SUCCESS;
   pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo));
+  if(pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
 
   uint64_t tableUid = pScanNode->uid;
 
@@ -338,6 +339,11 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
     STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0};
     taosArrayPush(pListInfo->pTableList, &info);
   }
+  pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
+  if(pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+
+  //put into list as default group, remove it if grouping sorting is required later
+  taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList);
 
   return code;
 }

source/libs/executor/src/executor.c

@@ -145,10 +145,10 @@ static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo,
       continue;
     }
 
-    ASSERT(mr.me.type == TSDB_CHILD_TABLE);
-    if (mr.me.ctbEntry.suid != pScanInfo->tableUid) {
+    if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) {
       continue;
     }
+    // TODO handle ntb case
 
     taosArrayPush(qa, id);
   }

source/libs/executor/src/executorimpl.c

@@ -3862,9 +3862,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
   return pTaskInfo;
 }
 
-static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                      STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId);
-
 static SArray* extractColumnInfo(SNodeList* pNodeList);
 
 int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) {
@@ -3895,8 +3892,67 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey) {
-  if (groupKey == NULL) {
+static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum){
+  taosArrayClear(pTableListInfo->pGroupList);
+  SArray *sortSupport = taosArrayInit(groupNum, sizeof(uint64_t));
+  if(sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
+    STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
+    uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t));
+
+    int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ);
+    if (index == -1){
+      void *p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT);
+      SArray *tGroup = taosArrayInit(8, sizeof(STableKeyInfo));
+      if(tGroup == NULL) {
+        taosArrayDestroy(sortSupport);
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      if(taosArrayPush(tGroup, info) == NULL){
+        qError("taos push info array error");
+        taosArrayDestroy(sortSupport);
+        return TSDB_CODE_QRY_APP_ERROR;
+      }
+      if(p == NULL){
+        if(taosArrayPush(sortSupport, groupId) != NULL){
+          qError("taos push support array error");
+          taosArrayDestroy(sortSupport);
+          return TSDB_CODE_QRY_APP_ERROR;
+        }
+        if(taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL){
+          qError("taos push group array error");
+          taosArrayDestroy(sortSupport);
+          return TSDB_CODE_QRY_APP_ERROR;
+        }
+      }else{
+        int32_t pos = TARRAY_ELEM_IDX(sortSupport, p);
+        if(taosArrayInsert(sortSupport, pos, groupId) == NULL){
+          qError("taos insert support array error");
+          taosArrayDestroy(sortSupport);
+          return TSDB_CODE_QRY_APP_ERROR;
+        }
+        if(taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL){
+          qError("taos insert group array error");
+          taosArrayDestroy(sortSupport);
+          return TSDB_CODE_QRY_APP_ERROR;
+        }
+      }
+    }else{
+      SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index);
+      if(taosArrayPush(tGroup, info) == NULL){
+        qError("taos push uid array error");
+        taosArrayDestroy(sortSupport);
+        return TSDB_CODE_QRY_APP_ERROR;
+      }
+    }
+
+  }
+  taosArrayDestroy(sortSupport);
+  return TDB_CODE_SUCCESS;
+}
+
+int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group) {
+  if (group == NULL) {
     return TDB_CODE_SUCCESS;
   }
 
@@ -3906,13 +3962,14 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
   }
   int32_t keyLen = 0;
   void*   keyBuf = NULL;
-  int32_t numOfGroupCols = taosArrayGetSize(groupKey);
-  for (int32_t j = 0; j < numOfGroupCols; ++j) {
-    SColumn* pCol = taosArrayGet(groupKey, j);
-    keyLen += pCol->bytes;  // actual data + null_flag
+
+  SNode*           node;
+  FOREACH(node, group) {
+    SExprNode *pExpr =  (SExprNode *)node;
+    keyLen += pExpr->resType.bytes;
   }
 
-  int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols;
+  int32_t nullFlagSize = sizeof(int8_t) * LIST_LENGTH(group);
   keyLen += nullFlagSize;
 
   keyBuf = taosMemoryCalloc(1, keyLen);
@@ -3920,103 +3977,110 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  int32_t groupNum = 0;
   for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
     STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
     SMetaReader    mr = {0};
     metaReaderInit(&mr, pHandle->meta, 0);
     metaGetTableEntryByUid(&mr, info->uid);
 
+    SNodeList *groupNew = nodesCloneList(group);
+
+    nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr);
     char* isNull = (char*)keyBuf;
-    char* pStart = (char*)keyBuf + sizeof(int8_t) * numOfGroupCols;
-    for (int32_t j = 0; j < numOfGroupCols; ++j) {
-      SColumn* pCol = taosArrayGet(groupKey, j);
-
-      if (strcmp(pCol->name, "tbname") == 0) {
-        isNull[i] = 0;
-        memcpy(pStart, mr.me.name, strlen(mr.me.name));
-        pStart += strlen(mr.me.name);
+    char* pStart = (char*)keyBuf + nullFlagSize;
+
+    SNode* pNode;
+    int32_t index = 0;
+    FOREACH(pNode, groupNew){
+      SNode*  pNew = NULL;
+      int32_t code = scalarCalculateConstants(pNode, &pNew);
+      if (TSDB_CODE_SUCCESS == code) {
+        REPLACE_NODE(pNew);
       } else {
-        STagVal tagVal = {0};
-        tagVal.cid = pCol->colId;
-        const char* p = metaGetTableTagVal(&mr.me, pCol->type, &tagVal);
-        if (p == NULL) {
-          isNull[j] = 1;
-          continue;
+        taosMemoryFree(keyBuf);
+        nodesClearList(groupNew);
+        return code;
       }
-        isNull[i] = 0;
-        if (pCol->type == TSDB_DATA_TYPE_JSON) {
-          //          int32_t dataLen = getJsonValueLen(pkey->pData);
-          //          memcpy(pStart, (pkey->pData), dataLen);
-          //          pStart += dataLen;
-        } else if (IS_VAR_DATA_TYPE(pCol->type)) {
-          memcpy(pStart, tagVal.pData, tagVal.nData);
-          pStart += tagVal.nData;
-          ASSERT(tagVal.nData <= pCol->bytes);
+
+      ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
+      SValueNode *pValue = (SValueNode *)pNew;
+
+      if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) {
+        isNull[index++] = 1;
+        continue;
+      } else {
+        isNull[index++] = 0;
+        char*       data = nodesGetValueFromNode(pValue);
+        if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){
+          int32_t len = getJsonValueLen(data);
+          memcpy(pStart, data, len);
+          pStart += len;
+        } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) {
+          memcpy(pStart, data, varDataTLen(data));
+          pStart += varDataTLen(data);
         } else {
-          memcpy(pStart, &(tagVal.i64), pCol->bytes);
-          pStart += pCol->bytes;
+          memcpy(pStart, data, pValue->node.resType.bytes);
+          pStart += pValue->node.resType.bytes;
         }
       }
     }
     int32_t   len = (int32_t)(pStart - (char*)keyBuf);
+    uint64_t groupId = calcGroupId(keyBuf, len);
+    taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t));
+    info->groupId = groupId;
+    groupNum++;
 
-    uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len);
-
-    if (!pGroupId) {
-      uint64_t tmpId = calcGroupId(keyBuf, len);
-      info->groupId = tmpId;
-      taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t));
-    } else {
-      info->groupId = *pGroupId;
-    }
-
+    nodesClearList(groupNew);
     metaReaderClear(&mr);
   }
   taosMemoryFree(keyBuf);
+
+  if(pTableListInfo->needSortTableByGroupId){
+    return sortTableGroup(pTableListInfo, groupNum);
+  }
+
   return TDB_CODE_SUCCESS;
 }
 
 SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle,
-                                  uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo) {
+                                  uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, const char* pUser) {
   int32_t type = nodeType(pPhyNode);
 
   if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) {
     if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
 
-      tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
-      if (pDataReader == NULL && terrno != 0) {
-        pTaskInfo->code = terrno;
-        return NULL;
-      }
-
-      int32_t code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
-      if (code) {
-        tsdbCleanupReadHandle(pDataReader);
-        pTaskInfo->code = terrno;
+      int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
+      if(code){
+        pTaskInfo->code = code;
         return NULL;
       }
-
-      SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
-      code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys);  // todo for json
-      taosArrayDestroy(groupKeys);
+      code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
       if (code) {
-        tsdbCleanupReadHandle(pDataReader);
         pTaskInfo->code = terrno;
         return NULL;
       }
 
-      SOperatorInfo*  pOperator = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
+      SOperatorInfo*  pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId);
       STableScanInfo* pScanInfo = pOperator->info;
       pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
       return pOperator;
 
     } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
       STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
-      createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
-      extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
-      SOperatorInfo* pOperator =
-          createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
+      int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
+      if(code){
+        return NULL;
+      }
+      code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
+      if (code) {
+        pTaskInfo->code = terrno;
+        return NULL;
+      }
+
+      SOperatorInfo*  pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
+
       STableScanInfo* pScanInfo = pOperator->info;
       pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
       return pOperator;
@@ -4025,52 +4089,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo);
 
     } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
-      SScanPhysiNode*      pScanPhyNode = (SScanPhysiNode*)pPhyNode;  // simple child table.
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
       STimeWindowAggSupp   twSup = {
             .waterMark = pTableScanNode->watermark,
             .calTrigger = pTableScanNode->triggerType,
             .maxTs = INT64_MIN,
       };
-      tsdbReaderT pDataReader = NULL;
-
       if (pHandle) {
-        if (pHandle->initTsdbReader) {
-          // for stream
-          ASSERT(pHandle->vnode);
-          pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
-        } else {
-          // for tq
-          ASSERT(pHandle->meta);
-          getTableList(pHandle->meta, pScanPhyNode, pTableListInfo);
-        }
-      }
-
-#if 0
-      if (pDataReader == NULL && terrno != 0) {
-        qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo));
-        // return NULL;
-      } else {
-        qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo));
-      }
-#endif
-
-      SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
-      int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys);  // todo for json
-      taosArrayDestroy(groupKeys);
-      if (code) {
-        tsdbCleanupReadHandle(pDataReader);
-        return NULL;
+        createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
       }
 
-      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup);
-
+      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId);
       return pOperator;
 
     } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
       SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode;
-      return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo);
-
+      return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo);
     } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
       STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
 
@@ -4118,7 +4152,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
         cond.suid = pBlockNode->suid;
         cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER;
       }
-      tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId);
+      tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId);
       cleanupQueryTableDataCond(&cond);
 
       return createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo);
@@ -4133,7 +4167,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
   SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES);
   for (int32_t i = 0; i < size; ++i) {
     SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
-    ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo);
+    ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pUser);
     if (ops[i] == NULL) {
       return NULL;
     }
@@ -4355,7 +4389,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle*
     goto _error;
   }
 
-  tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId);
+  tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId);
   cleanupQueryTableDataCond(&cond);
 
   return pReader;
@@ -4365,6 +4399,26 @@ _error:
   return NULL;
 }
 
+static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) {
+  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+    if (pOperator->numOfDownstream == 0) {
+      qError("failed to find stream scan operator");
+      return TSDB_CODE_QRY_APP_ERROR;
+    }
+
+    if (pOperator->numOfDownstream > 1) {
+      qError("join not supported for stream block scan");
+      return TSDB_CODE_QRY_APP_ERROR;
+    }
+    return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo);
+  } else {
+    SStreamBlockScanInfo* pInfo = pOperator->info;
+    ASSERT(pInfo->pSnapshotReadOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
+    *ppInfo = pInfo->pSnapshotReadOp->info;
+    return 0;
+  }
+}
+
 int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) {
   if (pNode->pChildren == NULL || LIST_LENGTH(pNode->pChildren) == 0) {
     if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == pNode->type) {
@@ -4387,29 +4441,19 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) {
   return -1;
 }
 
-int32_t doRebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle) {
-  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-    if (pOperator->numOfDownstream == 0) {
-      qError("failed to find stream scan operator");
-      return TSDB_CODE_QRY_APP_ERROR;
-    }
-
-    if (pOperator->numOfDownstream > 1) {
-      qError("join not supported for stream block scan");
-      return TSDB_CODE_QRY_APP_ERROR;
+int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) {
+  STableScanInfo* pTableScanInfo = NULL;
+  if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) {
+    return -1;
   }
-    return doRebuildReader(pOperator->pDownstream[0], plan, pHandle);
-  } else {
-    SStreamBlockScanInfo* pInfo = pOperator->info;
-    ASSERT(pInfo->pSnapshotReadOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
-    STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info;
 
-    tsdbCleanupReadHandle(pTableScanInfo->dataReader);
   STableScanPhysiNode* pNode = NULL;
   if (extractTableScanNode(plan->pNode, &pNode) < 0) {
     ASSERT(0);
   }
 
+  tsdbCleanupReadHandle(pTableScanInfo->dataReader);
+
   STableListInfo info = {0};
   pTableScanInfo->dataReader = doCreateDataReader(pNode, pHandle, &info, 0, 0);
   if (pTableScanInfo->dataReader == NULL) {
@@ -4417,7 +4461,7 @@ int32_t doRebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* p
     qError("failed to create data reader");
     return TSDB_CODE_QRY_APP_ERROR;
   }
-  }
+  // TODO: set uid and ts to data reader
   return 0;
 }
 
@@ -4556,7 +4600,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
   (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond;
   (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond;
   (*pTaskInfo)->pRoot =
-      createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList);
+      createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user);
 
   if (NULL == (*pTaskInfo)->pRoot) {
     code = (*pTaskInfo)->code;
@@ -4574,6 +4618,13 @@ _complete:
 static void doDestroyTableList(STableListInfo* pTableqinfoList) {
   taosArrayDestroy(pTableqinfoList->pTableList);
   taosHashCleanup(pTableqinfoList->map);
+  if(pTableqinfoList->needSortTableByGroupId){
+    for(int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++){
+      SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i);
+      taosArrayDestroy(tmp);
+    }
+  }
+  taosArrayDestroy(pTableqinfoList->pGroupList);
 
   pTableqinfoList->pTableList = NULL;
   pTableqinfoList->map = NULL;

source/libs/executor/src/scanoperator.c

@@ -418,7 +418,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
   return NULL;
 }
 
-static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
+static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
   STableScanInfo* pTableScanInfo = pOperator->info;
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
 
@@ -500,6 +500,49 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
     }
   }
 
+  return NULL;
+}
+
+static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
+  STableScanInfo* pInfo = pOperator->info;
+  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
+
+  if (pInfo->currentGroupId == -1) {
+    pInfo->currentGroupId++;
+    if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
+      setTaskStatus(pTaskInfo, TASK_COMPLETED);
+      return NULL;
+    }
+    SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
+    tsdbCleanupReadHandle(pInfo->dataReader);
+    tsdbReaderT* pReader =
+        tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId);
+    pInfo->dataReader = pReader;
+  }
+
+  SSDataBlock* result = doTableScanGroup(pOperator);
+  if (result) {
+    return result;
+  }
+
+  pInfo->currentGroupId++;
+  if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) {
+    setTaskStatus(pTaskInfo, TASK_COMPLETED);
+    return NULL;
+  }
+
+  SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId);
+  tsdbSetTableList(pInfo->dataReader, tableList);
+
+  tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
+  pInfo->curTWinIdx = 0;
+  pInfo->scanTimes = 0;
+
+  result = doTableScanGroup(pOperator);
+  if (result) {
+    return result;
+  }
+
   setTaskStatus(pTaskInfo, TASK_COMPLETED);
   return NULL;
 }
@@ -525,8 +568,8 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) {
   }
 }
 
-SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader,
-                                           SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
+                                           SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId) {
   STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
   SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -561,10 +604,12 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired;
   pInfo->pResBlock = createResDataBlock(pDescNode);
   pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
-  pInfo->dataReader = pDataReader;
   pInfo->scanFlag = MAIN_SCAN;
   pInfo->pColMatchInfo = pColList;
   pInfo->curTWinIdx = 0;
+  pInfo->queryId = queryId;
+  pInfo->taskId = taskId;
+  pInfo->currentGroupId = -1;
 
   pOperator->name = "TableScanOperator";  // for debug purpose
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
@@ -732,15 +777,14 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
   return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
 }
 
-static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
-  SSDataBlock* pSDB = pInfo->pUpdateRes;
+static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
   STimeWindow win = {
       .skey = INT64_MIN,
       .ekey = INT64_MAX,
   };
   bool needRead = false;
-  if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) {
-    SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, pInfo->primaryTsIndex);
+  if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) {
+    SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, tsColIndex);
     TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
     SResultRowInfo   dumyInfo;
     dumyInfo.cur.pageId = -1;
@@ -749,15 +793,14 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
       int64_t              gap = pInfo->sessionSup.gap;
       int32_t              winIndex = 0;
       SResultWindowInfo*   pCurWin =
-          getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
+          getSessionTimeWindow(pAggSup, tsCols[(*pRowIndex)], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
       win = pCurWin->win;
-      pInfo->updateResIndex +=
-          updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, pInfo->updateResIndex, gap, NULL);
+      (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL);
     } else {
-      win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[pInfo->updateResIndex], &pInfo->interval,
-                                pInfo->interval.precision, NULL);
-      pInfo->updateResIndex += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, pInfo->updateResIndex, win.ekey,
-                                                        binarySearchForKey, NULL, TSDB_ORDER_ASC);
+      win =
+          getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
+      (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
+                                               TSDB_ORDER_ASC);
     }
     needRead = true;
   } else if (isStateWindow(pInfo)) {
@@ -778,8 +821,12 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
   STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info;
   pTableScanInfo->cond.twindows[0] = win;
   pTableScanInfo->curTWinIdx = 0;
-  tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
+  //  tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
+  // if (!pTableScanInfo->dataReader) {
+  //   return false;
+  // }
   pTableScanInfo->scanTimes = 0;
+  pTableScanInfo->currentGroupId = -1;
   return true;
 }
 
@@ -817,12 +864,12 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_
   */
 }
 
-static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) {
+static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
   while (1) {
     SSDataBlock* pResult = NULL;
     pResult = doTableScan(pInfo->pSnapshotReadOp);
     if (pResult == NULL) {
-      if (prepareDataScan(pInfo)) {
+      if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) {
         // scan next window data
         pResult = doTableScan(pInfo->pSnapshotReadOp);
       }
@@ -871,7 +918,7 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
     pUpdateBlock->info.rows = i;
     pInfo->tsArrayIndex += i;
     pUpdateBlock->info.groupId = pInfo->groupId;
-    pUpdateBlock->info.type = STREAM_REPROCESS;
+    pUpdateBlock->info.type = STREAM_CLEAR;
     blockDataUpdateTsWindow(pUpdateBlock, 0);
   }
   // all rows have same group id
@@ -925,6 +972,14 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
     int32_t      current = pInfo->validBlockIndex++;
     SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
     blockDataUpdateTsWindow(pBlock, 0);
+    if (pBlock->info.type == STREAM_RETRIEVE) {
+      pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK;
+      pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
+      copyDataBlock(pInfo->pPullDataRes, pBlock);
+      pInfo->pullDataResIndex = 0;
+      prepareDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
+      updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
+    }
     return pBlock;
   } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
     if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
@@ -934,28 +989,39 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
     } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) {
       pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
       if (!isStateWindow(pInfo)) {
-        prepareDataScan(pInfo);
+        prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
       }
       return pInfo->pUpdateRes;
+    } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) {
+      SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
+      if (pSDB != NULL) {
+        getUpdateDataBlock(pInfo, true, pSDB, NULL);
+        pSDB->info.type = STREAM_PUSH_DATA;
+        return pSDB;
+      }
+      pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
     } else {
       if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) {
         pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
         pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
-        prepareDataScan(pInfo);
+        prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
       }
       if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
-        SSDataBlock* pSDB = doDataScan(pInfo);
+        SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
         if (pSDB == NULL) {
           setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes);
           if (pInfo->pUpdateRes->info.rows > 0) {
             if (!isStateWindow(pInfo)) {
-              prepareDataScan(pInfo);
+              // Todo(liuyao) mybe can delete this.
+              bool test = prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
+              ASSERT(test == false);
             }
             return pInfo->pUpdateRes;
           } else {
             pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
           }
         } else {
+          pSDB->info.type = STREAM_NORMAL;
           getUpdateDataBlock(pInfo, true, pSDB, NULL);
           return pSDB;
         }
@@ -967,12 +1033,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
 
     while (tqNextDataBlock(pInfo->streamBlockReader)) {
       SSDataBlock block = {0};
-      uint64_t    groupId = 0;
-      uint64_t    uid = 0;
-      int32_t     numOfRows = 0;
 
       // todo refactor
-      int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows);
+      int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader);
+
+      uint64_t groupId = block.info.groupId;
+      uint64_t uid = block.info.uid;
+      int32_t  numOfRows = block.info.rows;
 
       if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
         pTaskInfo->code = code;
@@ -1022,15 +1089,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
         }
       }
 
-      // TODO refactor @liao
       taosArrayDestroy(block.pDataBlock);
-
       if (pInfo->pRes->pDataBlock == NULL) {
         // TODO add log
+        updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
         pOperator->status = OP_EXEC_DONE;
         pTaskInfo->code = terrno;
         return NULL;
       }
+
       // currently only the tbname pseudo column
       if (pInfo->numOfPseudoExpr > 0) {
         addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes);
@@ -1048,12 +1115,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
     pOperator->resultInfo.totalRows += pBlockInfo->rows;
 
     if (pBlockInfo->rows == 0) {
+      updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
       pOperator->status = OP_EXEC_DONE;
     } else if (pInfo->pUpdateInfo) {
       pInfo->tsArrayIndex = 0;
       getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes);
       if (pInfo->pUpdateRes->info.rows > 0) {
-        if (pInfo->pUpdateRes->info.type == STREAM_REPROCESS) {
+        if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) {
           pInfo->updateResIndex = 0;
           pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
         } else if (pInfo->pUpdateRes->info.type == STREAM_INVERT) {
@@ -1087,9 +1155,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) {
   return tableIdList;
 }
 
-SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle,
-                                            STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo,
-                                            STimeWindowAggSupp* pTwSup) {
+SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
+                                            SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId,
+                                            uint64_t taskId) {
   SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
   SOperatorInfo*        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
 
@@ -1129,7 +1197,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan
   }
 
   if (pHandle) {
-    SOperatorInfo*  pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo);
+    SOperatorInfo*  pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId);
     STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info;
     if (pSTInfo->interval.interval > 0) {
       pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark);
@@ -1166,6 +1234,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan
   pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
   pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
   pInfo->groupId = 0;
+  pInfo->pPullDataRes = createPullDataBlock();
 
   pOperator->name = "StreamBlockScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
@@ -1529,6 +1598,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
     while (1) {
       int64_t startTs = taosGetTimestampUs();
       strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
+      strcpy(pInfo->req.user, pInfo->pUser);
 
 /*
       if (pInfo->showRewrite) {
@@ -1663,7 +1733,7 @@ int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbT
 }
 
 SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode,
-                                              SExecTaskInfo* pTaskInfo) {
+                                              const char* pUser, SExecTaskInfo* pTaskInfo) {
   SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo));
   SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -1679,6 +1749,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
   SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
 
   pInfo->accountId = pScanPhyNode->accountId;
+  pInfo->pUser = taosMemoryStrDup((void*)pUser);
   pInfo->showRewrite = pScanPhyNode->showRewrite;
   pInfo->pRes = pResBlock;
   pInfo->pCondition = pScanNode->node.pConditions;
@@ -1896,6 +1967,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
   pInfo->pRes = createResDataBlock(pDescNode);
   pInfo->readHandle = *pReadHandle;
   pInfo->curPos = 0;
+
   pOperator->name = "TagScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
 
@@ -1970,12 +2042,6 @@ typedef struct STableMergeScanInfo {
   SSampleExecInfo sample;  // sample execution info
 } STableMergeScanInfo;
 
-int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
-  const STableKeyInfo* info1 = p1;
-  const STableKeyInfo* info2 = p2;
-  return info1->groupId - info2->groupId;
-}
-
 int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
                                 STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) {
   int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo);
@@ -1987,55 +2053,25 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
     qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
     return TSDB_CODE_SUCCESS;
   }
-  SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
-  generateGroupIdMap(pTableListInfo, pHandle, groupKeys);  // todo for json
-  if (groupKeys) {
-    taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
-  }
-  taosArrayDestroy(groupKeys);
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                    STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId,
-                                    uint64_t taskId) {
-  SQueryTableDataCond cond = {0};
-  int32_t             code = initQueryTableDataCond(&cond, pTableScanNode);
+  code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags);
   if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-  for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); ++i) {
-    STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo));
-    subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
-    taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i));
-
-    tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, subListInfo, queryId, taskId);
-    taosArrayPush(arrayReader, &pReader);
-
-    taosArrayDestroy(subListInfo->pTableList);
-    taosMemoryFree(subListInfo);
+    return code;
   }
-  cleanupQueryTableDataCond(&cond);
 
   return TSDB_CODE_SUCCESS;
-
-_error:
-  return code;
 }
 
 int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo,
                                   int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId,
                                   uint64_t taskId) {
   for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) {
-    STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo));
-    subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
-    taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i));
+    SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo));
+    taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i));
 
-    tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subListInfo, queryId, taskId);
+    tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, queryId, taskId);
     taosArrayPush(arrayReader, &pReader);
 
-    taosArrayDestroy(subListInfo->pTableList);
-    taosMemoryFree(subListInfo);
+    taosArrayDestroy(subTableList);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -2407,6 +2443,12 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
+  const STableKeyInfo* info1 = p1;
+  const STableKeyInfo* info2 = p2;
+  return info1->groupId - info2->groupId;
+}
+
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
                                                 SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId,
                                                 uint64_t taskId) {
@@ -2415,6 +2457,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   if (pInfo == NULL || pOperator == NULL) {
     goto _error;
   }
+  if (pTableScanNode->pPartitionTags) {
+    taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
+  }
 
   SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
 

source/libs/function/inc/builtinsimpl.h

@@ -202,6 +202,10 @@ bool blockDistSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t blockDistFunction(SqlFunctionCtx *pCtx);
 int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 
+bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+int32_t groupKeyFunction(SqlFunctionCtx* pCtx);
+int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/function/src/builtins.c

@@ -1220,18 +1220,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 
 static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // The number of parameters has been limited by the syntax definition
-  uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+
   // The function return type has been set during syntax parsing
   uint8_t para2Type = pFunc->node.resType.type;
-  if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
-      para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
-      para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
-    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  }
-  if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
-      (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
-    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
-  }
+  //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
+  //    para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
+  //    para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
+  //  return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  //}
+  //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
+  //    (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
+  //  return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  //}
 
   int32_t para2Bytes = pFunc->node.resType.bytes;
   if (IS_VAR_DATA_TYPE(para2Type)) {
@@ -1279,8 +1280,7 @@ static bool validateTimestampDigits(const SValueNode* pVal) {
   int32_t tsDigits = (int32_t)strlen(fraction);
 
   if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
-    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS ||
-        tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
+    if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
         tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
       return true;
     } else {
@@ -1510,11 +1510,26 @@ static int32_t translateBlockDistInfoFunc(SFunctionNode* pFunc, char* pErrBuf, i
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateGroupKeyFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
+  return TSDB_CODE_SUCCESS;
+}
+
 static bool getBlockDistFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(STableBlockDistInfo);
   return true;
 }
 
+static int32_t translateGroupKey(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
+  pFunc->node.resType = ((SExprNode*)pPara)->resType;
+  return TSDB_CODE_SUCCESS;
+}
+
 // clang-format off
 const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
@@ -2499,7 +2514,19 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .type = FUNCTION_TYPE_BLOCK_DIST_INFO,
     .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC,
     .translateFunc = translateBlockDistInfoFunc,
-  }
+  },
+  {
+    .name = "_group_key",
+    .type = FUNCTION_TYPE_GROUP_KEY,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateGroupKey,
+    .getEnvFunc   = getGroupKeyFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = groupKeyFunction,
+    .finalizeFunc = groupKeyFinalize,
+    .pPartialFunc = "_group_key",
+    .pMergeFunc   = "_group_key"
+  },
 };
 // clang-format on