Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli

.gitignore

@@ -87,7 +87,7 @@ tests/comparisonTest/opentsdb/opentsdbtest/.settings/
 tests/examples/JDBC/JDBCDemo/.classpath
 tests/examples/JDBC/JDBCDemo/.project
 tests/examples/JDBC/JDBCDemo/.settings/
-source/libs/parser/inc/new_sql.*
+source/libs/parser/inc/sql.*
 
 # Emacs
 # -*- mode: gitignore; -*-

CMakeLists.txt

@@ -6,40 +6,14 @@ project(
     DESCRIPTION "An open-source big data platform designed and optimized for the Internet of Things(IOT)"
 )
 
-IF ("${BUILD_TOOLS}" STREQUAL "")
-  IF (TD_LINUX)
-    IF (TD_ARM_32)
-      SET(BUILD_TOOLS "false")
-    ELSEIF (TD_ARM_64)
-      SET(BUILD_TOOLS "false")
-    ELSE ()
-      SET(BUILD_TOOLS "false")
-    ENDIF ()
-  ELSEIF (TD_DARWIN)
-    SET(BUILD_TOOLS "false")
-  ELSE ()
-    SET(BUILD_TOOLS "false")
-  ENDIF ()
-ENDIF ()
-
-IF ("${BUILD_TOOLS}" MATCHES "false")
-    MESSAGE("${Yellow} Will _not_ build taos_tools! ${ColourReset}")
-    SET(TD_TAOS_TOOLS FALSE)
-ELSE ()
-    MESSAGE("")
-    MESSAGE("${Green} Will build taos_tools! ${ColourReset}")
-    MESSAGE("")
-    SET(TD_TAOS_TOOLS TRUE)
-ENDIF ()
-
 set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
 set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
+
+include(${CMAKE_SUPPORT_DIR}/cmake.platform)
+include(${CMAKE_SUPPORT_DIR}/cmake.define)
 include(${CMAKE_SUPPORT_DIR}/cmake.options)
 include(${CMAKE_SUPPORT_DIR}/cmake.version)
 
-SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
-
 # contrib
 add_subdirectory(contrib)
 
@@ -52,6 +26,7 @@ if(${BUILD_TEST})
     include(CTest)
     enable_testing()
 endif(${BUILD_TEST})
+
 add_subdirectory(source)
 add_subdirectory(tools)
 add_subdirectory(tests)

cmake/cmake.define

+cmake_minimum_required(VERSION 3.16)
+
+IF ("${BUILD_TOOLS}" STREQUAL "")
+  IF (TD_LINUX)
+    IF (TD_ARM_32)
+      SET(BUILD_TOOLS "false")
+    ELSEIF (TD_ARM_64)
+      SET(BUILD_TOOLS "false")
+    ELSE ()
+      SET(BUILD_TOOLS "false")
+    ENDIF ()
+  ELSEIF (TD_DARWIN)
+    SET(BUILD_TOOLS "false")
+  ELSE ()
+    SET(BUILD_TOOLS "false")
+  ENDIF ()
+ENDIF ()
+
+IF ("${BUILD_TOOLS}" MATCHES "false")
+    MESSAGE("${Yellow} Will _not_ build taos_tools! ${ColourReset}")
+    SET(TD_TAOS_TOOLS FALSE)
+ELSE ()
+    MESSAGE("")
+    MESSAGE("${Green} Will build taos_tools! ${ColourReset}")
+    MESSAGE("")
+    SET(TD_TAOS_TOOLS TRUE)
+ENDIF ()
+
+IF (TD_WINDOWS)
+    MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
+    SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
+    SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+
+    IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
+      SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
+    ENDIF ()
+
+ELSE ()
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
+
+ENDIF ()

cmake/cmake.options

 # =========================================================
 # Deps options
 # =========================================================
+
+IF(${TD_WINDOWS})
+
+    MESSAGE("build pthread Win32")
+    option(
+            BUILD_PTHREAD
+            "If build pthread on Windows"
+            ON
+          )
+
+    MESSAGE("build gnu regex for Windows")
+    option(
+            BUILD_GNUREGEX
+            "If build gnu regex on Windows"
+            ON
+          )
+
+ENDIF ()
+
+IF(${TD_LINUX} MATCHES TRUE)
+
 option(
     BUILD_TEST 
     "If build unit tests using googletest" 
     ON
 )
 
+ENDIF ()
+
 option(
     BUILD_WITH_LEVELDB 
     "If build with leveldb" 
@@ -25,11 +48,16 @@ option(
     OFF
 )
 
-option(
-    BUILD_WITH_BDB 
-    "If build with BerkleyDB" 
-    ON
-)
+IF(${TD_WINDOWS})
+    MESSAGE("Not build BDB on Windows")
+ELSE ()
+    option(
+            BUILD_WITH_BDB
+            "If build with BerkleyDB"
+            ON
+          )
+
+ENDIF ()
 
 option(
     BUILD_WITH_LUCENE 
@@ -68,12 +96,16 @@ option(
     OFF
 )
 
+IF(${TD_LINUX} MATCHES TRUE)
+
 option(
     BUILD_DEPENDENCY_TESTS 
     "If build dependency tests" 
     ON
 )
 
+ENDIF ()
+
 option(
     BUILD_DOCS
     "If use doxygen build documents" 

cmake/cmake.platform

+cmake_minimum_required(VERSION 3.16)
+
+MESSAGE("Current system is ${CMAKE_SYSTEM_NAME}")
+
+# init
+SET(TD_LINUX FALSE)
+SET(TD_WINDOWS FALSE)
+SET(TD_DARWIN FALSE)
+
+IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+
+    SET(TD_LINUX TRUE)
+    SET(OSTYPE "Linux")
+    ADD_DEFINITIONS("-DLINUX")
+
+    IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
+        SET(TD_LINUX_64 TRUE)
+    ELSE ()
+        SET(TD_LINUX_32 TRUE)
+    ENDIF ()
+
+ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+
+    SET(TD_DARWIN TRUE)
+    SET(OSTYPE "macOS")
+    ADD_DEFINITIONS("-DDARWIN")
+    IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64")
+        MESSAGE("Current system arch is arm64")
+        SET(TD_DARWIN_64 TRUE)
+        ADD_DEFINITIONS("-D_TD_DARWIN_64")
+    ENDIF ()
+
+    ADD_DEFINITIONS("-DHAVE_UNISTD_H")
+
+ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+
+    SET(TD_WINDOWS TRUE)
+    SET(OSTYPE "Windows")
+    ADD_DEFINITIONS("-DWINDOWS")
+
+    IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
+        SET(TD_WINDOWS_64 TRUE)
+    	ADD_DEFINITIONS("-D_TD_WINDOWS_64")
+    ELSE ()
+        SET(TD_WINDOWS_32 TRUE)
+    	ADD_DEFINITIONS("-D_TD_WINDOWS_32")
+    ENDIF ()
+ENDIF()
+
+MESSAGE("C Compiler ID: ${CMAKE_C_COMPILER_ID}")
+MESSAGE("CXX Compiler ID: ${CMAKE_CXX_COMPILER_ID}")

cmake/gnuregex_CMakeLists.txt.in

+
+# gnuregex
+ExternalProject_Add(gnuregex
+        URL https://launchpad.net/gnuregex/trunk/2.9/+download/libgnurx-src-2.9.zip
+        DOWNLOAD_NAME libgnurx-src.zip
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/gnuregex"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

cmake/pthread_CMakeLists.txt.in

+
+# pthread
+ExternalProject_Add(pthread
+        GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32
+        GIT_TAG v3.0.3.1
+        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/pthread-win32"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)

contrib/CMakeLists.txt

@@ -9,6 +9,16 @@ endfunction(cat IN_FILE OUT_FILE)
 set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
 configure_file("${CMAKE_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
+# pthread
+if(${BUILD_PTHREAD})
+    cat("${CMAKE_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# gnu regex
+if(${BUILD_GNUREGEX})
+    cat("${CMAKE_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
 # googletest
 if(${BUILD_TEST})
     cat("${CMAKE_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -193,7 +203,10 @@ endif(${BUILD_WITH_TRAFT})
 
 # LIBUV
 if(${BUILD_WITH_UV})
-    add_compile_options(-Wno-sign-compare)
+    if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+        MESSAGE("Windows need set no-sign-compare")
+        add_compile_options(-Wno-sign-compare)
+    endif ()
     add_subdirectory(libuv)
 endif(${BUILD_WITH_UV})
 
@@ -224,6 +237,7 @@ if(${BUILD_WITH_SQLITE})
     )
 endif(${BUILD_WITH_SQLITE})
 
+# pthread
 
 
 # ================================================================================================

include/client/taos.h

@@ -51,6 +51,7 @@ typedef void **TAOS_ROW;
 #define TSDB_DATA_TYPE_JSON      17  // json
 #define TSDB_DATA_TYPE_DECIMAL   18  // decimal
 #define TSDB_DATA_TYPE_BLOB      19  // binary
+#define TSDB_DATA_TYPE_MEDIUMBLOB 20
 
 typedef enum {
   TSDB_OPTION_LOCALE,

include/common/tcommon.h

@@ -56,8 +56,9 @@ typedef struct SColumnDataAgg {
 typedef struct SDataBlockInfo {
   STimeWindow window;
   int32_t     rows;
+  int32_t     rowSize;
   int32_t     numOfCols;
-  int64_t     uid;
+  union {int64_t uid; int64_t blockId;};
 } SDataBlockInfo;
 
 typedef struct SConstantItem {
@@ -69,10 +70,10 @@ typedef struct SConstantItem {
 
 // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
 typedef struct SSDataBlock {
-  SColumnDataAgg* pBlockAgg;
-  SArray*         pDataBlock;  // SArray<SColumnInfoData>
-  SArray* pConstantList;       // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
-  SDataBlockInfo info;
+  SColumnDataAgg *pBlockAgg;
+  SArray         *pDataBlock;    // SArray<SColumnInfoData>
+  SArray         *pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
+  SDataBlockInfo  info;
 } SSDataBlock;
 
 typedef struct SVarColAttr {
@@ -108,7 +109,7 @@ static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlo
     SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
     tlen += taosEncodeFixedI16(buf, pColData->info.colId);
     tlen += taosEncodeFixedI16(buf, pColData->info.type);
-    tlen += taosEncodeFixedI16(buf, pColData->info.bytes);
+    tlen += taosEncodeFixedI32(buf, pColData->info.bytes);
     int32_t colSz = rows * pColData->info.bytes;
     tlen += taosEncodeBinary(buf, pColData->pData, colSz);
   }
@@ -127,7 +128,7 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock)
     SColumnInfoData data = {0};
     buf = taosDecodeFixedI16(buf, &data.info.colId);
     buf = taosDecodeFixedI16(buf, &data.info.type);
-    buf = taosDecodeFixedI16(buf, &data.info.bytes);
+    buf = taosDecodeFixedI32(buf, &data.info.bytes);
     int32_t colSz = pBlock->info.rows * data.info.bytes;
     buf = taosDecodeBinary(buf, (void**)&data.pData, colSz);
     taosArrayPush(pBlock->pDataBlock, &data);
@@ -212,10 +213,22 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
 //======================================================================================================================
 // the following structure shared by parser and executor
 typedef struct SColumn {
-  uint64_t    uid;
-  char        name[TSDB_COL_NAME_LEN];
-  int8_t      flag;  // column type: normal column, tag, or user-input column (integer/float/string)
-  SColumnInfo info;
+  union {
+    uint64_t uid;
+    int64_t  dataBlockId;
+  };
+
+  union {
+    int16_t colId;
+    int16_t slotId;
+  };
+
+  char    name[TSDB_COL_NAME_LEN];
+  int8_t  flag;  // column type: normal column, tag, or user-input column (integer/float/string)
+  int16_t type;
+  int32_t bytes;
+  uint8_t precision;
+  uint8_t scale;
 } SColumn;
 
 typedef struct SLimit {
@@ -233,21 +246,32 @@ typedef struct SGroupbyExpr {
   bool    groupbyTag;  // group by tag or column
 } SGroupbyExpr;
 
-// the structure for sql function in select clause
-typedef struct SSqlExpr {
-  char    token[TSDB_COL_NAME_LEN];  // original token
-  SSchema resSchema;
+typedef struct SFunctParam {
+  int32_t  type;
+  SColumn *pCol;
+  SVariant param;
+} SFunctParam;
 
-  int32_t  numOfCols;
-  SColumn* pColumns;     // data columns that are required by query
-  int32_t  interBytes;   // inter result buffer size
-  int16_t  numOfParams;  // argument value of each function
-  SVariant param[3];     // parameters are not more than 3
-} SSqlExpr;
+// the structure for sql function in select clause
+typedef struct SResSchame {
+  int8_t  type;
+  int32_t colId;
+  int32_t bytes;
+  int32_t precision;
+  int32_t scale;
+  char    name[TSDB_COL_NAME_LEN];
+} SResSchema;
+
+// TODO move away to executor.h
+typedef struct SExprBasicInfo {
+  SResSchema   resSchema;
+  int16_t      numOfParams;  // argument value of each function
+  SFunctParam *pParam;
+} SExprBasicInfo;
 
 typedef struct SExprInfo {
-  struct SSqlExpr   base;
-  struct tExprNode* pExpr;
+  struct SExprBasicInfo  base;
+  struct tExprNode      *pExpr;
 } SExprInfo;
 
 typedef struct SStateWindow {

include/common/tdatablock.h

@@ -94,8 +94,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
 void    colDataTrim(SColumnInfoData* pColumnInfoData);
 
-size_t colDataGetNumOfCols(const SSDataBlock* pBlock);
-size_t colDataGetNumOfRows(const SSDataBlock* pBlock);
+size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
+size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
 
 int32_t      blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
 int32_t      blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,

include/common/tmsg.h

@@ -140,6 +140,8 @@ typedef enum _mgmt_table {
 
 #define TSDB_KILL_MSG_LEN 30
 
+#define TSDB_TABLE_NUM_UNIT 100000
+
 #define TSDB_VN_READ_ACCCESS  ((char)0x1)
 #define TSDB_VN_WRITE_ACCCESS ((char)0x2)
 #define TSDB_VN_ALL_ACCCESS   (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS)
@@ -169,6 +171,7 @@ typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
   int64_t dbId;
   int32_t vgVersion;
+  int32_t numOfTable;      // unit is TSDB_TABLE_NUM_UNIT
 } SBuildUseDBInput;
 
 typedef struct SField {
@@ -416,10 +419,15 @@ typedef struct {
  * But for data in vnode side, we need all the following information.
  */
 typedef struct {
-  int16_t           colId;
-  int16_t           type;
-  int16_t           bytes;
-  SColumnFilterList flist;
+  union {
+    int16_t colId;
+    int16_t slotId;
+  };
+
+  int16_t   type;
+  int32_t   bytes;
+  uint8_t   precision;
+  uint8_t   scale;
 } SColumnInfo;
 
 typedef struct {
@@ -449,57 +457,6 @@ typedef struct {
   int64_t offset;
 } SInterval;
 
-typedef struct {
-  SMsgHead head;
-  char     version[TSDB_VERSION_LEN];
-
-  bool stableQuery;        // super table query or not
-  bool topBotQuery;        // TODO used bitwise flag
-  bool interpQuery;        // interp query or not
-  bool groupbyColumn;      // denote if this is a groupby normal column query
-  bool hasTagResults;      // if there are tag values in final result or not
-  bool timeWindowInterpo;  // if the time window start/end required interpolation
-  bool queryBlockDist;     // if query data block distribution
-  bool stabledev;          // super table stddev query
-  bool tsCompQuery;        // is tscomp query
-  bool simpleAgg;
-  bool pointInterpQuery;  // point interpolation query
-  bool needReverseScan;   // need reverse scan
-  bool stateWindow;       // state window flag
-
-  STimeWindow window;
-  int32_t     numOfTables;
-  int16_t     order;
-  int16_t     orderColId;
-  int16_t     numOfCols;  // the number of columns will be load from vnode
-  SInterval   interval;
-  //  SSessionWindow sw;            // session window
-  int16_t tagCondLen;      // tag length in current query
-  int16_t colCondLen;      // column length in current query
-  int16_t numOfGroupCols;  // num of group by columns
-  int16_t orderByIdx;
-  int16_t orderType;    // used in group by xx order by xxx
-  int64_t vgroupLimit;  // limit the number of rows for each table, used in order by + limit in stable projection query.
-  int16_t prjOrder;     // global order in super table projection query.
-  int64_t limit;
-  int64_t offset;
-  int32_t queryType;    // denote another query process
-  int16_t numOfOutput;  // final output columns numbers
-  int16_t fillType;     // interpolate type
-  int64_t fillVal;      // default value array list
-  int32_t secondStageOutput;
-  STsBufInfo  tsBuf;          // tsBuf info
-  int32_t     numOfTags;      // number of tags columns involved
-  int32_t     sqlstrLen;      // sql query string
-  int32_t     prevResultLen;  // previous result length
-  int32_t     numOfOperator;
-  int32_t     tableScanOperator;  // table scan operator. -1 means no scan operator
-  int32_t     udfNum;             // number of udf function
-  int32_t     udfContentOffset;
-  int32_t     udfContentLen;
-  SColumnInfo tableCols[];
-} SQueryTableReq;
-
 typedef struct {
   int32_t code;
 } SQueryTableRsp;
@@ -566,6 +523,7 @@ typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
   int64_t dbId;
   int32_t vgVersion;
+  int32_t numOfTable;    // unit is TSDB_TABLE_NUM_UNIT
 } SUseDbReq;
 
 int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq);
@@ -584,6 +542,23 @@ int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp);
 int32_t tDeserializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp);
 void    tFreeSUsedbRsp(SUseDbRsp* pRsp);
 
+typedef struct {
+  int32_t rowNum;
+} SQnodeListReq;
+
+int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
+int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
+
+typedef struct {
+  SArray *epSetList; // SArray<SEpSet>
+} SQnodeListRsp;
+
+int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
+int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
+void    tFreeSQnodeListRsp(SQnodeListRsp* pRsp);
+
+
+
 typedef struct {
   SArray* pArray;  // Array of SUseDbRsp
 } SUseDbBatchRsp;
@@ -796,8 +771,10 @@ typedef struct SVgroupInfo {
   uint32_t hashBegin;
   uint32_t hashEnd;
   SEpSet   epSet;
+  int32_t  numOfTable;  // unit is TSDB_TABLE_NUM_UNIT
 } SVgroupInfo;
 
+
 typedef struct {
   int32_t     numOfVgroups;
   SVgroupInfo vgroups[];
@@ -876,6 +853,8 @@ int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
 void    tFreeSShowRsp(SShowRsp* pRsp);
 
 typedef struct {
+  int32_t type;
+  char    db[TSDB_DB_FNAME_LEN];
   int64_t showId;
   int8_t  free;
 } SRetrieveTableReq;
@@ -903,6 +882,8 @@ int32_t tDeserializeSCreateDnodeReq(void* buf, int32_t bufLen, SCreateDnodeReq*
 
 typedef struct {
   int32_t dnodeId;
+  char    fqdn[TSDB_FQDN_LEN];
+  int32_t port;
 } SDropDnodeReq;
 
 int32_t tSerializeSDropDnodeReq(void* buf, int32_t bufLen, SDropDnodeReq* pReq);
@@ -1026,6 +1007,7 @@ typedef struct SSubQueryMsg {
   uint64_t sId;
   uint64_t queryId;
   uint64_t taskId;
+  int64_t  refId;
   int8_t   taskType;
   uint32_t sqlLen;  // the query sql,
   uint32_t phyLen;
@@ -1072,19 +1054,57 @@ typedef struct {
 typedef struct {
   uint64_t queryId;
   uint64_t taskId;
+  int64_t  refId;
   int8_t   status;
 } STaskStatus;
 
 typedef struct {
-  uint32_t    num;
-  STaskStatus status[];
+  int64_t  refId;
+  SArray  *taskStatus;  //SArray<STaskStatus>
 } SSchedulerStatusRsp;
 
+typedef struct {
+  uint64_t queryId;
+  uint64_t taskId;
+  int8_t   action;
+} STaskAction;
+
+
+typedef struct SQueryNodeEpId {
+  int32_t nodeId;  // vgId or qnodeId
+  SEp     ep;
+} SQueryNodeEpId;
+
+
+typedef struct {
+  SMsgHead       header;
+  uint64_t       sId;
+  SQueryNodeEpId epId;
+  SArray        *taskAction;  //SArray<STaskAction>
+} SSchedulerHbReq;
+
+int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq);
+int32_t tDeserializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq);
+void tFreeSSchedulerHbReq(SSchedulerHbReq *pReq);
+
+
+typedef struct {
+  uint64_t       seqId;
+  SQueryNodeEpId epId;
+  SArray        *taskStatus;  //SArray<STaskStatus>
+} SSchedulerHbRsp;
+
+int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp);
+int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp);
+void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp);
+
+
 typedef struct {
   SMsgHead header;
   uint64_t sId;
   uint64_t queryId;
   uint64_t taskId;
+  int64_t  refId;
 } STaskCancelReq;
 
 typedef struct {
@@ -1096,6 +1116,7 @@ typedef struct {
   uint64_t sId;
   uint64_t queryId;
   uint64_t taskId;
+  int64_t  refId;
 } STaskDropReq;
 
 typedef struct {
@@ -1108,18 +1129,34 @@ typedef struct {
   char*  sql;
   char*  physicalPlan;
   char*  logicalPlan;
-} SMCreateTopicReq;
+} SCMCreateStreamReq;
 
-int32_t tSerializeMCreateTopicReq(void* buf, int32_t bufLen, const SMCreateTopicReq* pReq);
-int32_t tDeserializeSMCreateTopicReq(void* buf, int32_t bufLen, SMCreateTopicReq* pReq);
-void    tFreeSMCreateTopicReq(SMCreateTopicReq* pReq);
+typedef struct {
+  int64_t streamId;
+} SCMCreateStreamRsp;
+
+int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateStreamReq* pReq);
+int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq);
+void    tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq);
+
+typedef struct {
+  char   name[TSDB_TOPIC_FNAME_LEN];
+  int8_t igExists;
+  char*  sql;
+  char*  physicalPlan;
+  char*  logicalPlan;
+} SCMCreateTopicReq;
+
+int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq);
+int32_t tDeserializeSCMCreateTopicReq(void* buf, int32_t bufLen, SCMCreateTopicReq* pReq);
+void    tFreeSCMCreateTopicReq(SCMCreateTopicReq* pReq);
 
 typedef struct {
   int64_t topicId;
-} SMCreateTopicRsp;
+} SCMCreateTopicRsp;
 
-int32_t tSerializeSMCreateTopicRsp(void* buf, int32_t bufLen, const SMCreateTopicRsp* pRsp);
-int32_t tDeserializeSMCreateTopicRsp(void* buf, int32_t bufLen, SMCreateTopicRsp* pRsp);
+int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp);
+int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp);
 
 typedef struct {
   int32_t topicNum;
@@ -1319,6 +1356,7 @@ typedef struct SVCreateTbReq {
 } SVCreateTbReq, SVUpdateTbReq;
 
 typedef struct {
+  int tmp; // TODO: to avoid compile error
 } SVCreateTbRsp, SVUpdateTbRsp;
 
 int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
@@ -1330,6 +1368,7 @@ typedef struct {
 } SVCreateTbBatchReq;
 
 typedef struct {
+  int tmp; // TODO: to avoid compile error
 } SVCreateTbBatchRsp;
 
 int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
@@ -1343,6 +1382,7 @@ typedef struct {
 } SVDropTbReq;
 
 typedef struct {
+  int tmp; // TODO: to avoid compile error
 } SVDropTbRsp;
 
 int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq);
@@ -1858,24 +1898,19 @@ typedef enum {
 } ETDTimeUnit;
 
 typedef struct {
-  uint16_t  funcId;
-  uint16_t  nColIds;
-  col_id_t* colIds;  // sorted colIds
-} SFuncColIds;
-
-typedef struct {
-  uint8_t      version;  // for compatibility
-  uint8_t      intervalUnit;
-  uint8_t      slidingUnit;
-  char         indexName[TSDB_INDEX_NAME_LEN];
-  char         timezone[TD_TIMEZONE_LEN];
-  uint16_t     nFuncColIds;
-  uint16_t     tagsFilterLen;
-  tb_uid_t     tableUid;  // super/common table uid
-  int64_t      interval;
-  int64_t      sliding;
-  SFuncColIds* funcColIds;  // sorted funcIds
-  char*        tagsFilter;
+  int8_t   version;  // for compatibility(default 0)
+  int8_t   intervalUnit;
+  int8_t   slidingUnit;
+  char     indexName[TSDB_INDEX_NAME_LEN];
+  char     timezone[TD_TIMEZONE_LEN];  // sma data is invalid if timezone change.
+  uint16_t exprLen;
+  uint16_t tagsFilterLen;
+  int64_t  indexUid;
+  tb_uid_t tableUid;  // super/child/common table uid
+  int64_t  interval;
+  int64_t  sliding;
+  char*    expr;  // sma expression
+  char*    tagsFilter;
 } STSma;  // Time-range-wise SMA
 
 typedef struct {
@@ -1893,7 +1928,9 @@ typedef struct {
   int64_t ver;  // use a general definition
   char    indexName[TSDB_INDEX_NAME_LEN];
 } SVDropTSmaReq;
+
 typedef struct {
+  int tmp; // TODO: to avoid compile error
 } SVCreateTSmaRsp, SVDropTSmaRsp;
 
 int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq);
@@ -1902,24 +1939,30 @@ int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq);
 void*   tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
 
 typedef struct {
-  STimeWindow tsWindow;     // [skey, ekey]
-  uint64_t    tableUid;     // sub/common table uid
-  int32_t     numOfBlocks;  // number of sma blocks for each column, total number is numOfBlocks*numOfColId
-  int32_t     dataLen;      // total data length
-  col_id_t*   colIds;       // e.g. 2,4,9,10
-  col_id_t    numOfColIds;  // e.g. 4
-  char        data[];       // the sma blocks
-} STSmaData;
-
-// TODO: move to the final location afte schema of STSma/STSmaData defined
-static FORCE_INLINE void tdDestroySmaData(STSmaData* pSmaData) {
-  if (pSmaData) {
-    if (pSmaData->colIds) {
-      tfree(pSmaData->colIds);
-    }
-    tfree(pSmaData);
-  }
-}
+  col_id_t colId;
+  uint16_t blockSize;  // sma data block size
+  char     data[];
+} STSmaColData;
+
+typedef struct {
+  tb_uid_t tableUid;  // super/child/normal table uid
+  int32_t  dataLen;   // not including head
+  char     data[];
+} STSmaTbData;
+
+typedef struct {
+  int64_t indexUid;
+  TSKEY   skey;  // startTS of one interval/sliding
+  int64_t interval;
+  int32_t dataLen;  // not including head
+  int8_t  intervalUnit;
+  char    data[];
+} STSmaDataWrapper;  // sma data for a interval/sliding window
+
+// interval/sliding => window
+
+// => window->table->colId
+// => 当一个window下所有的表均计算完成时，流计算告知tsdb清除window的过期标记
 
 // RSma: Rollup SMA
 typedef struct {
@@ -1942,13 +1985,7 @@ typedef struct {
 
 static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
   if (pSma) {
-    if (pSma->funcColIds != NULL) {
-      for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
-        tfree((pSma->funcColIds + i)->colIds);
-      }
-      tfree(pSma->funcColIds);
-    }
-
+    tfree(pSma->expr);
     tfree(pSma->tagsFilter);
   }
 }
@@ -1967,24 +2004,20 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) {
 static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
   int32_t tlen = 0;
 
-  tlen += taosEncodeFixedU8(buf, pSma->version);
-  tlen += taosEncodeFixedU8(buf, pSma->intervalUnit);
-  tlen += taosEncodeFixedU8(buf, pSma->slidingUnit);
+  tlen += taosEncodeFixedI8(buf, pSma->version);
+  tlen += taosEncodeFixedI8(buf, pSma->intervalUnit);
+  tlen += taosEncodeFixedI8(buf, pSma->slidingUnit);
   tlen += taosEncodeString(buf, pSma->indexName);
   tlen += taosEncodeString(buf, pSma->timezone);
-  tlen += taosEncodeFixedU16(buf, pSma->nFuncColIds);
+  tlen += taosEncodeFixedU16(buf, pSma->exprLen);
   tlen += taosEncodeFixedU16(buf, pSma->tagsFilterLen);
+  tlen += taosEncodeFixedI64(buf, pSma->indexUid);
   tlen += taosEncodeFixedI64(buf, pSma->tableUid);
   tlen += taosEncodeFixedI64(buf, pSma->interval);
   tlen += taosEncodeFixedI64(buf, pSma->sliding);
-
-  for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
-    SFuncColIds* funcColIds = pSma->funcColIds + i;
-    tlen += taosEncodeFixedU16(buf, funcColIds->funcId);
-    tlen += taosEncodeFixedU16(buf, funcColIds->nColIds);
-    for (uint16_t j = 0; j < funcColIds->nColIds; ++j) {
-      tlen += taosEncodeFixedU16(buf, *(funcColIds->colIds + j));
-    }
+  
+  if (pSma->exprLen > 0) {
+    tlen += taosEncodeString(buf, pSma->expr);
   }
 
   if (pSma->tagsFilterLen > 0) {
@@ -2005,43 +2038,30 @@ static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* p
 }
 
 static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
-  buf = taosDecodeFixedU8(buf, &pSma->version);
-  buf = taosDecodeFixedU8(buf, &pSma->intervalUnit);
-  buf = taosDecodeFixedU8(buf, &pSma->slidingUnit);
+  buf = taosDecodeFixedI8(buf, &pSma->version);
+  buf = taosDecodeFixedI8(buf, &pSma->intervalUnit);
+  buf = taosDecodeFixedI8(buf, &pSma->slidingUnit);
   buf = taosDecodeStringTo(buf, pSma->indexName);
   buf = taosDecodeStringTo(buf, pSma->timezone);
-  buf = taosDecodeFixedU16(buf, &pSma->nFuncColIds);
+  buf = taosDecodeFixedU16(buf, &pSma->exprLen);
   buf = taosDecodeFixedU16(buf, &pSma->tagsFilterLen);
+  buf = taosDecodeFixedI64(buf, &pSma->indexUid);
   buf = taosDecodeFixedI64(buf, &pSma->tableUid);
   buf = taosDecodeFixedI64(buf, &pSma->interval);
   buf = taosDecodeFixedI64(buf, &pSma->sliding);
 
-  if (pSma->nFuncColIds > 0) {
-    pSma->funcColIds = (SFuncColIds*)calloc(pSma->nFuncColIds, sizeof(SFuncColIds));
-    if (pSma->funcColIds == NULL) {
+
+  if (pSma->exprLen > 0) {
+    pSma->expr = (char*)calloc(pSma->exprLen, 1);
+    if (pSma->expr != NULL) {
+      buf = taosDecodeStringTo(buf, pSma->expr);
+    } else {
       tdDestroyTSma(pSma);
       return NULL;
     }
-    for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
-      SFuncColIds* funcColIds = pSma->funcColIds + i;
-      buf = taosDecodeFixedU16(buf, &funcColIds->funcId);
-      buf = taosDecodeFixedU16(buf, &funcColIds->nColIds);
-      if (funcColIds->nColIds > 0) {
-        funcColIds->colIds = (col_id_t*)calloc(funcColIds->nColIds, sizeof(col_id_t));
-        if (funcColIds->colIds != NULL) {
-          for (uint16_t j = 0; j < funcColIds->nColIds; ++j) {
-            buf = taosDecodeFixedU16(buf, funcColIds->colIds + j);
-          }
-        } else {
-          tdDestroyTSma(pSma);
-          return NULL;
-        }
-      } else {
-        funcColIds->colIds = NULL;
-      }
-    }
+
   } else {
-    pSma->funcColIds = NULL;
+    pSma->expr = NULL;
   }
 
   if (pSma->tagsFilterLen > 0) {

include/common/tmsgdef.h

@@ -129,11 +129,13 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "mnode-qnode-list", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_SYSTABLE_RETRIEVE, "mnode-systable-retrieve", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL)
@@ -148,6 +150,9 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL)
 
   // Requests handled by VNODE
   TD_NEW_MSG_SEG(TDMT_VND_MSG)
@@ -180,6 +185,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_QUERY_HEARTBEAT, "vnode-query-heartbeat", NULL, NULL)
 
   TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)

include/common/trow.h

@@ -103,6 +103,7 @@ typedef struct {
 
 typedef struct {
   // TODO
+  int tmp; // TODO: to avoid compile error
 } STpRow;  // tuple
 
 #pragma pack(push, 1)
@@ -1098,4 +1099,4 @@ const STSRow *tRowBatchIterNext(STSRowBatchIter *pRowBatchIter);
 }
 #endif
 
-#endif /*_TD_COMMON_ROW_H_*/
+#endif /*_TD_COMMON_ROW_H_*/
\ No newline at end of file

include/common/ttokendef.h

@@ -16,279 +16,159 @@
 #ifndef _TD_COMMON_TOKEN_H_
 #define _TD_COMMON_TOKEN_H_
 
-#define TK_ID           1
-#define TK_BOOL         2
-#define TK_INTEGER      3
-#define TK_FLOAT        4
-#define TK_STRING       5
-#define TK_TIMESTAMP    6
-#define TK_OR           7
-#define TK_AND          8
-#define TK_NOT          9
-#define TK_EQ           10
-#define TK_NE           11
-#define TK_ISNULL       12
-#define TK_NOTNULL      13
-#define TK_IS           14
-#define TK_LIKE         15
-#define TK_MATCH        16
-#define TK_NMATCH       17
-#define TK_GLOB         18
-#define TK_BETWEEN      19
-#define TK_IN           20
-#define TK_GT           21
-#define TK_GE           22
-#define TK_LT           23
-#define TK_LE           24
-#define TK_BITAND       25
-#define TK_BITOR        26
-#define TK_LSHIFT       27
-#define TK_RSHIFT       28
-#define TK_PLUS         29
-#define TK_MINUS        30
-#define TK_DIVIDE       31
-#define TK_TIMES        32
-#define TK_STAR         33
-#define TK_SLASH        34
-#define TK_REM          35
-#define TK_CONCAT       36
-#define TK_UMINUS       37
-#define TK_UPLUS        38
-#define TK_BITNOT       39
-#define TK_SHOW         40
-#define TK_DATABASES    41
-#define TK_TOPICS       42
-#define TK_FUNCTIONS    43
-#define TK_MNODES       44
-#define TK_DNODES       45
-#define TK_ACCOUNTS     46
-#define TK_USERS        47
-#define TK_MODULES      48
-#define TK_QUERIES      49
-#define TK_CONNECTIONS  50
-#define TK_STREAMS      51
-#define TK_VARIABLES    52
-#define TK_SCORES       53
-#define TK_GRANTS       54
-#define TK_VNODES       55
-#define TK_DOT          56
-#define TK_CREATE       57
-#define TK_TABLE        58
-#define TK_STABLE       59
-#define TK_DATABASE     60
-#define TK_TABLES       61
-#define TK_STABLES      62
-#define TK_VGROUPS      63
-#define TK_DROP         64
-#define TK_TOPIC        65
-#define TK_FUNCTION     66
-#define TK_DNODE        67
-#define TK_USER         68
-#define TK_ACCOUNT      69
-#define TK_USE          70
-#define TK_DESCRIBE     71
-#define TK_DESC         72
-#define TK_ALTER        73
-#define TK_PASS         74
-#define TK_PRIVILEGE    75
-#define TK_LOCAL        76
-#define TK_COMPACT      77
-#define TK_LP           78
-#define TK_RP           79
-#define TK_IF           80
-#define TK_EXISTS       81
-#define TK_PORT         82
-#define TK_IPTOKEN      83
-#define TK_AS           84
-#define TK_OUTPUTTYPE   85
-#define TK_AGGREGATE    86
-#define TK_BUFSIZE      87
-#define TK_PPS          88
-#define TK_TSERIES      89
-#define TK_DBS          90
-#define TK_STORAGE      91
-#define TK_QTIME        92
-#define TK_CONNS        93
-#define TK_STATE        94
-#define TK_COMMA        95
-#define TK_KEEP         96
-#define TK_CACHE        97
-#define TK_REPLICA      98
-#define TK_QUORUM       99
-#define TK_DAYS         100
-#define TK_MINROWS      101
-#define TK_MAXROWS      102
-#define TK_BLOCKS       103
-#define TK_CTIME        104
-#define TK_WAL          105
-#define TK_FSYNC        106
-#define TK_COMP         107
-#define TK_PRECISION    108
-#define TK_UPDATE       109
-#define TK_CACHELAST    110
-#define TK_STREAM       111
-#define TK_MODE         112
-#define TK_UNSIGNED     113
-#define TK_TAGS         114
-#define TK_USING        115
-#define TK_NULL         116
-#define TK_NOW          117
-#define TK_SELECT       118
-#define TK_UNION        119
-#define TK_ALL          120
-#define TK_DISTINCT     121
-#define TK_FROM         122
-#define TK_VARIABLE     123
-#define TK_INTERVAL     124
-#define TK_EVERY        125
-#define TK_SESSION      126
-#define TK_STATE_WINDOW 127
-#define TK_FILL         128
-#define TK_SLIDING      129
-#define TK_ORDER        130
-#define TK_BY           131
-#define TK_ASC          132
-#define TK_GROUP        133
-#define TK_HAVING       134
-#define TK_LIMIT        135
-#define TK_OFFSET       136
-#define TK_SLIMIT       137
-#define TK_SOFFSET      138
-#define TK_WHERE        139
-#define TK_RESET        140
-#define TK_QUERY        141
-#define TK_SYNCDB       142
-#define TK_ADD          143
-#define TK_COLUMN       144
-#define TK_MODIFY       145
-#define TK_TAG          146
-#define TK_CHANGE       147
-#define TK_SET          148
-#define TK_KILL         149
-#define TK_CONNECTION   150
-#define TK_COLON        151
-#define TK_ABORT        152
-#define TK_AFTER        153
-#define TK_ATTACH       154
-#define TK_BEFORE       155
-#define TK_BEGIN        156
-#define TK_CASCADE      157
-#define TK_CLUSTER      158
-#define TK_CONFLICT     159
-#define TK_COPY         160
-#define TK_DEFERRED     161
-#define TK_DELIMITERS   162
-#define TK_DETACH       163
-#define TK_EACH         164
-#define TK_END          165
-#define TK_EXPLAIN      166
-#define TK_FAIL         167
-#define TK_FOR          168
-#define TK_IGNORE       169
-#define TK_IMMEDIATE    170
-#define TK_INITIALLY    171
-#define TK_INSTEAD      172
-#define TK_KEY          173
-#define TK_OF           174
-#define TK_RAISE        175
-#define TK_REPLACE      176
-#define TK_RESTRICT     177
-#define TK_ROW          178
-#define TK_STATEMENT    179
-#define TK_TRIGGER      180
-#define TK_VIEW         181
-#define TK_SEMI         182
-#define TK_NONE         183
-#define TK_PREV         184
-#define TK_LINEAR       185
-#define TK_IMPORT       186
-#define TK_TBNAME       187
-#define TK_JOIN         188
-#define TK_INSERT       189
-#define TK_INTO         190
-#define TK_VALUES       191
+#define TK_OR                               1
+#define TK_AND                              2
+#define TK_UNION                            3
+#define TK_ALL                              4
+#define TK_MINUS                            5
+#define TK_EXCEPT                           6
+#define TK_INTERSECT                        7
+#define TK_NK_BITAND                        8
+#define TK_NK_BITOR                         9
+#define TK_NK_LSHIFT                       10
+#define TK_NK_RSHIFT                       11
+#define TK_NK_PLUS                         12
+#define TK_NK_MINUS                        13
+#define TK_NK_STAR                         14
+#define TK_NK_SLASH                        15
+#define TK_NK_REM                          16
+#define TK_NK_CONCAT                       17
+#define TK_CREATE                          18
+#define TK_USER                            19
+#define TK_PASS                            20
+#define TK_NK_STRING                       21
+#define TK_ALTER                           22
+#define TK_PRIVILEGE                       23
+#define TK_DROP                            24
+#define TK_SHOW                            25
+#define TK_USERS                           26
+#define TK_DNODE                           27
+#define TK_PORT                            28
+#define TK_NK_INTEGER                      29
+#define TK_DNODES                          30
+#define TK_NK_ID                           31
+#define TK_NK_IPTOKEN                      32
+#define TK_DATABASE                        33
+#define TK_DATABASES                       34
+#define TK_USE                             35
+#define TK_IF                              36
+#define TK_NOT                             37
+#define TK_EXISTS                          38
+#define TK_BLOCKS                          39
+#define TK_CACHE                           40
+#define TK_CACHELAST                       41
+#define TK_COMP                            42
+#define TK_DAYS                            43
+#define TK_FSYNC                           44
+#define TK_MAXROWS                         45
+#define TK_MINROWS                         46
+#define TK_KEEP                            47
+#define TK_PRECISION                       48
+#define TK_QUORUM                          49
+#define TK_REPLICA                         50
+#define TK_TTL                             51
+#define TK_WAL                             52
+#define TK_VGROUPS                         53
+#define TK_SINGLE_STABLE                   54
+#define TK_STREAM_MODE                     55
+#define TK_TABLE                           56
+#define TK_NK_LP                           57
+#define TK_NK_RP                           58
+#define TK_STABLE                          59
+#define TK_TABLES                          60
+#define TK_STABLES                         61
+#define TK_USING                           62
+#define TK_TAGS                            63
+#define TK_NK_DOT                          64
+#define TK_NK_COMMA                        65
+#define TK_COMMENT                         66
+#define TK_BOOL                            67
+#define TK_TINYINT                         68
+#define TK_SMALLINT                        69
+#define TK_INT                             70
+#define TK_INTEGER                         71
+#define TK_BIGINT                          72
+#define TK_FLOAT                           73
+#define TK_DOUBLE                          74
+#define TK_BINARY                          75
+#define TK_TIMESTAMP                       76
+#define TK_NCHAR                           77
+#define TK_UNSIGNED                        78
+#define TK_JSON                            79
+#define TK_VARCHAR                         80
+#define TK_MEDIUMBLOB                      81
+#define TK_BLOB                            82
+#define TK_VARBINARY                       83
+#define TK_DECIMAL                         84
+#define TK_SMA                             85
+#define TK_MNODES                          86
+#define TK_NK_FLOAT                        87
+#define TK_NK_BOOL                         88
+#define TK_NK_VARIABLE                     89
+#define TK_BETWEEN                         90
+#define TK_IS                              91
+#define TK_NULL                            92
+#define TK_NK_LT                           93
+#define TK_NK_GT                           94
+#define TK_NK_LE                           95
+#define TK_NK_GE                           96
+#define TK_NK_NE                           97
+#define TK_NK_EQ                           98
+#define TK_LIKE                            99
+#define TK_MATCH                          100
+#define TK_NMATCH                         101
+#define TK_IN                             102
+#define TK_FROM                           103
+#define TK_AS                             104
+#define TK_JOIN                           105
+#define TK_ON                             106
+#define TK_INNER                          107
+#define TK_SELECT                         108
+#define TK_DISTINCT                       109
+#define TK_WHERE                          110
+#define TK_PARTITION                      111
+#define TK_BY                             112
+#define TK_SESSION                        113
+#define TK_STATE_WINDOW                   114
+#define TK_INTERVAL                       115
+#define TK_SLIDING                        116
+#define TK_FILL                           117
+#define TK_VALUE                          118
+#define TK_NONE                           119
+#define TK_PREV                           120
+#define TK_LINEAR                         121
+#define TK_NEXT                           122
+#define TK_GROUP                          123
+#define TK_HAVING                         124
+#define TK_ORDER                          125
+#define TK_SLIMIT                         126
+#define TK_SOFFSET                        127
+#define TK_LIMIT                          128
+#define TK_OFFSET                         129
+#define TK_ASC                            130
+#define TK_DESC                           131
+#define TK_NULLS                          132
+#define TK_FIRST                          133
+#define TK_LAST                           134
 
-#define NEW_TK_OR           1
-#define NEW_TK_AND          2
-#define NEW_TK_UNION        3
-#define NEW_TK_ALL          4
-#define NEW_TK_MINUS        5
-#define NEW_TK_EXCEPT       6
-#define NEW_TK_INTERSECT    7
-#define NEW_TK_NK_PLUS      8
-#define NEW_TK_NK_MINUS     9
-#define NEW_TK_NK_STAR      10
-#define NEW_TK_NK_SLASH     11
-#define NEW_TK_NK_REM       12
-#define NEW_TK_SHOW         13
-#define NEW_TK_DATABASES    14
-#define NEW_TK_NK_INTEGER   15
-#define NEW_TK_NK_FLOAT     16
-#define NEW_TK_NK_STRING    17
-#define NEW_TK_NK_BOOL      18
-#define NEW_TK_TIMESTAMP    19
-#define NEW_TK_NK_VARIABLE  20
-#define NEW_TK_NK_COMMA     21
-#define NEW_TK_NK_ID        22
-#define NEW_TK_NK_LP        23
-#define NEW_TK_NK_RP        24
-#define NEW_TK_NK_DOT       25
-#define NEW_TK_BETWEEN      26
-#define NEW_TK_NOT          27
-#define NEW_TK_IS           28
-#define NEW_TK_NULL         29
-#define NEW_TK_NK_LT        30
-#define NEW_TK_NK_GT        31
-#define NEW_TK_NK_LE        32
-#define NEW_TK_NK_GE        33
-#define NEW_TK_NK_NE        34
-#define NEW_TK_NK_EQ        35
-#define NEW_TK_LIKE         36
-#define NEW_TK_MATCH        37
-#define NEW_TK_NMATCH       38
-#define NEW_TK_IN           39
-#define NEW_TK_FROM         40
-#define NEW_TK_AS           41
-#define NEW_TK_JOIN         42
-#define NEW_TK_ON           43
-#define NEW_TK_INNER        44
-#define NEW_TK_SELECT       45
-#define NEW_TK_DISTINCT     46
-#define NEW_TK_WHERE        47
-#define NEW_TK_PARTITION    48
-#define NEW_TK_BY           49
-#define NEW_TK_SESSION      50
-#define NEW_TK_STATE_WINDOW 51
-#define NEW_TK_INTERVAL     52
-#define NEW_TK_SLIDING      53
-#define NEW_TK_FILL         54
-#define NEW_TK_VALUE        55
-#define NEW_TK_NONE         56
-#define NEW_TK_PREV         57
-#define NEW_TK_LINEAR       58
-#define NEW_TK_NEXT         59
-#define NEW_TK_GROUP        60
-#define NEW_TK_HAVING       61
-#define NEW_TK_ORDER        62
-#define NEW_TK_SLIMIT       63
-#define NEW_TK_SOFFSET      64
-#define NEW_TK_LIMIT        65
-#define NEW_TK_OFFSET       66
-#define NEW_TK_ASC          67
-#define NEW_TK_DESC         68
-#define NEW_TK_NULLS        69
-#define NEW_TK_FIRST        70
-#define NEW_TK_LAST         71
+#define TK_NK_SPACE                       300
+#define TK_NK_COMMENT                     301
+#define TK_NK_ILLEGAL                     302
+#define TK_NK_HEX                         303   // hex number  0x123
+#define TK_NK_OCT                         304   // oct number
+#define TK_NK_BIN                         305   // bin format data 0b111
+#define TK_NK_FILE                        306
+#define TK_NK_QUESTION                    307   // denoting the placeholder of "?",when invoking statement bind query
 
-#define TK_SPACE    300
-#define TK_COMMENT  301
-#define TK_ILLEGAL  302
-#define TK_HEX      303  // hex number  0x123
-#define TK_OCT      304  // oct number
-#define TK_BIN      305  // bin format data 0b111
-#define TK_FILE     306
-#define TK_QUESTION 307  // denoting the placeholder of "?",when invoking statement bind query
+#define TK_NK_COLON                       500
+#define TK_NK_BITNOT                      501
+#define TK_INSERT                         502
+#define TK_INTO                           503
+#define TK_NOW                            504
+#define TK_VALUES                         507
+#define TK_IMPORT                         507
+#define TK_NK_SEMI                        508
 
-#define TK_NIL 65535
+#define TK_NK_NIL                         65535
 
 #endif /*_TD_COMMON_TOKEN_H_*/

include/dnode/mnode/sdb/sdb.h

@@ -113,15 +113,16 @@ typedef enum {
   SDB_USER = 7,
   SDB_AUTH = 8,
   SDB_ACCT = 9,
-  SDB_OFFSET = 10,
-  SDB_SUBSCRIBE = 11,
-  SDB_CONSUMER = 12,
-  SDB_TOPIC = 13,
-  SDB_VGROUP = 14,
-  SDB_STB = 15,
-  SDB_DB = 16,
-  SDB_FUNC = 17,
-  SDB_MAX = 18
+  SDB_STREAM = 10,
+  SDB_OFFSET = 11,
+  SDB_SUBSCRIBE = 12,
+  SDB_CONSUMER = 13,
+  SDB_TOPIC = 14,
+  SDB_VGROUP = 15,
+  SDB_STB = 16,
+  SDB_DB = 17,
+  SDB_FUNC = 18,
+  SDB_MAX = 19
 } ESdbType;
 
 typedef struct SSdb SSdb;

include/libs/catalog/catalog.h

@@ -51,7 +51,7 @@ typedef struct SMetaData {
   SArray    *pTableMeta;  // STableMeta array
   SArray    *pVgroupInfo; // SVgroupInfo list
   SArray    *pUdfList;    // udf info list
-  SEpSet    *pEpSet;      // qnode epset list
+  SArray    *pEpSetList;  // qnode epset list, SArray<SEpSet>
 } SMetaData;
 
 typedef struct SCatalogCfg {
@@ -74,9 +74,9 @@ typedef struct SDbVgVersion {
   char    dbFName[TSDB_DB_FNAME_LEN];
   int64_t dbId;
   int32_t vgVersion;
+  int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
 } SDbVgVersion;
 
-
 int32_t catalogInit(SCatalogCfg *cfg);
 
 /**
@@ -95,7 +95,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle);
  */
 void catalogFreeHandle(SCatalog* pCatalog);
 
-int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId);
+int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum);
 
 /**
  * Get a DB's all vgroup info.

include/libs/executor/dataSinkMgt.h

@@ -23,6 +23,7 @@ extern "C" {
 #include "os.h"
 #include "thash.h"
 #include "executor.h"
+#include "plannodes.h"
 
 #define DS_BUF_LOW   1
 #define DS_BUF_FULL  2
@@ -59,7 +60,7 @@ typedef struct SOutputData {
  * @param pHandle output
  * @return error code
  */
-int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pHandle);
+int32_t dsCreateDataSinker(const SDataSinkNode* pDataSink, DataSinkHandle* pHandle);
 
 /**
  * Put the result set returned by the executor into datasinker.

include/libs/function/function.h

@@ -20,9 +20,30 @@
 extern "C" {
 #endif
 
+#include "tbuffer.h"
 #include "tcommon.h"
 #include "tvariant.h"
-#include "tbuffer.h"
+
+struct SqlFunctionCtx;
+struct SResultRowEntryInfo;
+
+typedef struct SFunctionNode SFunctionNode;
+
+typedef struct SFuncExecEnv {
+  int32_t calcMemSize;
+} SFuncExecEnv;
+
+typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
+typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx);
+typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
+
+typedef struct SFuncExecFuncs {
+  FExecGetEnv getEnv;
+  FExecInit init;
+  FExecProcess process;
+  FExecFinalize finalize;
+} SFuncExecFuncs;
 
 #define MAX_INTERVAL_TIME_WINDOW 1000000  // maximum allowed time windows in final results
 
@@ -111,64 +132,65 @@ struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
 
 //for selectivity query, the corresponding tag value is assigned if the data is qualified
-typedef struct SExtTagsInfo {
-  int16_t                 tagsLen;      // keep the tags data for top/bottom query result
-  int16_t                 numOfTagCols;
-  struct SqlFunctionCtx **pTagCtxList;
-} SExtTagsInfo;
+typedef struct SSubsidiaryResInfo {
+  int16_t                 bufLen;      // keep the tags data for top/bottom query result
+  int16_t                 numOfCols;
+  struct SqlFunctionCtx **pCtx;
+} SSubsidiaryResInfo;
 
 typedef struct SResultDataInfo {
+  int16_t precision;
+  int16_t scale;
   int16_t type;
   int16_t bytes;
-  int32_t intermediateBytes;
+  int32_t interBufSize;
 } SResultDataInfo;
 
 #define GET_RES_INFO(ctx) ((ctx)->resultInfo)
 
-typedef struct SFunctionFpSet {
-  bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);  // setup the execute environment
-  void (*addInput)(struct SqlFunctionCtx *pCtx);
-
-  // finalizer must be called after all exec has been executed to generated final result.
-  void (*finalize)(struct SqlFunctionCtx *pCtx);
-  void (*combine)(struct SqlFunctionCtx *pCtx);
-} SFunctionFpSet;
-
-extern SFunctionFpSet fpSet[1];
+typedef struct SInputColumnInfoData {
+  int32_t           totalRows;      // total rows in current columnar data
+  int32_t           startRowIndex;  // handle started row index
+  int32_t           numOfRows;      // the number of rows needs to be handled
+  int32_t           numOfInputCols; // PTS is not included
+  bool              colDataAggIsSet;// if agg is set or not
+  SColumnInfoData  *pPTS;           // primary timestamp column
+  SColumnInfoData **pData;
+  SColumnDataAgg  **pColumnDataAgg;
+} SInputColumnInfoData;
 
 // sql function runtime context
 typedef struct SqlFunctionCtx {
-  int32_t      startRow;
-  int32_t      size;      // number of rows
+  SInputColumnInfoData input;
+  SResultDataInfo      resDataInfo;
+  uint32_t             order;  // asc|desc
+  ////////////////////////////////////////////////////////////////
+  int32_t          startRow;   // start row index
+  int32_t          size;       // handled processed row number
   SColumnInfoData* pInput;
-
-  uint32_t     order;     // asc|desc
-  int16_t      inputType;
-  int16_t      inputBytes;
-
-  SResultDataInfo resDataInfo;
-  bool         hasNull;       // null value exist in current block
-  bool         requireNull;   // require null in some function
-  bool         stableQuery;
-  int16_t      functionId;    // function id
-  char *       pOutput;       // final result output buffer, point to sdata->data
-  uint8_t      currentStage;  // record current running step, default: 0
-  int64_t      startTs;       // timestamp range of current query when function is executed on a specific data block
-  int32_t      numOfParams;
-  SVariant     param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
-  int64_t     *ptsList;       // corresponding timestamp array list
-  void        *ptsOutputBuf;  // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
-  SVariant     tag;
-
-  bool        isAggSet;
-  SColumnDataAgg agg;
+  SColumnDataAgg   agg;
+  int16_t          inputType;    // TODO remove it
+  int16_t          inputBytes;   // TODO remove it
+  bool             hasNull;      // null value exist in current block, TODO remove it
+  bool             requireNull;  // require null in some function, TODO remove it
+  int32_t          columnIndex;  // TODO remove it
+  uint8_t          currentStage;  // record current running step, default: 0
+  bool             isAggSet;
+  /////////////////////////////////////////////////////////////////
+  bool             stableQuery;
+  int16_t          functionId;    // function id
+  char *           pOutput;       // final result output buffer, point to sdata->data
+  int64_t          startTs;       // timestamp range of current query when function is executed on a specific data block
+  int32_t          numOfParams;
+  SVariant         param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
+  int64_t         *ptsList;       // corresponding timestamp array list
+  void            *ptsOutputBuf;  // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
+  SVariant         tag;
   struct  SResultRowEntryInfo *resultInfo;
-  SExtTagsInfo tagInfo;
-  SPoint1      start;
-  SPoint1      end;
-
-  int32_t      columnIndex;
-  SFunctionFpSet* fpSet;
+  SSubsidiaryResInfo     subsidiaryRes;
+  SPoint1          start;
+  SPoint1          end;
+  SFuncExecFuncs   fpSet;
 } SqlFunctionCtx;
 
 enum {
@@ -194,9 +216,10 @@ typedef struct tExprNode {
     struct SVariant    *pVal;   // value node
 
     struct {// function node
-      char              functionName[FUNCTIONS_NAME_MAX_LENGTH];
+      char              functionName[FUNCTIONS_NAME_MAX_LENGTH];  // todo refactor
+      int32_t           functionId;
       int32_t           num;
-
+      SFunctionNode    *pFunctNode;
       // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the
       // calculation instead.
       // E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes.
@@ -207,7 +230,6 @@ typedef struct tExprNode {
   };
 } tExprNode;
 
-//TODO create?
 void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
 void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
 
@@ -294,7 +316,7 @@ tExprNode* exprdup(tExprNode* pTree);
 
 void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num);
 void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell);
-int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num);
+int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock);
 bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry);
 bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry);
 

include/libs/function/functionMgt.h

@@ -102,22 +102,6 @@ struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
 struct STimeWindow;
 
-typedef struct SFuncExecEnv {
-  int32_t calcMemSize;
-} SFuncExecEnv;
-
-typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
-typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
-typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx);
-typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
-
-typedef struct SFuncExecFuncs {
-  FExecGetEnv getEnv;
-  FExecInit init;
-  FExecProcess process;
-  FExecFinalize finalize;
-} SFuncExecFuncs;
-
 typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 typedef struct SScalarFuncExecFuncs {
@@ -127,6 +111,8 @@ typedef struct SScalarFuncExecFuncs {
 
 int32_t fmFuncMgtInit();
 
+void fmFuncMgtDestroy();
+
 int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
 
 int32_t fmGetFuncResultType(SFunctionNode* pFunc);

include/libs/nodes/cmdnodes.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_CMD_NODES_H_
+#define _TD_CMD_NODES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "querynodes.h"
+
+typedef struct SDatabaseOptions {
+  int32_t numOfBlocks;
+  int32_t cacheBlockSize;
+  int8_t cachelast;
+  int32_t compressionLevel;
+  int32_t daysPerFile;
+  int32_t fsyncPeriod;
+  int32_t maxRowsPerBlock;
+  int32_t minRowsPerBlock;
+  int32_t keep;
+  int32_t precision;
+  int32_t quorum;
+  int32_t replica;
+  int32_t ttl;
+  int32_t walLevel;
+  int32_t numOfVgroups;
+  int8_t singleStable;
+  int8_t streamMode;
+} SDatabaseOptions;
+
+typedef struct SCreateDatabaseStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  bool ignoreExists;
+  SDatabaseOptions options;
+} SCreateDatabaseStmt;
+
+typedef struct SUseDatabaseStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+} SUseDatabaseStmt;
+
+typedef struct SDropDatabaseStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  bool ignoreNotExists;
+} SDropDatabaseStmt;
+
+typedef struct STableOptions {
+  int32_t keep;
+  int32_t ttl;
+  char comments[TSDB_STB_COMMENT_LEN];
+  SNodeList* pSma;
+} STableOptions;
+
+typedef struct SColumnDefNode {
+  ENodeType type;
+  char colName[TSDB_COL_NAME_LEN];
+  SDataType dataType;
+  char comments[TSDB_STB_COMMENT_LEN];
+} SColumnDefNode;
+
+typedef struct SCreateTableStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  bool ignoreExists;
+  SNodeList* pCols;
+  SNodeList* pTags;
+  STableOptions options;
+} SCreateTableStmt;
+
+typedef struct SCreateSubTableClause {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  char useDbName[TSDB_DB_NAME_LEN];
+  char useTableName[TSDB_TABLE_NAME_LEN];
+  bool ignoreExists;
+  SNodeList* pSpecificTags;
+  SNodeList* pValsOfTags;
+} SCreateSubTableClause;
+
+typedef struct SCreateMultiTableStmt {
+  ENodeType type;
+  SNodeList* pSubTables;
+} SCreateMultiTableStmt;
+
+typedef struct SDropTableClause {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  bool ignoreNotExists;
+} SDropTableClause;
+
+typedef struct SDropTableStmt {
+  ENodeType type;
+  SNodeList* pTables;
+} SDropTableStmt;
+
+typedef struct SDropSuperTableStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  bool ignoreNotExists;
+} SDropSuperTableStmt;
+
+typedef struct SCreateUserStmt {
+  ENodeType type;
+  char useName[TSDB_USER_LEN];
+  char password[TSDB_USET_PASSWORD_LEN];
+} SCreateUserStmt;
+
+typedef struct SAlterUserStmt {
+  ENodeType type;
+  char useName[TSDB_USER_LEN];
+  char password[TSDB_USET_PASSWORD_LEN];
+  int8_t alterType;
+} SAlterUserStmt;
+
+typedef struct SDropUserStmt {
+  ENodeType type;
+  char useName[TSDB_USER_LEN];
+} SDropUserStmt;
+
+typedef struct SCreateDnodeStmt {
+  ENodeType type;
+  char fqdn[TSDB_FQDN_LEN];
+  int32_t port;
+} SCreateDnodeStmt;
+
+typedef struct SDropDnodeStmt {
+  ENodeType type;
+  int32_t dnodeId;
+  char fqdn[TSDB_FQDN_LEN];
+  int32_t port;
+} SDropDnodeStmt;
+
+typedef struct SShowStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+} SShowStmt;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_CMD_NODES_H_*/

include/libs/nodes/nodes.h

@@ -65,24 +65,60 @@ typedef enum ENodeType {
   QUERY_NODE_TARGET,
   QUERY_NODE_DATABLOCK_DESC,
   QUERY_NODE_SLOT_DESC,
+  QUERY_NODE_COLUMN_DEF,
+  QUERY_NODE_DOWNSTREAM_SOURCE,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,
   QUERY_NODE_SELECT_STMT,
-  QUERY_NODE_SHOW_STMT,
+  QUERY_NODE_VNODE_MODIF_STMT,
+  QUERY_NODE_CREATE_DATABASE_STMT,
+  QUERY_NODE_DROP_DATABASE_STMT,
+  QUERY_NODE_SHOW_DATABASES_STMT, // temp
+  QUERY_NODE_CREATE_TABLE_STMT,
+  QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
+  QUERY_NODE_CREATE_MULTI_TABLE_STMT,
+  QUERY_NODE_DROP_TABLE_CLAUSE,
+  QUERY_NODE_DROP_TABLE_STMT,
+  QUERY_NODE_DROP_SUPER_TABLE_STMT,
+  QUERY_NODE_SHOW_TABLES_STMT, // temp
+  QUERY_NODE_SHOW_STABLES_STMT,
+  QUERY_NODE_CREATE_USER_STMT,
+  QUERY_NODE_ALTER_USER_STMT,
+  QUERY_NODE_DROP_USER_STMT,
+  QUERY_NODE_SHOW_USERS_STMT,
+  QUERY_NODE_USE_DATABASE_STMT,
+  QUERY_NODE_CREATE_DNODE_STMT,
+  QUERY_NODE_DROP_DNODE_STMT,
+  QUERY_NODE_SHOW_DNODES_STMT,
+  QUERY_NODE_SHOW_VGROUPS_STMT,
+  QUERY_NODE_SHOW_MNODES_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
   QUERY_NODE_LOGIC_PLAN_JOIN,
   QUERY_NODE_LOGIC_PLAN_AGG,
   QUERY_NODE_LOGIC_PLAN_PROJECT,
+  QUERY_NODE_LOGIC_PLAN_VNODE_MODIF,
+  QUERY_NODE_LOGIC_PLAN_EXCHANGE,
+  QUERY_NODE_LOGIC_SUBPLAN,
+  QUERY_NODE_LOGIC_PLAN,
 
   // physical plan node
   QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
   QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN,
   QUERY_NODE_PHYSICAL_PLAN_PROJECT,
   QUERY_NODE_PHYSICAL_PLAN_JOIN,
-  QUERY_NODE_PHYSICAL_PLAN_AGG
+  QUERY_NODE_PHYSICAL_PLAN_AGG,
+  QUERY_NODE_PHYSICAL_PLAN_EXCHANGE,
+  QUERY_NODE_PHYSICAL_PLAN_SORT,
+  QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
+  QUERY_NODE_PHYSICAL_PLAN_INSERT,
+  QUERY_NODE_PHYSICAL_SUBPLAN,
+  QUERY_NODE_PHYSICAL_PLAN
 } ENodeType;
 
 /**
@@ -105,15 +141,20 @@ typedef struct SNodeList {
   SListCell* pTail;
 } SNodeList;
 
-SNode* nodesMakeNode(ENodeType type);
-void nodesDestroyNode(SNode* pNode);
+#define SNodeptr void*  
+
+SNodeptr nodesMakeNode(ENodeType type);
+void nodesDestroyNode(SNodeptr pNode);
 
 SNodeList* nodesMakeList();
-int32_t nodesListAppend(SNodeList* pList, SNode* pNode);
+int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
+int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
 int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
 SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
-SNode* nodesListGetNode(SNodeList* pList, int32_t index);
+SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
 void nodesDestroyList(SNodeList* pList);
+// Only clear the linked list structure, without releasing the elements inside
+void nodesClearList(SNodeList* pList);
 
 typedef enum EDealRes {
   DEAL_RES_CONTINUE = 1,
@@ -122,9 +163,9 @@ typedef enum EDealRes {
 } EDealRes;
 
 typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext);
-void nodesWalkNode(SNode* pNode, FNodeWalker walker, void* pContext);
+void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext);
 void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext);
-void nodesWalkNodePostOrder(SNode* pNode, FNodeWalker walker, void* pContext);
+void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext);
 void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext);
 
 typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext);
@@ -133,12 +174,13 @@ void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
 void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext);
 void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
 
-bool nodesEqualNode(const SNode* a, const SNode* b);
+bool nodesEqualNode(const SNodeptr a, const SNodeptr b);
 
-SNode* nodesCloneNode(const SNode* pNode);
+SNodeptr nodesCloneNode(const SNodeptr pNode);
 SNodeList* nodesCloneList(const SNodeList* pList);
 
-int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen);
+const char* nodesNodeName(ENodeType type);
+int32_t nodesNodeToString(const SNodeptr pNode, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToNode(const char* pStr, SNode** pNode);
 
 #ifdef __cplusplus

include/libs/nodes/plannodes.h

@@ -21,7 +21,8 @@ extern "C" {
 #endif
 
 #include "querynodes.h"
-#include "tmsg.h"
+#include "query.h"
+#include "tname.h"
 
 typedef struct SLogicNode {
   ENodeType type;
@@ -43,9 +44,11 @@ typedef struct SScanLogicNode {
   SLogicNode node;
   SNodeList* pScanCols;
   struct STableMeta* pMeta;
+  SVgroupsInfo* pVgroupList;
   EScanType scanType;
   uint8_t scanFlag;         // denotes reversed scan of data or not
   STimeWindow scanRange;
+  SName tableName;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -65,6 +68,49 @@ typedef struct SProjectLogicNode {
   SNodeList* pProjections;
 } SProjectLogicNode;
 
+typedef struct SVnodeModifLogicNode {
+  SLogicNode node;;
+  int32_t msgType;
+  SArray* pDataBlocks;
+  SVgDataBlocks* pVgDataBlocks;
+} SVnodeModifLogicNode;
+
+typedef struct SExchangeLogicNode {
+  SLogicNode node;
+  int32_t srcGroupId;
+} SExchangeLogicNode;
+
+typedef enum ESubplanType {
+  SUBPLAN_TYPE_MERGE = 1,
+  SUBPLAN_TYPE_PARTIAL,
+  SUBPLAN_TYPE_SCAN,
+  SUBPLAN_TYPE_MODIFY
+} ESubplanType;
+
+typedef struct SSubplanId {
+  uint64_t queryId;
+  int32_t groupId;
+  int32_t subplanId;
+} SSubplanId;
+
+typedef struct SSubLogicPlan {
+  ENodeType type;
+  SSubplanId id;
+  SNodeList* pChildren;
+  SNodeList* pParents;
+  SLogicNode* pNode;
+  ESubplanType subplanType;
+  SVgroupsInfo* pVgroupList;
+  int32_t level;
+  int32_t splitFlag;
+} SSubLogicPlan;
+
+typedef struct SQueryLogicPlan {
+  ENodeType type;;
+  int32_t totalLevel;
+  SNodeList* pTopSubplans;
+} SQueryLogicPlan;
+
 typedef struct SSlotDescNode {
   ENodeType type;
   int16_t slotId;
@@ -78,11 +124,13 @@ typedef struct SDataBlockDescNode {
   ENodeType type;
   int16_t dataBlockId;
   SNodeList* pSlots;
+  int32_t resultRowSize;
+  int16_t precision;
 } SDataBlockDescNode;
 
 typedef struct SPhysiNode {
   ENodeType type;
-  SDataBlockDescNode outputDataBlockDesc;
+  SDataBlockDescNode* pOutputDataBlockDesc;
   SNode* pConditions;
   SNodeList* pChildren;
   struct SPhysiNode* pParent;
@@ -96,6 +144,7 @@ typedef struct SScanPhysiNode {
   int32_t order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
   int32_t count;         // repeat count
   int32_t reverse;       // reverse scan count
+  SName tableName;
 } SScanPhysiNode;
 
 typedef SScanPhysiNode SSystemTableScanPhysiNode;
@@ -129,6 +178,56 @@ typedef struct SAggPhysiNode {
   SNodeList* pAggFuncs;
 } SAggPhysiNode;
 
+typedef struct SDownstreamSourceNode {
+  ENodeType type;
+  SQueryNodeAddr addr;
+  uint64_t taskId;
+  uint64_t schedId;
+} SDownstreamSourceNode;
+
+typedef struct SExchangePhysiNode {
+  SPhysiNode node;
+  int32_t srcGroupId;  // group id of datasource suplans
+  SNodeList* pSrcEndPoints;  // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode
+} SExchangePhysiNode;
+
+typedef struct SDataSinkNode {
+  ENodeType type;
+  SDataBlockDescNode* pInputDataBlockDesc;
+} SDataSinkNode;
+
+typedef struct SDataDispatcherNode {
+  SDataSinkNode sink;
+} SDataDispatcherNode;
+
+typedef struct SDataInserterNode {
+  SDataSinkNode sink;
+  int32_t   numOfTables;
+  uint32_t  size;
+  char     *pData;
+} SDataInserterNode;
+
+typedef struct SSubplan {
+  ENodeType type;
+  SSubplanId id;           // unique id of the subplan
+  ESubplanType subplanType;
+  int32_t msgType;      // message type for subplan, used to denote the send message type to vnode.
+  int32_t level;        // the execution level of current subplan, starting from 0 in a top-down manner.
+  SQueryNodeAddr execNode;    // for the scan/modify subplan, the optional execution node
+  SQueryNodeStat execNodeStat; // only for scan subplan
+  SNodeList* pChildren;    // the datasource subplan,from which to fetch the result
+  SNodeList* pParents;     // the data destination subplan, get data from current subplan
+  SPhysiNode* pNode;        // physical plan of current subplan
+  SDataSinkNode* pDataSink;    // data of the subplan flow into the datasink
+} SSubplan;
+
+typedef struct SQueryPlan {
+  ENodeType type;;
+  uint64_t queryId;
+  int32_t numOfSubplans;
+  SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
+} SQueryPlan;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/querynodes.h

@@ -21,6 +21,7 @@ extern "C" {
 #endif
 
 #include "nodes.h"
+#include "tmsg.h"
 
 typedef struct SRawExprNode {
   ENodeType nodeType;
@@ -73,6 +74,7 @@ typedef struct SValueNode {
   SExprNode node; // QUERY_NODE_VALUE
   char* literal;
   bool isDuration;
+  bool translate;
   union {
     bool b;
     int64_t i;
@@ -122,6 +124,7 @@ struct STableMeta;
 typedef struct SRealTableNode {
   STableNode table; // QUERY_NODE_REAL_TABLE
   struct STableMeta* pMeta;
+  SVgroupsInfo* pVgroupList;
 } SRealTableNode;
 
 typedef struct STempTableNode {
@@ -248,6 +251,29 @@ typedef enum ESqlClause {
   SQL_CLAUSE_ORDER_BY
 } ESqlClause;
 
+
+typedef enum {
+  PAYLOAD_TYPE_KV = 0,
+  PAYLOAD_TYPE_RAW = 1,
+} EPayloadType;
+
+typedef struct SVgDataBlocks {
+  SVgroupInfo vg;
+  int32_t     numOfTables;  // number of tables in current submit block
+  uint32_t    size;
+  char       *pData;        // SMsgDesc + SSubmitReq + SSubmitBlk + ...
+} SVgDataBlocks;
+
+typedef struct SVnodeModifOpStmt {
+  ENodeType   nodeType;
+  ENodeType   sqlNodeType;
+  SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
+  int8_t      schemaAttache;       // denote if submit block is built with table schema or not
+  uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+  uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
+  const char* sql;                 // current sql statement position
+} SVnodeModifOpStmt;
+
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 

include/libs/parser/parsenodes.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_PARSENODES_H_
-#define _TD_PARSENODES_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "catalog.h"
-#include "tcommon.h"
-#include "function.h"
-#include "tmsgtype.h"
-#include "tname.h"
-#include "tvariant.h"
-
-/**
- * The first field of a node of any type is guaranteed to be the int16_t.
- * Hence the type of any node can be gotten by casting it to SQueryNode. 
- */
-typedef struct SQueryNode {
-  int16_t type;
-} SQueryNode;
-
-#define queryNodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type)
-
-typedef struct SFieldInfo {
-  int16_t     numOfOutput;   // number of column in result
-  SField     *final;
-  SArray     *internalField; // SArray<SInternalField>
-} SFieldInfo;
-
-typedef struct SCond {
-  uint64_t uid;
-  int32_t  len;  // length of tag query condition data
-  char *   cond;
-} SCond;
-
-typedef struct SJoinNode {
-  uint64_t uid;
-  int16_t  tagColId;
-  SArray*  tsJoin;
-  SArray*  tagJoin;
-} SJoinNode;
-
-typedef struct SJoinInfo {
-  bool       hasJoin;
-  SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM];
-} SJoinInfo;
-
-typedef struct STagCond {
-  int16_t   relType;     // relation between tbname list and query condition, including : TK_AND or TK_OR
-  SCond     tbnameCond;  // tbname query condition, only support tbname query condition on one table
-  SJoinInfo joinInfo;    // join condition, only support two tables join currently
-  SArray   *pCond;       // for different table, the query condition must be seperated
-} STagCond;
-
-typedef struct STableMetaInfo {
-  STableMeta    *pTableMeta;      // table meta, cached in client side and acquired by name
-  SVgroupsInfo  *vgroupList;
-  SName         name;
-  char          aliasName[TSDB_TABLE_NAME_LEN];    // alias name of table specified in query sql
-  SArray       *tagColList;                        // SArray<SColumn*>, involved tag columns
-} STableMetaInfo;
-
-typedef struct SColumnIndex {
-  int16_t tableIndex;
-  int16_t columnIndex;
-  int16_t type;               // normal column/tag/ user input constant column
-} SColumnIndex;
-
-// select statement
-typedef struct SQueryStmtInfo {
-  int16_t          command;       // the command may be different for each subclause, so keep it seperately.
-  uint32_t         type;          // query/insert type
-  STimeWindow      window;        // the whole query time window
-  SInterval        interval;      // tumble time window
-  SSessionWindow   sessionWindow; // session time window
-  SStateWindow     stateWindow;   // state window query
-  SGroupbyExpr     groupbyExpr;   // groupby tags info
-  SArray *         colList;       // SArray<SColumn*>
-  SFieldInfo       fieldsInfo;
-  SArray**         exprList;      // SArray<SExprInfo*>
-  SLimit           limit;
-  SLimit           slimit;
-  STagCond         tagCond;
-  SArray *         colCond;
-  SArray *         order;
-  int16_t          numOfTables;
-  int16_t          curTableIdx;
-  STableMetaInfo **pTableMetaInfo;
-  struct STSBuf   *tsBuf;
-
-  int16_t          fillType;      // final result fill type
-  int64_t *        fillVal;       // default value for fill
-  int32_t          numOfFillVal;  // fill value size
-
-  char *           msg;           // pointer to the pCmd->payload to keep error message temporarily
-  int64_t          clauseLimit;   // limit for current sub clause
-
-  int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
-  int64_t          vgroupLimit;    // table limit in case of super table projection query + global order + limit
-
-  int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
-  int32_t          bufLen;
-  char*            buf;
-  SArray          *pUdfInfo;
-
-  struct SQueryStmtInfo *sibling;     // sibling
-  SMultiFunctionsDesc    info;
-  SArray            *pDownstream;   // SArray<struct SQueryStmtInfo>
-  int32_t            havingFieldNum;
-  int32_t            exprListLevelIndex;
-} SQueryStmtInfo;
-
-typedef enum {
-  PAYLOAD_TYPE_KV = 0,
-  PAYLOAD_TYPE_RAW = 1,
-} EPayloadType;
-
-typedef struct SVgDataBlocks {
-  SVgroupInfo vg;
-  int32_t     numOfTables;  // number of tables in current submit block
-  uint32_t    size;
-  char       *pData;        // SMsgDesc + SSubmitReq + SSubmitBlk + ...
-} SVgDataBlocks;
-
-typedef struct SVnodeModifOpStmtInfo {
-  int16_t     nodeType;
-  SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
-  int8_t      schemaAttache;       // denote if submit block is built with table schema or not
-  uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
-  uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
-  const char* sql;                 // current sql statement position
-} SVnodeModifOpStmtInfo;
-
-typedef struct SDclStmtInfo {
-  int16_t     nodeType;
-  int16_t     msgType;
-  SEpSet      epSet;
-  char*       pMsg;
-  int32_t     msgLen;
-  void*       pExtension;  // todo remove it soon
-} SDclStmtInfo;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_PARSENODES_H_*/

include/libs/parser/parser.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#include "parsenodes.h"
+#include "querynodes.h"
 
 typedef struct SParseContext {
   uint64_t         requestId;
@@ -32,67 +32,30 @@ typedef struct SParseContext {
   size_t           sqlLen;         // length of the sql string
   char            *pMsg;           // extended error message if exists to help identifying the problem in sql statement.
   int32_t          msgLen;         // max length of the msg
-
   struct SCatalog *pCatalog;
 } SParseContext;
 
-/**
- * Parse the sql statement and then return the SQueryStmtInfo as the result of bounded AST.
- * @param pSql     sql string
- * @param length   length of the sql string
- * @param id       operator id, generated by uuid generator
- * @param msg      extended error message if exists.
- * @return         error code
- */
-int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQueryNode);
-
-/**
- * Return true if it is a ddl/dcl sql statement
- * @param pQuery
- * @return
- */
-bool qIsDdlQuery(const SQueryNode* pQueryNode);
-
-/**
- * Destroy logic query plan
- * @param pQueryNode
- */
-void qDestroyQuery(SQueryNode* pQueryNode);
-
-/**
- * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that
- * involved in the subscribe procedure.
- * @param pSql
- * @param length
- * @param pConvertSql
- * @return
- */
-int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql);
-
-void assignExprInfo(SExprInfo* dst, const SExprInfo* src);
-void columnListCopy(SArray* dst, const SArray* src, uint64_t uid);
-void columnListDestroy(SArray* pColumnList);
-
-void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel);
-void dropOneLevelExprInfo(SArray* pExprInfo);
-
-typedef struct SSourceParam {
-  SArray    *pExprNodeList; //Array<struct tExprNode*>
-  SArray    *pColumnList;   //Array<struct SColumn>
-  int32_t    num;
-} SSourceParam;
-
-SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize);
-int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);
-int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy);
-int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo);
-
-STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex);
-SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex);
-
-int32_t getNewResColId();
-void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn);
-SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema);
+typedef struct SCmdMsgInfo {
+  int16_t msgType;
+  SEpSet epSet;
+  void* pMsg;
+  int32_t msgLen;
+  void* pExtension;  // todo remove it soon
+} SCmdMsgInfo;
+
+typedef struct SQuery {
+  bool directRpc;
+  bool haveResultSet;
+  SNode* pRoot;
+  int32_t numOfResCols;
+  SSchema* pResSchema;
+  SCmdMsgInfo* pCmdMsg;
+  int32_t msgType;
+} SQuery;
+
+int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
+
+void qDestroyQuery(SQuery* pQueryNode);
 
 #ifdef __cplusplus
 }

include/libs/planner/planner.h

@@ -20,192 +20,31 @@
 extern "C" {
 #endif
 
-#include "query.h"
-#include "tmsg.h"
-#include "tarray.h"
-#include "trpc.h"
+#include "plannodes.h"
 
-#define QUERY_TYPE_MERGE       1
-#define QUERY_TYPE_PARTIAL     2
-#define QUERY_TYPE_SCAN        3
-#define QUERY_TYPE_MODIFY      4
-
-enum OPERATOR_TYPE_E {
-  OP_Unknown,
-#define INCLUDE_AS_ENUM
-#include "plannerOp.h"
-#undef INCLUDE_AS_ENUM
-  OP_TotalNum
-};
-
-enum DATASINK_TYPE_E {
-  DSINK_Unknown,
-  DSINK_Dispatch,
-  DSINK_Insert,
-  DSINK_TotalNum
-};
-
-struct SEpSet;
-struct SQueryStmtInfo;
-
-typedef SSchema SSlotSchema;
-
-typedef struct SDataBlockSchema {
-  SSlotSchema *pSchema;
-  int32_t      numOfCols;    // number of columns
-  int32_t      resultRowSize;
-  int16_t      precision;
-} SDataBlockSchema;
-
-typedef struct SQueryNodeBasicInfo {
-  int32_t     type;          // operator type
-  const char *name;          // operator name
-} SQueryNodeBasicInfo;
-
-typedef struct SDataSink {
-  SQueryNodeBasicInfo info;
-  SDataBlockSchema schema;
-} SDataSink;
-
-typedef struct SDataDispatcher {
-  SDataSink sink;
-} SDataDispatcher;
-
-typedef struct SDataInserter {
-  SDataSink sink;
-  int32_t   numOfTables;
-  uint32_t  size;
-  char     *pData;
-} SDataInserter;
-
-typedef struct SPhyNode {
-  SQueryNodeBasicInfo info;
-  SArray             *pTargets;      // target list to be computed or scanned at this node
-  SArray             *pConditions;   // implicitly-ANDed qual conditions
-  SDataBlockSchema    targetSchema;
-  // children plan to generated result for current node to process
-  // in case of join, multiple plan nodes exist.
-  SArray             *pChildren;
-  struct SPhyNode    *pParent;
-} SPhyNode;
-
-typedef struct SScanPhyNode {
-  SPhyNode    node;
-  uint64_t    uid;           // unique id of the table
-  int8_t      tableType;
-  int32_t     order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
-  int32_t     count;         // repeat count
-  int32_t     reverse;       // reverse scan count
-} SScanPhyNode;
-
-typedef SScanPhyNode SSystemTableScanPhyNode;
-typedef SScanPhyNode STagScanPhyNode;
-
-typedef struct STableScanPhyNode {
-  SScanPhyNode scan;
-  uint8_t      scanFlag;         // denotes reversed scan of data or not
-  STimeWindow  window;
-  SArray      *pTagsConditions; // implicitly-ANDed tag qual conditions
-} STableScanPhyNode;
-
-typedef STableScanPhyNode STableSeqScanPhyNode;
-
-typedef struct SProjectPhyNode {
-  SPhyNode node;
-} SProjectPhyNode;
-
-typedef struct SDownstreamSource {
-  SQueryNodeAddr addr;
-  uint64_t       taskId;
-  uint64_t       schedId;
-} SDownstreamSource;
-
-typedef struct SExchangePhyNode {
-  SPhyNode    node;
-  uint64_t    srcTemplateId;  // template id of datasource suplans
-  SArray     *pSrcEndPoints;  // SArray<SDownstreamSource>, scheduler fill by calling qSetSuplanExecutionNode
-} SExchangePhyNode;
-
-typedef enum EAggAlgo {
-	AGG_ALGO_PLAIN = 1, // simple agg across all input rows
-	AGG_ALGO_SORTED,    // grouped agg, input must be sorted
-	AGG_ALGO_HASHED     // grouped agg, use internal hashtable
-} EAggAlgo;
-
-typedef enum EAggSplit {
-	AGG_SPLIT_PRE = 1, // first level agg, maybe don't need calculate the final result
-	AGG_SPLIT_FINAL    // second level agg, must calculate the final result
-} EAggSplit;
-
-typedef struct SAggPhyNode {
-  SPhyNode    node;
-  EAggAlgo    aggAlgo;  // algorithm used by agg operator 
-  EAggSplit   aggSplit; // distributed splitting mode
-  SArray     *pExprs;   // SExprInfo list, these are expression list of group_by_clause and parameter expression of aggregate function
-  SArray     *pGroupByList; // SColIndex list, but these must be column node
-} SAggPhyNode;
-
-typedef struct SSubplanId {
+typedef struct SPlanContext {
   uint64_t queryId;
-  uint64_t templateId;
-  uint64_t subplanId;
-} SSubplanId;
+  int32_t acctId;
+  SNode* pAstRoot;
+} SPlanContext;
 
-typedef struct SSubplan {
-  SSubplanId id;           // unique id of the subplan
-  int32_t    type;         // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN|QUERY_TYPE_MODIFY
-  int32_t    msgType;      // message type for subplan, used to denote the send message type to vnode.
-  int32_t    level;        // the execution level of current subplan, starting from 0 in a top-down manner.
-  SQueryNodeAddr     execNode;    // for the scan/modify subplan, the optional execution node
-  SArray    *pChildren;    // the datasource subplan,from which to fetch the result
-  SArray    *pParents;     // the data destination subplan, get data from current subplan
-  SPhyNode  *pNode;        // physical plan of current subplan
-  SDataSink *pDataSink;    // data of the subplan flow into the datasink
-} SSubplan;
-
-typedef struct SQueryDag {
-  uint64_t queryId;
-  int32_t  numOfSubplans;
-  SArray  *pSubplans; // SArray*<SArray*<SSubplan*>>. The execution level of subplan, starting from 0.
-} SQueryDag;
-
-struct SQueryNode;
-
- /**
-  * Create the physical plan for the query, according to the AST.
-  * @param pQueryInfo
-  * @param pDag
-  * @param requestId
-  * @return
-  */
- int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId);
+// Create the physical plan for the query, according to the AST.
+int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNodeList);
 
 // Set datasource of this subplan, multiple calls may be made to a subplan.
-// @subplan subplan to be schedule
-// @templateId templateId of a group of datasource subplans of this @subplan
-// @ep one execution location of this group of datasource subplans 
-void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource);
+// @pSubplan subplan to be schedule
+// @groupId id of a group of datasource subplans of this @pSubplan
+// @pSource one execution location of this group of datasource subplans 
+int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, char** str);
-
-/**
- * Convert to subplan to string for the scheduler to send to the executor
- */
-int32_t qSubPlanToString(const SSubplan* subplan, char** str, int32_t* len);
+// Convert to subplan to string for the scheduler to send to the executor
+int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);
+int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan);
 
-int32_t qStringToSubplan(const char* str, SSubplan** subplan);
-
-void qDestroySubplan(SSubplan* pSubplan);
-
-/**
- * Destroy the physical plan.
- * @param pQueryPhyNode
- * @return
- */
-void qDestroyQueryDag(SQueryDag* pDag);
+char* qQueryPlanToString(const SQueryPlan* pPlan);
+SQueryPlan* qStringToQueryPlan(const char* pStr);
 
-char* qDagToString(const SQueryDag* pDag);
-SQueryDag* qStringToDag(const char* pStr);
+void qDestroyQueryPlan(SQueryPlan* pPlan);
 
 #ifdef __cplusplus
 }

include/libs/planner/plannerOp.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#if defined(INCLUDE_AS_ENUM)  // enum define mode
-  #undef OP_ENUM_MACRO
-  #define OP_ENUM_MACRO(op)    OP_##op,
-#elif defined(INCLUDE_AS_NAME)  // comment define mode
-  #undef OP_ENUM_MACRO
-  #define OP_ENUM_MACRO(op)    #op,
-#else
-  #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME
-#endif
-
-OP_ENUM_MACRO(StreamScan)
-OP_ENUM_MACRO(TableScan)
-OP_ENUM_MACRO(TableSeqScan)
-OP_ENUM_MACRO(TagScan)
-OP_ENUM_MACRO(SystemTableScan)
-OP_ENUM_MACRO(StreamBlockScan)
-OP_ENUM_MACRO(Aggregate)
-OP_ENUM_MACRO(Project)
-// OP_ENUM_MACRO(Groupby)
-OP_ENUM_MACRO(Limit)
-OP_ENUM_MACRO(SLimit)
-OP_ENUM_MACRO(TimeWindow)
-OP_ENUM_MACRO(SessionWindow)
-OP_ENUM_MACRO(StateWindow)
-OP_ENUM_MACRO(Fill)
-OP_ENUM_MACRO(MultiTableAggregate)
-OP_ENUM_MACRO(MultiTableTimeInterval)
-OP_ENUM_MACRO(Filter)
-OP_ENUM_MACRO(Distinct)
-OP_ENUM_MACRO(Join)
-OP_ENUM_MACRO(AllTimeWindow)
-OP_ENUM_MACRO(AllMultiTableTimeInterval)
-OP_ENUM_MACRO(Order)
-OP_ENUM_MACRO(Exchange)
-OP_ENUM_MACRO(SortedMerge)
-
-//OP_ENUM_MACRO(TableScan)

include/libs/qcom/query.h

@@ -35,7 +35,6 @@ enum {
   JOB_TASK_STATUS_CANCELLING,
   JOB_TASK_STATUS_CANCELLED,
   JOB_TASK_STATUS_DROPPING,
-  JOB_TASK_STATUS_FREEING,
 };
 
 enum {
@@ -85,7 +84,8 @@ typedef struct STableMeta {
 typedef struct SDBVgInfo {
   int32_t   vgVersion;
   int8_t    hashMethod;
-  SHashObj* vgHash;  // key:vgId, value:SVgroupInfo
+  int32_t   numOfTable;  // DB's table num, unit is TSDB_TABLE_NUM_UNIT
+  SHashObj *vgHash;  //key:vgId, value:SVgroupInfo
 } SDBVgInfo;
 
 typedef struct SUseDbOutput {
@@ -129,6 +129,11 @@ typedef struct SQueryNodeAddr {
   SEpSet  epSet;
 } SQueryNodeAddr;
 
+
+typedef struct SQueryNodeStat {
+  int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
+} SQueryNodeStat;
+
 int32_t initTaskQueue();
 int32_t cleanupTaskQueue();
 

include/libs/qworker/qworker.h

@@ -72,6 +72,8 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
 int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
+int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+
 int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
 int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);

include/libs/scheduler/scheduler.h

@@ -25,6 +25,7 @@ extern "C" {
 
 typedef struct SSchedulerCfg {
   uint32_t maxJobNum;
+  int32_t  maxNodeTableNum;
 } SSchedulerCfg;
 
 typedef struct SQueryProfileSummary {
@@ -70,7 +71,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg);
  * @param nodeList  Qnode/Vnode address list, element is SQueryNodeAddr
  * @return
  */
-int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, int64_t *pJob, const char* sql, SQueryResult *pRes);
+int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan* pDag, int64_t* pJob, const char* sql, SQueryResult *pRes);
 
 /**
  * Process the query job, generated according to the query physical plan.
@@ -78,7 +79,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, int
  * @param pNodeList  Qnode/Vnode address list, element is SQueryNodeAddr
  * @return
  */
-int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, int64_t *pJob);
+int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryPlan* pDag, const char* sql, int64_t *pJob);
 
 /**
  * Fetch query result from the remote query executor
@@ -110,7 +111,7 @@ void schedulerDestroy(void);
  * @param pTasks SArray**<STaskInfo>
  * @return
  */
-int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks);
+int32_t schedulerConvertDagToTaskList(SQueryPlan* pDag, SArray **pTasks);
 
 /**
  * make one task info's multiple copies

include/libs/transport/trpc.h

@@ -64,7 +64,6 @@ typedef struct SRpcInit {
   int8_t   connType;      // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
   int      idleTime;      // milliseconds, 0 means idle timer is disabled
 
-  bool noPool;  //  create conn pool or not
   // the following is for client app ecurity only
   char *user;     // user name
   char  spi;      // security parameter index
@@ -86,7 +85,7 @@ typedef struct SRpcInit {
 
 int32_t rpcInit();
 void    rpcCleanup();
-void   *rpcOpen(const SRpcInit *pRpc);
+void *  rpcOpen(const SRpcInit *pRpc);
 void    rpcClose(void *);
 void *  rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);
@@ -99,6 +98,9 @@ void    rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp)
 int     rpcReportProgress(void *pConn, char *pCont, int contLen);
 void    rpcCancelRequest(int64_t rid);
 
+void rpcRefHandle(void *handle, int8_t type);
+void rpcUnrefHandle(void *handle, int8_t type);
+
 #ifdef __cplusplus
 }
 #endif

include/os/os.h

@@ -22,7 +22,22 @@ extern "C" {
 
 #include <assert.h>
 #include <ctype.h>
+
+#if !defined(WINDOWS)
+#include <unistd.h>
 #include <dirent.h>
+#include <regex.h>
+#include <sched.h>
+#include <wordexp.h>
+#include <libgen.h>
+
+#include <sys/utsname.h>
+#include <sys/param.h>
+#include <sys/mman.h>
+#include <sys/prctl.h>
+
+#endif
+
 #include <errno.h>
 #include <fcntl.h>
 #include <float.h>
@@ -30,8 +45,6 @@ extern "C" {
 #include <limits.h>
 #include <locale.h>
 #include <math.h>
-#include <regex.h>
-#include <sched.h>
 #include <setjmp.h>
 #include <signal.h>
 #include <stdarg.h>
@@ -43,16 +56,9 @@ extern "C" {
 #include <string.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <sys/utsname.h>
-#include <sys/param.h>
-#include <unistd.h>
 #include <wchar.h>
 #include <wctype.h>
-#include <wordexp.h>
-#include <libgen.h>
 
-#include <sys/mman.h>
-#include <sys/prctl.h>
 
 #include "osAtomic.h"
 #include "osDef.h"

include/os/osDef.h

@@ -49,7 +49,7 @@ extern "C" {
 #endif
 
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#if defined(WINDOWS)
   char *stpcpy (char *dest, const char *src);
   char *stpncpy (char *dest, const char *src, size_t n);
 
@@ -63,16 +63,46 @@ extern "C" {
   #define strtok_r strtok_s
   #define snprintf _snprintf
   #define in_addr_t unsigned long
-  #define socklen_t int
+//  #define socklen_t int
 
-  struct tm *localtime_r(const time_t *timep, struct tm *result);
-  char *     strptime(const char *buf, const char *fmt, struct tm *tm);
   char *     strsep(char **stringp, const char *delim);
   char *     getpass(const char *prefix);
   char *     strndup(const char *s, size_t n);
-  
-#endif
+  int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 
+  // for send function in tsocket.c
+  #define MSG_NOSIGNAL             0
+  #define SO_NO_CHECK              0x1234
+  #define SOL_TCP                  0x1234
+
+  #define SHUT_RDWR                SD_BOTH
+  #define SHUT_RD                  SD_RECEIVE
+  #define SHUT_WR                  SD_SEND
+
+  #define LOCK_EX 1
+  #define LOCK_NB 2
+  #define LOCK_UN 3
+
+  #ifndef PATH_MAX
+  #define PATH_MAX 256
+  #endif
+
+  typedef struct {
+    int    we_wordc;
+    char  *we_wordv[1];
+    int    we_offs;
+    char   wordPos[1025];
+  } wordexp_t;
+  int  wordexp(char *words, wordexp_t *pwordexp, int flags);
+  void wordfree(wordexp_t *pwordexp);
+
+  #define openlog(a, b, c)
+  #define closelog()
+  #define LOG_ERR 0
+  #define LOG_INFO 1
+  void syslog(int unused, const char *format, ...);
+#endif // WINDOWS
+  
 #ifndef WINDOWS
   #ifndef O_BINARY
     #define O_BINARY 0
@@ -166,7 +196,7 @@ extern "C" {
   #define PRIzu "zu"  
 #endif
 
-#if defined(_TD_LINUX_64) || defined(_TD_LINUX_32) || defined(_TD_MIPS_64)  || defined(_TD_ARM_32) || defined(_TD_ARM_64)  || defined(_TD_DARWIN_64)
+#if !defined(WINDOWS)
   #if defined(_TD_DARWIN_64)
     // MacOS
     #if !defined(_GNU_SOURCE)
@@ -181,8 +211,7 @@ extern "C" {
   #endif
 #else
   // Windows
-//  #define setThreadName(name)
-#define setThreadName(name) do { prctl(PR_SET_NAME, (name)); } while (0)
+  #define setThreadName(name)
 #endif
 
 #if defined(_WIN32)
@@ -199,4 +228,4 @@ extern "C" {
 }
 #endif
 
-#endif /*_TD_OS_DEF_H_*/
+#endif /*_TD_OS_DEF_H_*/
\ No newline at end of file

include/os/osFile.h

@@ -22,6 +22,15 @@ extern "C" {
 
 #include "osSocket.h"
 
+#if defined(WINDOWS)
+typedef int32_t FileFd;
+typedef SOCKET  SocketFd;
+#else
+typedef int32_t FileFd;
+typedef int32_t SocketFd;
+#endif
+
+int64_t taosRead(FileFd fd, void *buf, int64_t count);
 // If the error is in a third-party library, place this header file under the third-party library header file.
 #ifndef ALLOW_FORBID_FUNC
     #define open OPEN_FUNC_TAOS_FORBID
@@ -76,7 +85,13 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
 int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
 int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
 void    taosFprintfFile(TdFilePtr pFile, const char *format, ...);
+
+#if defined(WINDOWS)
+#define __restrict__
+#endif // WINDOWS
+
 int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
+
 int32_t taosEOFFile(TdFilePtr pFile);
  
 int64_t taosCloseFile(TdFilePtr *ppFile);

include/os/osSemaphore.h

@@ -16,12 +16,13 @@
 #ifndef _TD_OS_SEMPHONE_H_
 #define _TD_OS_SEMPHONE_H_
 
-#include <semaphore.h>
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#include <pthread.h>
+#include <semaphore.h>
+
 #if defined (_TD_DARWIN_64)
   typedef struct tsem_s *tsem_t;
   int tsem_init(tsem_t *sem, int pshared, unsigned int value);

include/os/osSocket.h

@@ -27,7 +27,7 @@
     #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#if defined(WINDOWS)
   #include "winsock2.h"
   #include <WS2tcpip.h>
   #include <winbase.h>

include/os/osSysinfo.h

@@ -47,12 +47,18 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
 int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
 int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
 
-int32_t taosSystem(const char *cmd);
 void    taosKillSystem();
 int32_t taosGetSystemUUID(char *uid, int32_t uidlen);
 char   *taosGetCmdlineByPID(int32_t pid);
 void    taosSetCoreDump(bool enable);
 
+#if defined(WINDOWS)
+
+#define _UTSNAME_LENGTH 65
+#define _UTSNAME_MACHINE_LENGTH _UTSNAME_LENGTH
+
+#endif // WINDOWS
+
 typedef struct {
   char sysname[_UTSNAME_MACHINE_LENGTH];
   char nodename[_UTSNAME_MACHINE_LENGTH];

include/os/osSystem.h

@@ -20,11 +20,23 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define popen POPEN_FUNC_TAOS_FORBID
+    #define pclose PCLOSE_FUNC_TAOS_FORBID
+    #define tcsetattr TCSETATTR_FUNC_TAOS_FORBID
+    #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
+#endif
+
+int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize);
 void* taosLoadDll(const char* filename);
 void* taosLoadSym(void* handle, char* name);
 void  taosCloseDll(void* handle);
 
 int32_t taosSetConsoleEcho(bool on);
+void setTerminalMode();
+int32_t getOldTerminalMode();
+void resetTerminalMode();
 
 #ifdef __cplusplus
 }

include/os/osTime.h

@@ -20,7 +20,19 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define strptime STRPTIME_FUNC_TAOS_FORBID
+    #define gettimeofday GETTIMEOFDAY_FUNC_TAOS_FORBID
+    #define localtime_s LOCALTIMES_FUNC_TAOS_FORBID
+    #define localtime_r LOCALTIMER_FUNC_TAOS_FORBID
+    #define time TIME_FUNC_TAOS_FORBID
+#endif
+
 #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+
+  #define CLOCK_REALTIME 	0
+
   #ifdef _TD_GO_DLL_
     #define MILLISECOND_PER_SECOND (1000LL)
   #else
@@ -61,6 +73,9 @@ static FORCE_INLINE int64_t taosGetTimestampNs() {
   return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec;
 }
 
+char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm);
+struct tm *taosLocalTime(const time_t *timep, struct tm *result);
+
 #ifdef __cplusplus
 }
 #endif

include/os/osTimer.h

@@ -20,6 +20,15 @@
 extern "C" {
 #endif
 
+// If the error is in a third-party library, place this header file under the third-party library header file.
+#ifndef ALLOW_FORBID_FUNC
+    #define timer_create TIMER_CREATE_FUNC_TAOS_FORBID
+    #define timer_settime TIMER_SETTIME_FUNC_TAOS_FORBID
+    #define timer_delete TIMER_DELETE_FUNC_TAOS_FORBID
+    #define timeSetEvent TIMESETEVENT_SETTIME_FUNC_TAOS_FORBID
+    #define timeKillEvent TIMEKILLEVENT_SETTIME_FUNC_TAOS_FORBID
+#endif
+
 #define MSECONDS_PER_TICK 5
 
 int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms);

include/util/taoserror.h

@@ -267,9 +267,11 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
 #define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
-#define TSDB_CODE_MND_MQ_PLACEHOLDER            TAOS_DEF_ERROR_CODE(0, 0x03F0)
-
 
+// mnode-stream
+#define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
+#define TSDB_CODE_MND_STREAM_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03F1)
+#define TSDB_CODE_MND_INVALID_STREAM_OPTION     TAOS_DEF_ERROR_CODE(0, 0x03F2)
 
 // dnode
 #define TSDB_CODE_DND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0400)
@@ -458,6 +460,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SCH_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x2502)
 
 //parser
+#define TSDB_CODE_PAR_SYNTAX_ERROR              TAOS_DEF_ERROR_CODE(0, 0x2600)
 #define TSDB_CODE_PAR_INVALID_COLUMN            TAOS_DEF_ERROR_CODE(0, 0x2601)
 #define TSDB_CODE_PAR_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x2602)
 #define TSDB_CODE_PAR_AMBIGUOUS_COLUMN          TAOS_DEF_ERROR_CODE(0, 0x2603)
@@ -470,6 +473,13 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION   TAOS_DEF_ERROR_CODE(0, 0x260A)
 #define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION   TAOS_DEF_ERROR_CODE(0, 0x260B)
 #define TSDB_CODE_PAR_NOT_SINGLE_GROUP          TAOS_DEF_ERROR_CODE(0, 0x260C) 
+#define TSDB_CODE_PAR_TAGS_NOT_MATCHED          TAOS_DEF_ERROR_CODE(0, 0x260D)
+#define TSDB_CODE_PAR_INVALID_TAG_NAME          TAOS_DEF_ERROR_CODE(0, 0x260E)
+#define TSDB_CODE_PAR_INCOMPLETE_SQL            TAOS_DEF_ERROR_CODE(0, 0x260F)
+#define TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG   TAOS_DEF_ERROR_CODE(0, 0x2610)
+#define TSDB_CODE_PAR_PASSWD_EMPTY              TAOS_DEF_ERROR_CODE(0, 0x2611)
+#define TSDB_CODE_PAR_INVALID_PORT              TAOS_DEF_ERROR_CODE(0, 0x2612)
+#define TSDB_CODE_PAR_INVALID_ENDPOINT          TAOS_DEF_ERROR_CODE(0, 0x2613)
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -211,6 +211,7 @@ typedef enum ELogicConditionType {
 #define TSDB_TYPE_STR_MAX_LEN    32
 #define TSDB_TABLE_FNAME_LEN     (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
 #define TSDB_TOPIC_FNAME_LEN     TSDB_TABLE_FNAME_LEN
+#define TSDB_STREAM_FNAME_LEN    TSDB_TABLE_FNAME_LEN
 #define TSDB_SUBSCRIBE_KEY_LEN   (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
 #define TSDB_PARTITION_KEY_LEN   (TSDB_SUBSCRIBE_KEY_LEN + 20)
 #define TSDB_COL_NAME_LEN        65
@@ -344,6 +345,20 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_DB_QUORUM_OPTION     2
 #define TSDB_DEFAULT_DB_QUORUM_OPTION 1
 
+#define TSDB_MIN_DB_TTL_OPTION          1
+#define TSDB_DEFAULT_DB_TTL_OPTION      0
+
+#define TSDB_MIN_DB_SINGLE_STABLE_OPTION          0
+#define TSDB_MAX_DB_SINGLE_STABLE_OPTION          1
+#define TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION      0
+
+#define TSDB_MIN_DB_STREAM_MODE_OPTION          0
+#define TSDB_MAX_DB_STREAM_MODE_OPTION          1
+#define TSDB_DEFAULT_DB_STREAM_MODE_OPTION      0
+
+#define TSDB_MAX_JOIN_TABLE_NUM         10
+#define TSDB_MAX_UNION_CLAUSE           5
+
 #define TSDB_MIN_DB_UPDATE            0
 #define TSDB_MAX_DB_UPDATE            2
 #define TSDB_DEFAULT_DB_UPDATE_OPTION 0

include/util/tjson.h

@@ -54,10 +54,12 @@ typedef int32_t (*FToJson)(const void* pObj, SJson* pJson);
 
 int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj);
 int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj);
+int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, int32_t num);
 
 typedef int32_t (*FToObject)(const SJson* pJson, void* pObj);
 
 int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj);
+int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize);
 
 char* tjsonToString(const SJson* pJson);
 char* tjsonToUnformattedString(const SJson* pJson);

source/client/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 target_link_libraries(
     taos
     INTERFACE api
-    PRIVATE os util common transport parser planner catalog scheduler function qcom
+    PRIVATE os util common transport nodes parser planner catalog scheduler function qcom
 )
 
 if(${BUILD_TEST})

source/client/inc/clientInt.h

@@ -163,13 +163,13 @@ typedef struct SShowReqInfo {
 } SShowReqInfo;
 
 typedef struct SRequestSendRecvBody {
-  tsem_t            rspSem;  // not used now
-  void*             fp;
-  SShowReqInfo      showInfo;  // todo this attribute will be removed after the query framework being completed.
-  SDataBuf          requestMsg;
-  int64_t           queryJob;  // query job, created according to sql query DAG.
-  struct SQueryDag* pDag;      // the query dag, generated according to the sql statement.
-  SReqResultInfo    resInfo;
+  tsem_t             rspSem;  // not used now
+  void*              fp;
+  SShowReqInfo       showInfo;  // todo this attribute will be removed after the query framework being completed.
+  SDataBuf           requestMsg;
+  int64_t            queryJob;  // query job, created according to sql query DAG.
+  struct SQueryPlan* pDag;       // the query dag, generated according to the sql statement.
+  SReqResultInfo     resInfo;
 } SRequestSendRecvBody;
 
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
@@ -228,7 +228,7 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
-int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery);
+int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery);
 
 // --- heartbeat
 // global, called by mgmt

source/client/src/clientEnv.c

@@ -72,8 +72,6 @@ static void deregisterRequest(SRequestObj *pRequest) {
   taosReleaseRef(clientConnRefPool, pTscObj->id);
 }
 
-
-
 // todo close the transporter properly
 void closeTransporter(STscObj *pTscObj) {
   if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) {
@@ -184,7 +182,7 @@ static void doDestroyRequest(void *p) {
   tfree(pRequest->pInfo);
 
   doFreeReqResultInfo(&pRequest->body.resInfo);
-  qDestroyQueryDag(pRequest->body.pDag);
+  qDestroyQueryPlan(pRequest->body.pDag);
 
   if (pRequest->body.showInfo.pArray != NULL) {
     taosArrayDestroy(pRequest->body.showInfo.pArray);
@@ -241,6 +239,7 @@ void taos_init_imp(void) {
   clientConnRefPool = taosOpenRef(200, destroyTscObj);
   clientReqRefPool = taosOpenRef(40960, doDestroyRequest);
 
+  // transDestroyBuffer(&conn->readBuf);
   taosGetAppName(appInfo.appName, NULL);
   pthread_mutex_init(&appInfo.mutex, NULL);
 

source/client/src/clientHb.c

@@ -224,6 +224,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
     SDbVgVersion *db = &dbs[i];
     db->dbId = htobe64(db->dbId);
     db->vgVersion = htonl(db->vgVersion);
+    db->numOfTable = htonl(db->numOfTable);
   }
 
   SKv kv = {
@@ -434,11 +435,11 @@ static int32_t hbCreateThread() {
   pthread_attr_init(&thAttr);
   pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-  if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-  pthread_attr_destroy(&thAttr);
+//  if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) {
+//    terrno = TAOS_SYSTEM_ERROR(errno);
+//    return -1;
+//  }
+//  pthread_attr_destroy(&thAttr);
   return 0;
 }
 

source/client/src/clientImpl.c

@@ -139,7 +139,7 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) {
+int32_t parseSql(SRequestObj* pRequest, SQuery** pQuery) {
   STscObj* pTscObj = pRequest->pTscObj;
 
   SParseContext cxt = {
@@ -161,53 +161,44 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) {
   }
 
   code = qParseQuerySql(&cxt, pQuery);
+  if (TSDB_CODE_SUCCESS == code && ((*pQuery)->haveResultSet)) {
+    setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
+  }
 
   tfree(cxt.db);
   return code;
 }
 
-int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
-  SDclStmtInfo* pDcl = (SDclStmtInfo*)pQuery;
-  pRequest->type = pDcl->msgType;
-  pRequest->body.requestMsg = (SDataBuf){.pData = pDcl->pMsg, .len = pDcl->msgLen, .handle = NULL};
+int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
+  SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg;
+  pRequest->type = pMsgInfo->msgType;
+  pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL};
+  pMsgInfo->pMsg = NULL; // pMsg transferred to SMsgSendInfo management
 
   STscObj*      pTscObj = pRequest->pTscObj;
   SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest);
 
-  int64_t transporterId = 0;
-  if (pDcl->msgType == TDMT_VND_CREATE_TABLE || pDcl->msgType == TDMT_VND_SHOW_TABLES) {
-    if (pDcl->msgType == TDMT_VND_SHOW_TABLES) {
-      SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
-      if (pShowReqInfo->pArray == NULL) {
-        pShowReqInfo->currentIndex = 0;  // set the first vnode/ then iterate the next vnode
-        pShowReqInfo->pArray = pDcl->pExtension;
-      }
+  if (pMsgInfo->msgType == TDMT_VND_SHOW_TABLES) {
+    SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
+    if (pShowReqInfo->pArray == NULL) {
+      pShowReqInfo->currentIndex = 0;  // set the first vnode/ then iterate the next vnode
+      pShowReqInfo->pArray = pMsgInfo->pExtension;
     }
-    asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg);
-  } else {
-    asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg);
   }
+  int64_t transporterId = 0;
+  asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg);
 
   tsem_wait(&pRequest->body.rspSem);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, SArray* pNodeList) {
-  pRequest->type = pQueryNode->type;
-
-  SSchema* pSchema = NULL;
-  int32_t  numOfCols = 0;
-  int32_t  code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &numOfCols, pNodeList, pRequest->requestId);
+int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
+  pRequest->type = pQuery->msgType;
+  SPlanContext cxt = { .queryId = pRequest->requestId, .pAstRoot = pQuery->pRoot, .acctId = pRequest->pTscObj->acctId };
+  int32_t  code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
   if (code != 0) {
     return code;
   }
-
-  if (pQueryNode->type == TSDB_SQL_SELECT) {
-    setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols);
-    pRequest->type = TDMT_VND_QUERY;
-  }
-
-  tfree(pSchema);
   return code;
 }
 
@@ -224,8 +215,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
   }
 }
 
-int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) {
+
+int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
   void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;
+
   SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
   int32_t      code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, &res);
   if (code != TSDB_CODE_SUCCESS) {
@@ -237,14 +230,14 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList)
     return pRequest->code;
   }
 
-  if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
+  if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) {
     pRequest->body.resInfo.numOfRows = res.numOfRows;
-    
+
     if (pRequest->body.queryJob != 0) {
       schedulerFreeJob(pRequest->body.queryJob);
     }
   }
-  
+
   pRequest->code = res.code;
   return pRequest->code;
 }
@@ -258,24 +251,24 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
   }
 
   SRequestObj* pRequest = NULL;
-  SQueryNode*  pQueryNode = NULL;
-  SArray*      pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
+  SQuery* pQuery = NULL;
+  SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
 
   terrno = TSDB_CODE_SUCCESS;
   CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
-  CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
+  CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
 
-  if (qIsDdlQuery(pQueryNode)) {
-    CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return);
+  if (pQuery->directRpc) {
+    CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
   } else {
-    CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return);
+    CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return);
     CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return);
     pRequest->code = terrno;
   }
 
 _return:
   taosArrayDestroy(pNodeList);
-  qDestroyQuery(pQueryNode);
+  qDestroyQuery(pQuery);
   if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) {
     pRequest->code = terrno;
   }

source/client/src/tmq.c

@@ -459,7 +459,7 @@ void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->
 TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
   STscObj*     pTscObj = (STscObj*)taos;
   SRequestObj* pRequest = NULL;
-  SQueryNode*  pQueryNode = NULL;
+  SQuery*  pQueryNode = NULL;
   char*        pStr = NULL;
 
   terrno = TSDB_CODE_SUCCESS;
@@ -482,7 +482,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   }
 
   tscDebug("start to create topic, %s", topicName);
-
+#if 0
   CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
   CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
 
@@ -511,7 +511,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB);
   tNameFromString(&name, topicName, T_NAME_TABLE);
 
-  SMCreateTopicReq req = {
+  SCMCreateTopicReq req = {
       .igExists = 1,
       .physicalPlan = (char*)pStr,
       .sql = (char*)sql,
@@ -519,13 +519,13 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   };
   tNameExtractFullName(&name, req.name);
 
-  int   tlen = tSerializeMCreateTopicReq(NULL, 0, &req);
+  int   tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req);
   void* buf = malloc(tlen);
   if (buf == NULL) {
     goto _return;
   }
 
-  tSerializeMCreateTopicReq(buf, tlen, &req);
+  tSerializeSCMCreateTopicReq(buf, tlen, &req);
   /*printf("formatted: %s\n", dagStr);*/
 
   pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL};
@@ -538,7 +538,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
   asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
   tsem_wait(&pRequest->body.rspSem);
-
+#endif
 _return:
   qDestroyQuery(pQueryNode);
   /*if (sendInfo != NULL) {*/

source/client/test/clientTests.cpp

@@ -53,7 +53,6 @@ TEST(testCase, driverInit_Test) {
 //  taos_init();
 }
 
-#if 1
 TEST(testCase, connect_Test) {
 //  taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg");
 
@@ -564,20 +563,18 @@ TEST(testCase, insert_test) {
   taos_free_result(pRes);
   taos_close(pConn);
 }
-#endif
-
 
 TEST(testCase, projection_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
-  if (taos_errno(pRes) != 0) {
-    printf("error in create db, reason:%s\n", taos_errstr(pRes));
-  }
-  taos_free_result(pRes);
+//  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
+//  if (taos_errno(pRes) != 0) {
+//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
+//  }
+//  taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "use abc1");
+  TAOS_RES* pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
   pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
@@ -592,7 +589,7 @@ TEST(testCase, projection_query_tables) {
   }
   taos_free_result(pRes);
 
-  for(int32_t i = 0; i < 10000000; ++i) {
+  for(int32_t i = 0; i < 10000; ++i) {
     char sql[512] = {0};
     sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
     TAOS_RES* p = taos_query(pConn, sql);
@@ -623,7 +620,6 @@ TEST(testCase, projection_query_tables) {
   taos_free_result(pRes);
   taos_close(pConn);
 }
-#if 0
 
 TEST(testCase, projection_query_stables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -686,6 +682,5 @@ TEST(testCase, agg_query_tables) {
   taos_free_result(pRes);
   taos_close(pConn);
 }
-#endif
 
 #pragma GCC diagnostic pop

source/common/CMakeLists.txt

@@ -4,6 +4,10 @@ target_include_directories(
     common
     PUBLIC "${CMAKE_SOURCE_DIR}/include/common"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+IF(${TD_WINDOWS})
+    PRIVATE "${CMAKE_SOURCE_DIR}/contrib/pthread-win32"
+    PRIVATE "${CMAKE_SOURCE_DIR}/contrib/gnuregex"
+ENDIF ()
 )
 target_link_libraries(
     common

source/common/src/tdatablock.c

@@ -239,7 +239,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
   return numOfRow1 + numOfRow2;
 }
 
-size_t colDataGetNumOfCols(const SSDataBlock* pBlock) {
+size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) {
   ASSERT(pBlock);
 
   size_t constantCols = (pBlock->pConstantList != NULL)? taosArrayGetSize(pBlock->pConstantList):0;
@@ -247,7 +247,7 @@ size_t colDataGetNumOfCols(const SSDataBlock* pBlock) {
   return pBlock->info.numOfCols;
 }
 
-size_t colDataGetNumOfRows(const SSDataBlock* pBlock) {
+size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) {
   return pBlock->info.rows;
 }
 

source/common/src/tmsg.c

@@ -1007,11 +1007,32 @@ int32_t tDeserializeSMCreateDropQSBNodeReq(void *buf, int32_t bufLen, SMCreateQn
 }
 
 int32_t tSerializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) {
-  return tSerializeSMCreateDropQSBNodeReq(buf, bufLen, (SMCreateQnodeReq *)pReq);
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->fqdn) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->port) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
 }
 
 int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) {
-  return tDeserializeSMCreateDropQSBNodeReq(buf, bufLen, (SMCreateQnodeReq *)pReq);
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->fqdn) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->port) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
 }
 
 int32_t tSerializeSMCreateDropMnodeReq(void *buf, int32_t bufLen, SMCreateMnodeReq *pReq) {
@@ -1423,6 +1444,7 @@ int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) {
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->dbId) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->numOfTable) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1438,12 +1460,77 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) {
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->dbId) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numOfTable) < 0) return -1;
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);
   return 0;
 }
 
+
+int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->rowNum) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSQnodeListReq(void *buf, int32_t bufLen, SQnodeListReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->rowNum) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  int32_t num = taosArrayGetSize(pRsp->epSetList);
+  if (tEncodeI32(&encoder, num) < 0) return -1;    
+  for (int32_t i = 0; i < num; ++i) {
+    SEpSet *epSet = taosArrayGet(pRsp->epSetList, i);
+    if (tEncodeSEpSet(&encoder, epSet) < 0) return -1;
+  }
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  int32_t num = 0;
+  if (tDecodeI32(&decoder, &num) < 0) return -1;
+  pRsp->epSetList = taosArrayInit(num, sizeof(SEpSet));
+  if (NULL == pRsp->epSetList) return -1;
+  for (int32_t i = 0; i < num; ++i) {
+    if (tDecodeSEpSet(&decoder, TARRAY_GET_ELEM(pRsp->epSetList, i)) < 0) return -1;
+  }
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); }
+
 int32_t tSerializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -1482,6 +1569,7 @@ static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) {
     if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1;
     if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1;
     if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1;
+    if (tEncodeI32(pEncoder, pVgInfo->numOfTable) < 0) return -1;
   }
 
   return 0;
@@ -1542,6 +1630,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
     if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1;
     if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1;
     if (tDecodeSEpSet(pDecoder, &vgInfo.epSet) < 0) return -1;
+    if (tDecodeI32(pDecoder, &vgInfo.numOfTable) < 0) return -1;
     taosArrayPush(pRsp->pVgroupInfos, &vgInfo);
   }
 
@@ -1899,7 +1988,7 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR
   return 0;
 }
 
-int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopicReq *pReq) {
+int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTopicReq *pReq) {
   int32_t sqlLen = 0;
   int32_t physicalPlanLen = 0;
   int32_t logicalPlanLen = 0;
@@ -1927,7 +2016,7 @@ int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopic
   return tlen;
 }
 
-int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq *pReq) {
+int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicReq *pReq) {
   int32_t sqlLen = 0;
   int32_t physicalPlanLen = 0;
   int32_t logicalPlanLen = 0;
@@ -1956,13 +2045,13 @@ int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq
   return 0;
 }
 
-void tFreeSMCreateTopicReq(SMCreateTopicReq *pReq) {
+void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) {
   tfree(pReq->sql);
   tfree(pReq->physicalPlan);
   tfree(pReq->logicalPlan);
 }
 
-int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopicRsp *pRsp) {
+int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
@@ -1975,7 +2064,7 @@ int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopi
   return tlen;
 }
 
-int32_t tDeserializeSMCreateTopicRsp(void *buf, int32_t bufLen, SMCreateTopicRsp *pRsp) {
+int32_t tDeserializeSCMCreateTopicRsp(void *buf, int32_t bufLen, SCMCreateTopicRsp *pRsp) {
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
@@ -2391,6 +2480,149 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq)
   return 0;
 }
 
+int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+  if (buf != NULL) {
+    buf = (char *)buf + headLen;
+    bufLen -= headLen;
+  }
+
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeU64(&encoder, pReq->sId) < 0) return -1; 
+  if (tEncodeI32(&encoder, pReq->epId.nodeId) < 0) return -1; 
+  if (tEncodeU16(&encoder, pReq->epId.ep.port) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->epId.ep.fqdn) < 0) return -1;  
+  if (pReq->taskAction) {
+    int32_t num = taosArrayGetSize(pReq->taskAction);
+    if (tEncodeI32(&encoder, num) < 0) return -1;    
+    for (int32_t i = 0; i < num; ++i) {
+      STaskAction *action = taosArrayGet(pReq->taskAction, i);
+      if (tEncodeU64(&encoder, action->queryId) < 0) return -1;    
+      if (tEncodeU64(&encoder, action->taskId) < 0) return -1;    
+      if (tEncodeI8(&encoder, action->action) < 0) return -1;    
+    }
+  } else {
+    if (tEncodeI32(&encoder, 0) < 0) return -1;    
+  }
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  
+  if (buf != NULL) {
+    SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen);
+    pHead->vgId = htonl(pReq->header.vgId);
+    pHead->contLen = htonl(tlen + headLen);
+  }
+
+  return tlen + headLen;
+}
+
+int32_t tDeserializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+
+  SMsgHead *pHead = buf;
+  pHead->vgId = pReq->header.vgId;
+  pHead->contLen = pReq->header.contLen;
+
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, (char *)buf + headLen, bufLen - headLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeU64(&decoder, &pReq->sId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->epId.nodeId) < 0) return -1;
+  if (tDecodeU16(&decoder, &pReq->epId.ep.port) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->epId.ep.fqdn) < 0) return -1;
+  int32_t num = 0;
+  if (tDecodeI32(&decoder, &num) < 0) return -1;
+  if (num > 0) {
+    pReq->taskAction = taosArrayInit(num, sizeof(STaskStatus));
+    if (NULL == pReq->taskAction) return -1;
+    for (int32_t i = 0; i < num; ++i) {
+      STaskAction action = {0};
+      if (tDecodeU64(&decoder, &action.queryId) < 0) return -1;
+      if (tDecodeU64(&decoder, &action.taskId) < 0) return -1;
+      if (tDecodeI8(&decoder, &action.action) < 0) return -1;
+      taosArrayPush(pReq->taskAction, &action);
+    }
+  } else {
+    pReq->taskAction = NULL;
+  }
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSSchedulerHbReq(SSchedulerHbReq *pReq) { taosArrayDestroy(pReq->taskAction); }
+
+
+
+int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeU64(&encoder, pRsp->seqId) < 0) return -1;    
+  if (tEncodeI32(&encoder, pRsp->epId.nodeId) < 0) return -1;    
+  if (tEncodeU16(&encoder, pRsp->epId.ep.port) < 0) return -1;
+  if (tEncodeCStr(&encoder, pRsp->epId.ep.fqdn) < 0) return -1;  
+  if (pRsp->taskStatus) {
+    int32_t num = taosArrayGetSize(pRsp->taskStatus);
+    if (tEncodeI32(&encoder, num) < 0) return -1;    
+    for (int32_t i = 0; i < num; ++i) {
+      STaskStatus *status = taosArrayGet(pRsp->taskStatus, i);
+      if (tEncodeU64(&encoder, status->queryId) < 0) return -1;    
+      if (tEncodeU64(&encoder, status->taskId) < 0) return -1;    
+      if (tEncodeI64(&encoder, status->refId) < 0) return -1;    
+      if (tEncodeI8(&encoder, status->status) < 0) return -1;    
+    }
+  } else {
+    if (tEncodeI32(&encoder, 0) < 0) return -1;    
+  }
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pRsp) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeU64(&decoder, &pRsp->seqId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->epId.nodeId) < 0) return -1;
+  if (tDecodeU16(&decoder, &pRsp->epId.ep.port) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pRsp->epId.ep.fqdn) < 0) return -1;
+  int32_t num = 0;
+  if (tDecodeI32(&decoder, &num) < 0) return -1;
+  if (num > 0) {
+    pRsp->taskStatus = taosArrayInit(num, sizeof(STaskStatus));
+    if (NULL == pRsp->taskStatus) return -1;
+    for (int32_t i = 0; i < num; ++i) {
+      STaskStatus status = {0};
+      if (tDecodeU64(&decoder, &status.queryId) < 0) return -1;
+      if (tDecodeU64(&decoder, &status.taskId) < 0) return -1;
+      if (tDecodeI64(&decoder, &status.refId) < 0) return -1;
+      if (tDecodeI8(&decoder, &status.status) < 0) return -1;
+      taosArrayPush(pRsp->taskStatus, &status);
+    }
+  } else {
+    pRsp->taskStatus = NULL;
+  }
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); }
+
 int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) {
   int32_t tlen = 0;
 
@@ -2423,3 +2655,43 @@ void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) {
 
   return buf;
 }
+
+int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStreamReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeCStr(&decoder, (const char **)&pReq->sql) < 0) return -1;
+  if (tDecodeCStr(&decoder, (const char **)&pReq->physicalPlan) < 0) return -1;
+  if (tDecodeCStr(&decoder, (const char **)&pReq->logicalPlan) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
+  tfree(pReq->sql);
+  tfree(pReq->physicalPlan);
+  tfree(pReq->logicalPlan);
+}

source/common/src/tname.c

@@ -61,7 +61,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
     }
     struct tm tm;
     time_t t = (time_t)start;
-    localtime_r(&t, &tm);
+    taosLocalTime(&t, &tm);
     tm.tm_sec = 0;
     tm.tm_min = 0;
     tm.tm_hour = 0;

source/common/src/ttime.c

@@ -69,7 +69,7 @@ static int64_t m_deltaUtc = 0;
 void           deltaToUtcInitOnce() {
   struct tm tm = {0};
 
-  (void)strptime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm);
+  (void)taosStrpTime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm);
   m_deltaUtc = (int64_t)mktime(&tm);
   // printf("====delta:%lld\n\n", seconds);
 }
@@ -236,9 +236,9 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
 
   char* str;
   if (delim == 'T') {
-    str = strptime(timestr, "%Y-%m-%dT%H:%M:%S", &tm);
+    str = taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm);
   } else if (delim == 0) {
-    str = strptime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+    str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
   } else {
     str = NULL;
   }
@@ -303,7 +303,7 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) {
   *time = 0;
   struct tm tm = {0};
 
-  char* str = strptime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+  char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
   if (str == NULL) {
     return -1;
   }
@@ -338,7 +338,7 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) {
   struct tm tm = {0};
   tm.tm_isdst = -1;
 
-  char* str = strptime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+  char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
   if (str == NULL) {
     return -1;
   }
@@ -466,7 +466,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
 
   struct tm tm;
   time_t    tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision));
-  localtime_r(&tt, &tm);
+  taosLocalTime(&tt, &tm);
   int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)duration;
   tm.tm_year = mon / 12;
   tm.tm_mon = mon % 12;
@@ -489,11 +489,11 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char
 
   struct tm tm;
   time_t    t = (time_t)skey;
-  localtime_r(&t, &tm);
+  taosLocalTime(&t, &tm);
   int32_t smon = tm.tm_year * 12 + tm.tm_mon;
 
   t = (time_t)ekey;
-  localtime_r(&t, &tm);
+  taosLocalTime(&t, &tm);
   int32_t emon = tm.tm_year * 12 + tm.tm_mon;
 
   if (unit == 'y') {
@@ -514,7 +514,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio
     start /= (int64_t)(TSDB_TICK_PER_SECOND(precision));
     struct tm tm;
     time_t    tt = (time_t)start;
-    localtime_r(&tt, &tm);
+    taosLocalTime(&tt, &tm);
     tm.tm_sec = 0;
     tm.tm_min = 0;
     tm.tm_hour = 0;
@@ -597,13 +597,13 @@ const char* fmtts(int64_t ts) {
 
   if (ts > -62135625943 && ts < 32503651200) {
     time_t t = (time_t)ts;
-    localtime_r(&t, &tm);
+    taosLocalTime(&t, &tm);
     pos += strftime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm);
   }
 
   if (ts > -62135625943000 && ts < 32503651200000) {
     time_t t = (time_t)(ts / 1000);
-    localtime_r(&t, &tm);
+    taosLocalTime(&t, &tm);
     if (pos > 0) {
       buf[pos++] = ' ';
       buf[pos++] = '|';
@@ -615,7 +615,7 @@ const char* fmtts(int64_t ts) {
 
   {
     time_t t = (time_t)(ts / 1000000);
-    localtime_r(&t, &tm);
+    taosLocalTime(&t, &tm);
     if (pos > 0) {
       buf[pos++] = ' ';
       buf[pos++] = '|';

source/common/src/tvariant.c

@@ -94,11 +94,11 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) {
       bool sign = true;
 
       int32_t base = 10;
-      if (type == TK_HEX) {
+      if (type == TK_NK_HEX) {
         base = 16;
-      } else if (type == TK_OCT) {
+      } else if (type == TK_NK_OCT) {
         base = 8;
-      } else if (type == TK_BIN) {
+      } else if (type == TK_NK_BIN) {
         base = 2;
       }
 

source/common/test/commonTests.cpp

@@ -160,8 +160,8 @@ TEST(testCase, Datablock_test) {
 
   printf("binary column length:%d\n", *(int32_t*) p1->pData);
 
-  ASSERT_EQ(colDataGetNumOfCols(b), 2);
-  ASSERT_EQ(colDataGetNumOfRows(b), 40);
+  ASSERT_EQ(blockDataGetNumOfCols(b), 2);
+  ASSERT_EQ(blockDataGetNumOfRows(b), 40);
 
   char* pData = colDataGetData(p1, 3);
   printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));

source/dnode/mgmt/impl/src/dndTransport.c

@@ -25,8 +25,8 @@
 #include "dndMnode.h"
 #include "dndVnodes.h"
 
-#define INTERNAL_USER   "_dnd"
-#define INTERNAL_CKEY   "_key"
+#define INTERNAL_USER "_dnd"
+#define INTERNAL_CKEY "_key"
 #define INTERNAL_SECRET "_pwd"
 
 static void dndInitMsgFp(STransMgmt *pMgmt) {
@@ -152,10 +152,11 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB)] = dndProcessVnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg;
+  pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_HEARTBEAT)] = dndProcessVnodeFetchMsg;
 }
 
 static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
-  SDnode     *pDnode = parent;
+  SDnode *    pDnode = parent;
   STransMgmt *pMgmt = &pDnode->tmgmt;
 
   tmsg_t msgType = pRsp->msgType;
@@ -193,7 +194,6 @@ static int32_t dndInitClient(SDnode *pDnode) {
   rpcInit.ckey = INTERNAL_CKEY;
   rpcInit.spi = 1;
   rpcInit.parent = pDnode;
-  rpcInit.noPool = true;
 
   char pass[TSDB_PASSWORD_LEN + 1] = {0};
   taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
@@ -219,7 +219,7 @@ static void dndCleanupClient(SDnode *pDnode) {
 }
 
 static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
-  SDnode     *pDnode = param;
+  SDnode *    pDnode = param;
   STransMgmt *pMgmt = &pDnode->tmgmt;
 
   tmsg_t msgType = pReq->msgType;
@@ -313,7 +313,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char
   SAuthReq authReq = {0};
   tstrncpy(authReq.user, user, TSDB_USER_LEN);
   int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
-  void   *pReq = rpcMallocCont(contLen);
+  void *  pReq = rpcMallocCont(contLen);
   tSerializeSAuthReq(pReq, contLen, &authReq);
 
   SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};

source/dnode/mnode/impl/inc/mndDef.h

@@ -84,6 +84,7 @@ typedef enum {
   TRN_TYPE_SUBSCRIBE = 1016,
   TRN_TYPE_REBALANCE = 1017,
   TRN_TYPE_COMMIT_OFFSET = 1018,
+  TRN_TYPE_CREATE_STREAM = 1019,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -353,6 +354,23 @@ typedef struct {
   char    payload[];
 } SShowObj;
 
+typedef struct {
+  int64_t id;
+  int8_t  type;
+  int8_t  replica;
+  int16_t numOfColumns;
+  int32_t rowSize;
+  int32_t numOfRows;
+  int32_t numOfReads;
+  int32_t payloadLen;
+  void*   pIter;
+  SMnode* pMnode;
+  char    db[TSDB_DB_FNAME_LEN];
+  int16_t offset[TSDB_MAX_COLUMNS];
+  int32_t bytes[TSDB_MAX_COLUMNS];
+  char    payload[];
+} SSysTableRetrieveObj;
+
 typedef struct {
   int32_t vgId;  // -1 for unassigned
   int32_t status;
@@ -662,7 +680,23 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
 }
 
 typedef struct {
-} SStreamScheduler;
+  char     name[TSDB_TOPIC_FNAME_LEN];
+  char     db[TSDB_DB_FNAME_LEN];
+  int64_t  createTime;
+  int64_t  updateTime;
+  int64_t  uid;
+  int64_t  dbUid;
+  int32_t  version;
+  SRWLatch lock;
+  int8_t   status;
+  // int32_t  sqlLen;
+  char* sql;
+  char* logicalPlan;
+  char* physicalPlan;
+} SStreamObj;
+
+int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj);
+int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj);
 
 typedef struct SMnodeMsg {
   char    user[TSDB_USER_LEN];

include/libs/parser/newParser.h → source/dnode/mnode/impl/inc/mndStream.h

@@ -13,31 +13,26 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_NEW_PARSER_H_
-#define _TD_NEW_PARSER_H_
+#ifndef _TD_MND_STREAM_H_
+#define _TD_MND_STREAM_H_
+
+#include "mndInt.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "parser.h"
-
-typedef enum EStmtType {
-  STMT_TYPE_CMD = 1,
-  STMT_TYPE_QUERY
-} EStmtType;
+int32_t mndInitStream(SMnode *pMnode);
+void    mndCleanupStream(SMnode *pMnode);
 
-typedef struct SQuery {
-  EStmtType stmtType;
-  SNode* pRoot;
-  int32_t numOfResCols;
-  SSchema* pResSchema;
-} SQuery;
+SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName);
+void        mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
 
-int32_t parser(SParseContext* pParseCxt, SQuery* pQuery);
+SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
+SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_NEW_PARSER_H_*/
+#endif /*_TD_MND_STREAM_H_*/

source/dnode/mnode/impl/src/mndDb.c

@@ -885,6 +885,29 @@ DROP_DB_OVER:
   return code;
 }
 
+void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) {
+  int32_t vindex = 0;
+  SSdb   *pSdb = pMnode->pSdb;
+
+  void *pIter = NULL;
+  while (vindex < pDb->cfg.numOfVgroups) {
+    SVgObj *pVgroup = NULL;
+    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
+    if (pIter == NULL) break;
+
+    if (pVgroup->dbUid == pDb->uid) {
+      *num += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT;
+
+      vindex++;
+    }
+
+    sdbRelease(pSdb, pVgroup);
+  }
+
+  sdbCancelFetch(pSdb, pIter);
+}
+
+
 static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
   int32_t vindex = 0;
   SSdb   *pSdb = pMnode->pSdb;
@@ -900,6 +923,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
       vgInfo.vgId = pVgroup->vgId;
       vgInfo.hashBegin = pVgroup->hashBegin;
       vgInfo.hashEnd = pVgroup->hashEnd;
+      vgInfo.numOfTable = pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT;
       vgInfo.epSet.numOfEps = pVgroup->replica;
       for (int32_t gid = 0; gid < pVgroup->replica; ++gid) {
         SVnodeGid *pVgid = &pVgroup->vnodeGid[gid];
@@ -967,7 +991,10 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) {
         goto USE_DB_OVER;
       }
 
-      if (usedbReq.vgVersion < pDb->vgVersion || usedbReq.dbId != pDb->uid) {
+      int32_t numOfTable = 0;
+      mndGetDBTableNum(pDb, pMnode, &numOfTable);
+
+      if (usedbReq.vgVersion < pDb->vgVersion || usedbReq.dbId != pDb->uid || numOfTable != usedbReq.numOfTable) {
         mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos);
       }
 
@@ -1017,6 +1044,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
     SDbVgVersion *pDbVgVersion = &pDbs[i];
     pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId);
     pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion);
+    pDbVgVersion->numOfTable = htonl(pDbVgVersion->numOfTable);
 
     SUseDbRsp usedbRsp = {0};
 
@@ -1027,28 +1055,34 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
       usedbRsp.uid = pDbVgVersion->dbId;
       usedbRsp.vgVersion = -1;
       taosArrayPush(batchUseRsp.pArray, &usedbRsp);
-    } else if (pDbVgVersion->vgVersion >= pDb->vgVersion) {
-      mDebug("db:%s, version not changed", pDbVgVersion->dbFName);
-      mndReleaseDb(pMnode, pDb);
       continue;
-    } else {
-      usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
-      if (usedbRsp.pVgroupInfos == NULL) {
-        mndReleaseDb(pMnode, pDb);
-        mError("db:%s, failed to malloc usedb response", pDb->name);
-        continue;
-      }
+    }
 
-      mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos);
-      memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN);
-      usedbRsp.uid = pDb->uid;
-      usedbRsp.vgVersion = pDb->vgVersion;
-      usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos);
-      usedbRsp.hashMethod = pDb->hashMethod;
+    int32_t numOfTable = 0;
+    mndGetDBTableNum(pDb, pMnode, &numOfTable);
 
-      taosArrayPush(batchUseRsp.pArray, &usedbRsp);
+    if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) {
+      mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName);
       mndReleaseDb(pMnode, pDb);
+      continue;
     }
+    
+    usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
+    if (usedbRsp.pVgroupInfos == NULL) {
+      mndReleaseDb(pMnode, pDb);
+      mError("db:%s, failed to malloc usedb response", pDb->name);
+      continue;
+    }
+
+    mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos);
+    memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN);
+    usedbRsp.uid = pDb->uid;
+    usedbRsp.vgVersion = pDb->vgVersion;
+    usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos);
+    usedbRsp.hashMethod = pDb->hashMethod;
+
+    taosArrayPush(batchUseRsp.pArray, &usedbRsp);
+    mndReleaseDb(pMnode, pDb);
   }
 
   int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp);

source/dnode/mnode/impl/src/mndDef.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mndDef.h"
+
+int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
+  if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
+  if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
+  return pEncoder->pos;
+}
+
+int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
+  if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pObj->db) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
+  if (tDecodeCStr(pDecoder, (const char **)&pObj->sql) < 0) return -1;
+  if (tDecodeCStr(pDecoder, (const char **)&pObj->logicalPlan) < 0) return -1;
+  if (tDecodeCStr(pDecoder, (const char **)&pObj->physicalPlan) < 0) return -1;
+  return 0;
+}

source/dnode/mnode/impl/src/mndQnode.c

@@ -36,6 +36,7 @@ static int32_t  mndProcessDropQnodeRsp(SMnodeMsg *pRsp);
 static int32_t  mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
 static int32_t  mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
 static void     mndCancelGetNextQnode(SMnode *pMnode, void *pIter);
+static int32_t  mndProcessQnodeListReq(SMnodeMsg *pReq);
 
 int32_t mndInitQnode(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_QNODE,
@@ -48,6 +49,7 @@ int32_t mndInitQnode(SMnode *pMnode) {
 
   mndSetMsgHandle(pMnode, TDMT_MND_CREATE_QNODE, mndProcessCreateQnodeReq);
   mndSetMsgHandle(pMnode, TDMT_MND_DROP_QNODE, mndProcessDropQnodeReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_QNODE_LIST, mndProcessQnodeListReq);
   mndSetMsgHandle(pMnode, TDMT_DND_CREATE_QNODE_RSP, mndProcessCreateQnodeRsp);
   mndSetMsgHandle(pMnode, TDMT_DND_DROP_QNODE_RSP, mndProcessDropQnodeRsp);
 
@@ -430,6 +432,69 @@ DROP_QNODE_OVER:
   return code;
 }
 
+
+static int32_t mndProcessQnodeListReq(SMnodeMsg *pReq) {
+  int32_t        code = -1;
+  SQnodeListReq  qlistReq = {0};
+  int32_t numOfRows = 0;
+  SMnode    *pMnode = pReq->pMnode;
+  SSdb      *pSdb = pMnode->pSdb;
+  SQnodeObj *pObj = NULL;
+  SQnodeListRsp qlistRsp = {0};
+
+  if (tDeserializeSQnodeListReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &qlistReq) != 0) {
+    mError("invalid qnode list msg");
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto QNODE_LIST_OVER;
+  }
+
+  qlistRsp.epSetList = taosArrayInit(5, sizeof(SEpSet));
+  if (NULL == qlistRsp.epSetList) {
+    mError("taosArrayInit epSet failed");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto QNODE_LIST_OVER;
+  }
+  
+  while (true) {
+    void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SEpSet epSet = {0};
+    strcpy(epSet.eps[0].fqdn, pObj->pDnode->fqdn);
+    epSet.eps[0].port = pObj->pDnode->port;
+    epSet.numOfEps = 1;
+
+    taosArrayPush(qlistRsp.epSetList, &epSet);
+
+    numOfRows++;
+    sdbRelease(pSdb, pObj);
+
+    if (qlistReq.rowNum > 0 && numOfRows >= qlistReq.rowNum) {
+      break;
+    }
+  }
+
+  int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
+  void   *pRsp = malloc(rspLen);
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto QNODE_LIST_OVER;
+  }
+  
+  tSerializeSQnodeListRsp(pRsp, rspLen, &qlistRsp);
+  
+  pReq->contLen = rspLen;
+  pReq->pCont = pRsp;
+  code = 0;
+
+QNODE_LIST_OVER:
+
+  tFreeSQnodeListRsp(&qlistRsp);
+  
+  return code;
+}
+
+
 static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp) {
   mndTransProcessRsp(pRsp);
   return 0;

source/dnode/mnode/impl/src/mndScheduler.c

@@ -32,30 +32,30 @@
 int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
   SSdb*      pSdb = pMnode->pSdb;
   SVgObj*    pVgroup = NULL;
-  SQueryDag* pDag = qStringToDag(pTopic->physicalPlan);
-  if (pDag == NULL) {
+  SQueryPlan* pPlan = qStringToQueryPlan(pTopic->physicalPlan);
+  if (pPlan == NULL) {
     terrno = TSDB_CODE_QRY_INVALID_INPUT;
     return -1;
   }
 
   ASSERT(pSub->vgNum == 0);
 
-  int32_t levelNum = taosArrayGetSize(pDag->pSubplans);
+  int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
   if (levelNum != 1) {
-    qDestroyQueryDag(pDag);
+    qDestroyQueryPlan(pPlan);
     terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
     return -1;
   }
 
-  SArray* plans = taosArrayGetP(pDag->pSubplans, 0);
+  SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0);
 
-  int32_t opNum = taosArrayGetSize(plans);
+  int32_t opNum = LIST_LENGTH(inner->pNodeList);
   if (opNum != 1) {
-    qDestroyQueryDag(pDag);
+    qDestroyQueryPlan(pPlan);
     terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
     return -1;
   }
-  SSubplan* plan = taosArrayGetP(plans, 0);
+  SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
 
   void* pIter = NULL;
   while (1) {
@@ -78,14 +78,14 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     int32_t msgLen;
     if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
       sdbRelease(pSdb, pVgroup);
-      qDestroyQueryDag(pDag);
+      qDestroyQueryPlan(pPlan);
       terrno = TSDB_CODE_QRY_INVALID_INPUT;
       return -1;
     }
     taosArrayPush(pSub->unassignedVg, &consumerEp);
   }
 
-  qDestroyQueryDag(pDag);
+  qDestroyQueryPlan(pPlan);
 
   return 0;
 }

source/dnode/mnode/impl/src/mndShow.c

@@ -25,6 +25,7 @@ static void      mndReleaseShowObj(SShowObj *pShow, bool forceRemove);
 static int32_t   mndProcessShowReq(SMnodeMsg *pReq);
 static int32_t   mndProcessRetrieveReq(SMnodeMsg *pReq);
 static bool      mndCheckRetrieveFinished(SShowObj *pShow);
+static int32_t   mndProcessRetrieveSysTableReq(SMnodeMsg *pReq);
 
 int32_t mndInitShow(SMnode *pMnode) {
   SShowMgmt *pMgmt = &pMnode->showMgmt;
@@ -38,6 +39,7 @@ int32_t mndInitShow(SMnode *pMnode) {
 
   mndSetMsgHandle(pMnode, TDMT_MND_SHOW, mndProcessShowReq);
   mndSetMsgHandle(pMnode, TDMT_MND_SHOW_RETRIEVE, mndProcessRetrieveReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq);
   return 0;
 }
 
@@ -261,6 +263,106 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) {
+  SMnode    *pMnode = pReq->pMnode;
+  SShowMgmt *pMgmt = &pMnode->showMgmt;
+  int32_t    rowsToRead = 0;
+  int32_t    size = 0;
+  int32_t    rowsRead = 0;
+
+  SRetrieveTableReq retrieveReq = {0};
+  if (tDeserializeSRetrieveTableReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &retrieveReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  SShowObj* pShow = NULL;
+
+  if (retrieveReq.showId == 0) {
+    SShowReq req = {0};
+    req.type = retrieveReq.type;
+    strncpy(req.db, retrieveReq.db, tListLen(req.db));
+
+    pShow = mndCreateShowObj(pMnode, &req);
+    if (pShow == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      mError("failed to process show-meta req since %s", terrstr());
+      return -1;
+    }
+  } else {
+    pShow = mndAcquireShowObj(pMnode, retrieveReq.showId);
+    if (pShow == NULL) {
+      terrno = TSDB_CODE_MND_INVALID_SHOWOBJ;
+      mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
+      return -1;
+    }
+  }
+
+  ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
+  if (retrieveFp == NULL) {
+    mndReleaseShowObj((SShowObj*) pShow, false);
+    terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+    mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr());
+    return -1;
+  }
+
+  mDebug("show:0x%" PRIx64 ", start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow->numOfReads,
+         pShow->numOfRows, mndShowStr(pShow->type));
+
+  if (mndCheckRetrieveFinished((SShowObj*) pShow)) {
+    mDebug("show:0x%" PRIx64 ", read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads,
+           pShow->numOfRows);
+    pShow->numOfReads = pShow->numOfRows;
+  }
+
+  if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
+    rowsToRead = pShow->numOfRows - pShow->numOfReads;
+  }
+
+  /* return no more than 100 tables in one round trip */
+  if (rowsToRead > SHOW_STEP_SIZE) rowsToRead = SHOW_STEP_SIZE;
+
+  /*
+   * the actual number of table may be larger than the value of pShow->numOfRows, if a query is
+   * issued during a continuous create table operation. Therefore, rowToRead may be less than 0.
+   */
+  if (rowsToRead < 0) rowsToRead = 0;
+  size = pShow->rowSize * rowsToRead;
+
+  size += SHOW_STEP_SIZE;
+  SRetrieveTableRsp *pRsp = rpcMallocCont(size);
+  if (pRsp == NULL) {
+    mndReleaseShowObj((SShowObj*) pShow, false);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr());
+    return -1;
+  }
+
+  // if free flag is set, client wants to clean the resources
+  if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
+    rowsRead = (*retrieveFp)(pReq, (SShowObj*) pShow, pRsp->data, rowsToRead);
+  }
+
+  mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead);
+
+  pRsp->numOfRows = htonl(rowsRead);
+  pRsp->precision = TSDB_TIME_PRECISION_MILLI;  // millisecond time precision
+
+  pReq->pCont = pRsp;
+  pReq->contLen = size;
+
+  if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) {
+    pRsp->completed = 1;
+    mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id);
+    mndReleaseShowObj((SShowObj*) pShow, true);
+  } else {
+    mDebug("show:0x%" PRIx64 ", retrieve not completed yet", pShow->id);
+    mndReleaseShowObj((SShowObj*) pShow, false);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 char *mndShowStr(int32_t showType) {
   switch (showType) {
     case TSDB_MGMT_TABLE_ACCT:

source/dnode/mnode/impl/src/mndStream.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mndStream.h"
+#include "mndAuth.h"
+#include "mndDb.h"
+#include "mndDnode.h"
+#include "mndMnode.h"
+#include "mndShow.h"
+#include "mndStb.h"
+#include "mndTrans.h"
+#include "mndUser.h"
+#include "mndVgroup.h"
+#include "tname.h"
+
+#define MND_STREAM_VER_NUMBER   1
+#define MND_STREAM_RESERVE_SIZE 64
+
+static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
+static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
+static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
+static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq);
+/*static int32_t mndProcessDropStreamReq(SMnodeMsg *pReq);*/
+/*static int32_t mndProcessDropStreamInRsp(SMnodeMsg *pRsp);*/
+static int32_t mndProcessStreamMetaReq(SMnodeMsg *pReq);
+static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
+static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
+static void    mndCancelGetNextStream(SMnode *pMnode, void *pIter);
+
+int32_t mndInitStream(SMnode *pMnode) {
+  SSdbTable table = {.sdbType = SDB_STREAM,
+                     .keyType = SDB_KEY_BINARY,
+                     .encodeFp = (SdbEncodeFp)mndStreamActionEncode,
+                     .decodeFp = (SdbDecodeFp)mndStreamActionDecode,
+                     .insertFp = (SdbInsertFp)mndStreamActionInsert,
+                     .updateFp = (SdbUpdateFp)mndStreamActionUpdate,
+                     .deleteFp = (SdbDeleteFp)mndStreamActionDelete};
+
+  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq);
+  /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/
+  /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
+
+  mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TP, mndGetStreamMeta);
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TP, mndRetrieveStream);
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TP, mndCancelGetNextStream);
+
+  return sdbSetTable(pMnode->pSdb, table);
+}
+
+void mndCleanupStream(SMnode *pMnode) {}
+
+SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  void *buf = NULL;
+
+  SCoder encoder;
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
+  if (tEncodeSStreamObj(NULL, pStream) < 0) {
+    tCoderClear(&encoder);
+    goto STREAM_ENCODE_OVER;
+  }
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+
+  int32_t  size = sizeof(int32_t) + tlen + MND_STREAM_RESERVE_SIZE;
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_STREAM, MND_STREAM_VER_NUMBER, size);
+  if (pRaw == NULL) goto STREAM_ENCODE_OVER;
+
+  buf = malloc(tlen);
+  if (buf == NULL) goto STREAM_ENCODE_OVER;
+
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER);
+  if (tEncodeSStreamObj(NULL, pStream) < 0) {
+    tCoderClear(&encoder);
+    goto STREAM_ENCODE_OVER;
+  }
+  tCoderClear(&encoder);
+
+  int32_t dataPos = 0;
+  SDB_SET_INT32(pRaw, dataPos, tlen, STREAM_ENCODE_OVER);
+  SDB_SET_BINARY(pRaw, dataPos, buf, tlen, STREAM_ENCODE_OVER);
+  SDB_SET_DATALEN(pRaw, dataPos, STREAM_ENCODE_OVER);
+
+  terrno = TSDB_CODE_SUCCESS;
+
+STREAM_ENCODE_OVER:
+  tfree(buf);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr());
+    sdbFreeRaw(pRaw);
+    return NULL;
+  }
+
+  mTrace("stream:%s, encode to raw:%p, row:%p", pStream->name, pRaw, pStream);
+  return pRaw;
+}
+
+SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  void *buf = NULL;
+
+  int8_t sver = 0;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER;
+
+  if (sver != MND_STREAM_VER_NUMBER) {
+    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
+    goto STREAM_DECODE_OVER;
+  }
+
+  int32_t  size = sizeof(SStreamObj);
+  SSdbRow *pRow = sdbAllocRow(size);
+  if (pRow == NULL) goto STREAM_DECODE_OVER;
+
+  SStreamObj *pStream = sdbGetRowObj(pRow);
+  if (pStream == NULL) goto STREAM_DECODE_OVER;
+
+  int32_t tlen;
+  int32_t dataPos = 0;
+  SDB_GET_INT32(pRaw, dataPos, &tlen, STREAM_DECODE_OVER);
+  buf = malloc(tlen + 1);
+  if (buf == NULL) goto STREAM_DECODE_OVER;
+  SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER);
+
+  SCoder decoder;
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, NULL, 0, TD_DECODER);
+  if (tDecodeSStreamObj(&decoder, pStream) < 0) {
+    goto STREAM_DECODE_OVER;
+  }
+
+  terrno = TSDB_CODE_SUCCESS;
+
+STREAM_DECODE_OVER:
+  tfree(buf);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr());
+    tfree(pRow);
+    return NULL;
+  }
+
+  mTrace("stream:%s, decode from raw:%p, row:%p", pStream->name, pRaw, pStream);
+  return pRow;
+}
+
+static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream) {
+  mTrace("stream:%s, perform insert action", pStream->name);
+  return 0;
+}
+
+static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) {
+  mTrace("stream:%s, perform delete action", pStream->name);
+  return 0;
+}
+
+static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) {
+  mTrace("stream:%s, perform update action", pOldStream->name);
+  atomic_exchange_32(&pOldStream->updateTime, pNewStream->updateTime);
+  atomic_exchange_32(&pOldStream->version, pNewStream->version);
+
+  taosWLockLatch(&pOldStream->lock);
+
+  // TODO handle update
+
+  taosWUnLockLatch(&pOldStream->lock);
+  return 0;
+}
+
+SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) {
+  SSdb       *pSdb = pMnode->pSdb;
+  SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName);
+  if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
+    terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
+  }
+  return pStream;
+}
+
+void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbRelease(pSdb, pStream);
+}
+
+static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) {
+  SName name = {0};
+  tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+
+  char db[TSDB_STREAM_FNAME_LEN] = {0};
+  tNameGetFullDbName(&name, db);
+
+  return mndAcquireDb(pMnode, db);
+}
+
+static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) {
+  if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) {
+    terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION;
+    return -1;
+  }
+  return 0;
+}
+
+static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) {
+  mDebug("stream:%s to create", pCreate->name);
+  SStreamObj streamObj = {0};
+  tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
+  tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN);
+  streamObj.createTime = taosGetTimestampMs();
+  streamObj.updateTime = streamObj.createTime;
+  streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
+  streamObj.dbUid = pDb->uid;
+  streamObj.version = 1;
+  streamObj.sql = pCreate->sql;
+  streamObj.physicalPlan = pCreate->physicalPlan;
+  streamObj.logicalPlan = pCreate->logicalPlan;
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
+  if (pTrans == NULL) {
+    mError("stream:%s, failed to create since %s", pCreate->name, terrstr());
+    return -1;
+  }
+  mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
+
+  SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj);
+  if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+    mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
+
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  mndTransDrop(pTrans);
+  return 0;
+}
+
+static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq) {
+  SMnode            *pMnode = pReq->pMnode;
+  int32_t            code = -1;
+  SStreamObj        *pStream = NULL;
+  SDbObj            *pDb = NULL;
+  SUserObj          *pUser = NULL;
+  SCMCreateStreamReq createStreamReq = {0};
+
+  if (tDeserializeSCMCreateStreamReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createStreamReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto CREATE_STREAM_OVER;
+  }
+
+  mDebug("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql);
+
+  if (mndCheckCreateStreamReq(&createStreamReq) != 0) {
+    mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
+    goto CREATE_STREAM_OVER;
+  }
+
+  pStream = mndAcquireStream(pMnode, createStreamReq.name);
+  if (pStream != NULL) {
+    if (createStreamReq.igExists) {
+      mDebug("stream:%s, already exist, ignore exist is set", createStreamReq.name);
+      code = 0;
+      goto CREATE_STREAM_OVER;
+    } else {
+      terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
+      goto CREATE_STREAM_OVER;
+    }
+  } else if (terrno != TSDB_CODE_MND_STREAM_NOT_EXIST) {
+    goto CREATE_STREAM_OVER;
+  }
+
+  pDb = mndAcquireDbByStream(pMnode, createStreamReq.name);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    goto CREATE_STREAM_OVER;
+  }
+
+  pUser = mndAcquireUser(pMnode, pReq->user);
+  if (pUser == NULL) {
+    goto CREATE_STREAM_OVER;
+  }
+
+  if (mndCheckWriteAuth(pUser, pDb) != 0) {
+    goto CREATE_STREAM_OVER;
+  }
+
+  code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb);
+  if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
+
+CREATE_STREAM_OVER:
+  if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+    mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
+  }
+
+  mndReleaseStream(pMnode, pStream);
+  mndReleaseDb(pMnode, pDb);
+  mndReleaseUser(pMnode, pUser);
+
+  tFreeSCMCreateStreamReq(&createStreamReq);
+  return code;
+}
+
+static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) {
+  SSdb   *pSdb = pMnode->pSdb;
+  SDbObj *pDb = mndAcquireDb(pMnode, dbName);
+  if (pDb == NULL) {
+    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
+    return -1;
+  }
+
+  int32_t numOfStreams = 0;
+  void   *pIter = NULL;
+  while (1) {
+    SStreamObj *pStream = NULL;
+    pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
+    if (pIter == NULL) break;
+
+    if (pStream->dbUid == pDb->uid) {
+      numOfStreams++;
+    }
+
+    sdbRelease(pSdb, pStream);
+  }
+
+  *pNumOfStreams = numOfStreams;
+  mndReleaseDb(pMnode, pDb);
+  return 0;
+}
+
+static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {
+  SMnode *pMnode = pReq->pMnode;
+  SSdb   *pSdb = pMnode->pSdb;
+
+  if (mndGetNumOfStreams(pMnode, pShow->db, &pShow->numOfRows) != 0) {
+    return -1;
+  }
+
+  int32_t  cols = 0;
+  SSchema *pSchema = pMeta->pSchemas;
+
+  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "name");
+  pSchema[cols].bytes = pShow->bytes[cols];
+  cols++;
+
+  pShow->bytes[cols] = 8;
+  pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
+  strcpy(pSchema[cols].name, "create_time");
+  pSchema[cols].bytes = pShow->bytes[cols];
+  cols++;
+
+  pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "sql");
+  pSchema[cols].bytes = pShow->bytes[cols];
+  cols++;
+
+  pMeta->numOfColumns = cols;
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = sdbGetSize(pSdb, SDB_STREAM);
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  strcpy(pMeta->tbName, mndShowStr(pShow->type));
+
+  return 0;
+}
+
+static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
+  SMnode     *pMnode = pReq->pMnode;
+  SSdb       *pSdb = pMnode->pSdb;
+  int32_t     numOfRows = 0;
+  SStreamObj *pStream = NULL;
+  int32_t     cols = 0;
+  char       *pWrite;
+  char        prefix[TSDB_DB_FNAME_LEN] = {0};
+
+  SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
+  if (pDb == NULL) return 0;
+
+  tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
+  strcat(prefix, TS_PATH_DELIMITER);
+  int32_t prefixLen = (int32_t)strlen(prefix);
+
+  while (numOfRows < rows) {
+    pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
+    if (pShow->pIter == NULL) break;
+
+    if (pStream->dbUid != pDb->uid) {
+      if (strncmp(pStream->name, prefix, prefixLen) != 0) {
+        mError("Inconsistent stream data, name:%s, db:%s, dbUid:%" PRIu64, pStream->name, pDb->name, pDb->uid);
+      }
+
+      sdbRelease(pSdb, pStream);
+      continue;
+    }
+
+    cols = 0;
+
+    char streamName[TSDB_TABLE_NAME_LEN] = {0};
+    tstrncpy(streamName, pStream->name + prefixLen, TSDB_TABLE_NAME_LEN);
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, streamName);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int64_t *)pWrite = pStream->createTime;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pStream->sql, pShow->bytes[cols]);
+    cols++;
+
+    numOfRows++;
+    sdbRelease(pSdb, pStream);
+  }
+
+  mndReleaseDb(pMnode, pDb);
+  pShow->numOfReads += numOfRows;
+  mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
+  return numOfRows;
+}
+
+static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -773,10 +773,10 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
 static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
   SSdb      *pSdb = pMnode->pSdb;
   SVgObj    *pVgroup = NULL;
-  SQueryDag *pDag = qStringToDag(pTopic->physicalPlan);
+  SQueryPlan *pPlan = qStringToQueryPlan(pTopic->physicalPlan);
   SArray    *pArray = NULL;
-  SArray    *inner = taosArrayGet(pDag->pSubplans, 0);
-  SSubplan  *plan = taosArrayGetP(inner, 0);
+  SNodeListNode    *inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
+  SSubplan  *plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
   SArray    *unassignedVg = pSub->unassignedVg;
 
   void *pIter = NULL;
@@ -792,7 +792,7 @@ static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SM
     plan->execNode.nodeId = pVgroup->vgId;
     plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup);
 
-    if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) {
+    if (schedulerConvertDagToTaskList(pPlan, &pArray) < 0) {
       terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
       mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql);
       return -1;

source/dnode/mnode/impl/src/mndTopic.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
 #include "mndTopic.h"
 #include "mndAuth.h"
 #include "mndDb.h"
@@ -229,7 +228,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
   return pDrop;
 }
 
-static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) {
+static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
   if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
     return -1;
@@ -237,7 +236,7 @@ static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) {
   return 0;
 }
 
-static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq *pCreate, SDbObj *pDb) {
+static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
   mDebug("topic:%s to create", pCreate->name);
   SMqTopicObj topicObj = {0};
   tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
@@ -278,14 +277,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq
 }
 
 static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) {
-  SMnode          *pMnode = pReq->pMnode;
-  int32_t          code = -1;
-  SMqTopicObj     *pTopic = NULL;
-  SDbObj          *pDb = NULL;
-  SUserObj        *pUser = NULL;
-  SMCreateTopicReq createTopicReq = {0};
-
-  if (tDeserializeSMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) {
+  SMnode           *pMnode = pReq->pMnode;
+  int32_t           code = -1;
+  SMqTopicObj      *pTopic = NULL;
+  SDbObj           *pDb = NULL;
+  SUserObj         *pUser = NULL;
+  SCMCreateTopicReq createTopicReq = {0};
+
+  if (tDeserializeSCMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     goto CREATE_TOPIC_OVER;
   }
@@ -338,7 +337,7 @@ CREATE_TOPIC_OVER:
   mndReleaseDb(pMnode, pDb);
   mndReleaseUser(pMnode, pUser);
 
-  tFreeSMCreateTopicReq(&createTopicReq);
+  tFreeSCMCreateTopicReq(&createTopicReq);
   return code;
 }
 
@@ -409,35 +408,6 @@ static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) {
   return 0;
 }
 
-#if 0
-static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
-  SSdb   *pSdb = pMnode->pSdb;
-  SDbObj *pDb = mndAcquireDb(pMnode, dbName);
-  if (pDb == NULL) {
-    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    return -1;
-  }
-
-  int32_t numOfTopics = 0;
-  void   *pIter = NULL;
-  while (1) {
-    SMqTopicObj *pTopic = NULL;
-    pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
-    if (pIter == NULL) break;
-
-    if (pTopic->dbUid == pDb->uid) {
-      numOfTopics++;
-    }
-
-    sdbRelease(pSdb, pTopic);
-  }
-
-  *pNumOfTopics = numOfTopics;
-  mndReleaseDb(pMnode, pDb);
-  return 0;
-}
-#endif
-
 static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
   SSdb   *pSdb = pMnode->pSdb;
   SDbObj *pDb = mndAcquireDb(pMnode, dbName);

source/dnode/qnode/CMakeLists.txt

@@ -11,4 +11,7 @@ target_link_libraries(
     PRIVATE os
     PRIVATE common
     PRIVATE util
+    PRIVATE qworker
+    PRIVATE qcom
+    PRIVATE executor
 )
\ No newline at end of file

source/dnode/qnode/inc/qndInt.h

@@ -28,12 +28,16 @@
 extern "C" {
 #endif
 
+typedef struct SQWorkerMgmt SQHandle;
+
 typedef struct SQnode {
+  int32_t   qndId;
   SQnodeOpt opt;
+  SQHandle*  pQuery;  
 } SQnode;
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_QNODE_INT_H_*/
+#endif /*_TD_QNODE_INT_H_*/
\ No newline at end of file