Merge remote-tracking branch 'origin/3.0' into catalog_dev

.devcontainer/Dockerfile

+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile
+
+# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
+ARG VARIANT="bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
+
+# [Optional] Uncomment this section to install additional packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>

.devcontainer/devcontainer.json

+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp
+{
+	"name": "C++",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
+		// Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon
+		"args": { "VARIANT": "ubuntu-21.04" }
+	},
+	"runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-vscode.cpptools",
+		"ms-vscode.cmake-tools",
+		"austin.code-gnu-global",
+		"visualstudioexptteam.vscodeintel"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "gcc -v",
+
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode"
+}

include/client/taos.h

@@ -39,7 +39,7 @@ typedef void **TAOS_ROW;
 #define TSDB_DATA_TYPE_BIGINT     5     // 8 bytes
 #define TSDB_DATA_TYPE_FLOAT      6     // 4 bytes
 #define TSDB_DATA_TYPE_DOUBLE     7     // 8 bytes
-#define TSDB_DATA_TYPE_BINARY     8     // string
+#define TSDB_DATA_TYPE_BINARY     8     // string, alias for varchar
 #define TSDB_DATA_TYPE_TIMESTAMP  9     // 8 bytes
 #define TSDB_DATA_TYPE_NCHAR      10    // unicode string
 #define TSDB_DATA_TYPE_UTINYINT   11    // 1 byte
@@ -47,10 +47,10 @@ typedef void **TAOS_ROW;
 #define TSDB_DATA_TYPE_UINT       13    // 4 bytes
 #define TSDB_DATA_TYPE_UBIGINT    14    // 8 bytes
 #define TSDB_DATA_TYPE_VARCHAR    15    // string
-#define TSDB_DATA_TYPE_JSON       16    // json
-#define TSDB_DATA_TYPE_DECIMAL    17    // decimal
-#define TSDB_DATA_TYPE_BLOB       18    // binary string
-#define TSDB_DATA_TYPE_LONGBLOB   19    // long binary string
+#define TSDB_DATA_TYPE_VARBINARY  16    // binary
+#define TSDB_DATA_TYPE_JSON       17    // json
+#define TSDB_DATA_TYPE_DECIMAL    18    // decimal
+#define TSDB_DATA_TYPE_BLOB       19    // binary
 
 typedef enum {
   TSDB_OPTION_LOCALE,
@@ -61,6 +61,23 @@ typedef enum {
   TSDB_MAX_OPTIONS
 } TSDB_OPTION;
 
+typedef enum {
+  TSDB_SML_UNKNOWN_PROTOCOL = 0,
+  TSDB_SML_LINE_PROTOCOL    = 1,
+  TSDB_SML_TELNET_PROTOCOL  = 2,
+  TSDB_SML_JSON_PROTOCOL    = 3,
+} TSDB_SML_PROTOCOL_TYPE;
+
+typedef enum {
+  TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0,
+  TSDB_SML_TIMESTAMP_HOURS,
+  TSDB_SML_TIMESTAMP_MINUTES,
+  TSDB_SML_TIMESTAMP_SECONDS,
+  TSDB_SML_TIMESTAMP_MILLI_SECONDS,
+  TSDB_SML_TIMESTAMP_MICRO_SECONDS,
+  TSDB_SML_TIMESTAMP_NANO_SECONDS,
+} TSDB_SML_TIMESTAMP_TYPE;
+
 typedef struct taosField {
   char     name[65];
   uint8_t  type;
@@ -136,12 +153,15 @@ DLL_EXPORT int        taos_stmt_close(TAOS_STMT *stmt);
 DLL_EXPORT char *     taos_stmt_errstr(TAOS_STMT *stmt);
 
 DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
+DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
+
 DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
 DLL_EXPORT int  taos_result_precision(TAOS_RES *res);  // get the time precision of result
 DLL_EXPORT void taos_free_result(TAOS_RES *res);
 DLL_EXPORT int  taos_field_count(TAOS_RES *res);
 DLL_EXPORT int  taos_num_fields(TAOS_RES *res);
 DLL_EXPORT int  taos_affected_rows(TAOS_RES *res);
+
 DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
 DLL_EXPORT int  taos_select_db(TAOS *taos, const char *db);
 DLL_EXPORT int  taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
@@ -152,21 +172,17 @@ DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
 
 DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res);
 
-// TAOS_RES   *taos_list_tables(TAOS *mysql, const char *wild);
-// TAOS_RES   *taos_list_dbs(TAOS *mysql, const char *wild);
-
-// TODO: the return value should be `const`
 DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
 DLL_EXPORT const char *taos_get_client_info();
-DLL_EXPORT const char *taos_errstr(TAOS_RES *tres);
 
+DLL_EXPORT const char *taos_errstr(TAOS_RES *tres);
 DLL_EXPORT int   taos_errno(TAOS_RES *tres);
 
 DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
 DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
 
-typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
-DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval);
+typedef void (*__taos_sub_fn_t)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
+DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, __taos_sub_fn_t fp, void *param, int interval);
 DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
 
@@ -175,8 +191,7 @@ DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)
 DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
 
 DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList);
-
-DLL_EXPORT int taos_insert_lines(TAOS* taos, char* lines[], int numLines);
+DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision);
 
 #ifdef __cplusplus
 }

include/common/taosmsg.h

@@ -406,8 +406,10 @@ typedef struct {
 } SDropUserMsg, SDropAcctMsg;
 
 typedef struct {
+  int8_t  type;
   char    user[TSDB_USER_LEN];
   char    pass[TSDB_PASSWORD_LEN];
+  int8_t  superUser;      // denote if it is a super user or not
   int32_t reserve[8];
 } SCreateUserMsg, SAlterUserMsg;
 
@@ -831,7 +833,7 @@ typedef struct {
 } SVgroupsMsg, SVgroupsInfo;
 
 typedef struct {
-  char       tbFname[TSDB_TABLE_FNAME_LEN];  // table id
+  char       tbFname[TSDB_TABLE_FNAME_LEN];  // table full name
   char       stbFname[TSDB_TABLE_FNAME_LEN];
   int32_t    numOfTags;
   int32_t    numOfColumns;

include/dnode/vnode/tq/tq.h

@@ -16,9 +16,9 @@
 #ifndef _TD_TQ_H_
 #define _TD_TQ_H_
 
+#include "mallocator.h"
 #include "os.h"
 #include "tutil.h"
-#include "mallocator.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -97,102 +97,102 @@ typedef struct TmqHeartbeatReq {
 typedef struct TmqHeartbeatRsp {
 } TmqHeartbeatRsp;
 
-typedef struct TqTopicVhandle {
+typedef struct STqTopicVhandle {
   int64_t topicId;
   // executor for filter
   void* filterExec;
   // callback for mnode
   // trigger when vnode list associated topic change
   void* (*mCallback)(void*, void*);
-} TqTopicVhandle;
-
+} STqTopicVhandle;
 
 #define TQ_BUFFER_SIZE 8
 
-typedef struct TqBufferItem {
+typedef struct STqBufferItem {
   int64_t offset;
   // executors are identical but not concurrent
   // so there must be a copy in each item
   void*   executor;
   int64_t size;
   void*   content;
-} TqBufferItem;
+} STqBufferItem;
 
-typedef struct TqBufferHandle {
+typedef struct STqBufferHandle {
   // char* topic; //c style, end with '\0'
   // int64_t cgId;
   // void* ahandle;
   int64_t       nextConsumeOffset;
+  int64_t       floatingCursor;
   int64_t       topicId;
   int32_t       head;
   int32_t       tail;
-  TqBufferItem buffer[TQ_BUFFER_SIZE];
-} TqBufferHandle;
+  STqBufferItem buffer[TQ_BUFFER_SIZE];
+} STqBufferHandle;
 
-typedef struct TqListHandle {
-  TqBufferHandle       bufHandle;
-  struct TqListHandle* next;
-} TqListHandle;
+typedef struct STqListHandle {
+  STqBufferHandle       bufHandle;
+  struct STqListHandle* next;
+} STqListHandle;
 
-typedef struct TqGroupHandle {
+typedef struct STqGroupHandle {
   int64_t        cId;
   int64_t        cgId;
   void*          ahandle;
   int32_t        topicNum;
-  TqListHandle* head;
-} TqGroupHandle;
+  STqListHandle* head;
+} STqGroupHandle;
 
-typedef struct TqQueryExec {
+typedef struct STqQueryExec {
   void*          src;
-  TqBufferItem* dest;
+  STqBufferItem* dest;
   void*          executor;
-} TqQueryExec;
+} STqQueryExec;
 
-typedef struct TqQueryMsg {
-  TqQueryExec*       exec;
-  struct TqQueryMsg* next;
-} TqQueryMsg;
+typedef struct STqQueryMsg {
+  STqQueryExec*       exec;
+  struct STqQueryMsg* next;
+} STqQueryMsg;
 
-typedef struct TqLogReader {
+typedef struct STqLogReader {
   void* logHandle;
   int32_t (*logRead)(void* logHandle, void** data, int64_t ver);
   int64_t (*logGetFirstVer)(void* logHandle);
   int64_t (*logGetSnapshotVer)(void* logHandle);
   int64_t (*logGetLastVer)(void* logHandle);
-} TqLogReader;
+} STqLogReader;
 
 typedef struct STqCfg {
   // TODO
 } STqCfg;
 
-typedef struct TqMemRef {
-  SMemAllocatorFactory *pAlloctorFactory;
-  SMemAllocator *pAllocator;
-} TqMemRef;
+typedef struct STqMemRef {
+  SMemAllocatorFactory* pAlloctorFactory;
+  SMemAllocator*        pAllocator;
+} STqMemRef;
 
-typedef struct TqSerializedHead {
+typedef struct STqSerializedHead {
   int16_t ver;
   int16_t action;
   int32_t checksum;
   int64_t ssize;
   char    content[];
-} TqSerializedHead;
+} STqSerializedHead;
 
-typedef int (*TqSerializeFun)(const void* pObj, TqSerializedHead** ppHead);
-typedef const void* (*TqDeserializeFun)(const TqSerializedHead* pHead, void** ppObj);
-typedef void (*TqDeleteFun)(void*);
+typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead);
+typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj);
+typedef void (*FTqDelete)(void*);
 
 #define TQ_BUCKET_MASK 0xFF
 #define TQ_BUCKET_SIZE 256
 
 #define TQ_PAGE_SIZE 4096
-//key + offset + size
+// key + offset + size
 #define TQ_IDX_SIZE 24
-//4096 / 24
+// 4096 / 24
 #define TQ_MAX_IDX_ONE_PAGE 170
-//24 * 170
+// 24 * 170
 #define TQ_IDX_PAGE_BODY_SIZE 4080
-//4096 - 4080
+// 4096 - 4080
 #define TQ_IDX_PAGE_HEAD_SIZE 16
 
 #define TQ_ACTION_CONST 0
@@ -202,22 +202,19 @@ typedef void (*TqDeleteFun)(void*);
 
 #define TQ_SVER 0
 
-//TODO: inplace mode is not implemented
+// TODO: inplace mode is not implemented
 #define TQ_UPDATE_INPLACE 0
 #define TQ_UPDATE_APPEND 1
 
 #define TQ_DUP_INTXN_REWRITE 0
 #define TQ_DUP_INTXN_REJECT 2
 
-static inline bool TqUpdateAppend(int32_t tqConfigFlag) {
-  return tqConfigFlag & TQ_UPDATE_APPEND;
-}
+static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
 
-static inline bool TqDupIntxnReject(int32_t tqConfigFlag) {
-  return tqConfigFlag & TQ_DUP_INTXN_REJECT;
-}
+static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
 
 static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
+
 #define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
 
 typedef struct TqMetaHandle {
@@ -231,25 +228,25 @@ typedef struct TqMetaHandle {
 typedef struct TqMetaList {
   STqMetaHandle      handle;
   struct TqMetaList* next;
-  //struct TqMetaList* inTxnPrev;
-  //struct TqMetaList* inTxnNext;
+  // struct TqMetaList* inTxnPrev;
+  // struct TqMetaList* inTxnNext;
   struct TqMetaList* unpersistPrev;
   struct TqMetaList* unpersistNext;
 } STqMetaList;
 
 typedef struct TqMetaStore {
   STqMetaList* bucket[TQ_BUCKET_SIZE];
-  //a table head
+  // a table head
   STqMetaList* unpersistHead;
- //TODO:temporaral use, to be replaced by unified tfile
+  // TODO:temporaral use, to be replaced by unified tfile
   int fileFd;
-  //TODO:temporaral use, to be replaced by unified tfile
+  // TODO:temporaral use, to be replaced by unified tfile
   int            idxFd;
   char*          dirPath;
   int32_t        tqConfigFlag;
-  TqSerializeFun   pSerializer;
-  TqDeserializeFun pDeserializer;
-  TqDeleteFun      pDeleter;
+  FTqSerialize   pSerializer;
+  FTqDeserialize pDeserializer;
+  FTqDelete      pDeleter;
 } STqMetaStore;
 
 typedef struct STQ {
@@ -257,13 +254,13 @@ typedef struct STQ {
   // the handle of kvstore
   char*         path;
   STqCfg*       tqConfig;
-  TqLogReader* tqLogReader; 
-  TqMemRef     tqMemRef;
+  STqLogReader* tqLogReader;
+  STqMemRef     tqMemRef;
   STqMetaStore* tqMeta;
 } STQ;
 
 // open in each vnode
-STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac);
+STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac);
 void tqDestroy(STQ*);
 
 // void* will be replace by a msg type
@@ -272,19 +269,19 @@ int tqCommit(STQ*);
 
 int tqConsume(STQ*, TmqConsumeReq*);
 
-TqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);
+STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);
 
-TqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
+STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
 int             tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
-int tqMoveOffsetToNext(TqGroupHandle*);
+int             tqMoveOffsetToNext(STqGroupHandle*);
 int             tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset);
-int tqRegisterContext(TqGroupHandle*, void* ahandle);
-int tqLaunchQuery(TqGroupHandle*);
-int tqSendLaunchQuery(TqGroupHandle*);
+int             tqRegisterContext(STqGroupHandle*, void* ahandle);
+int             tqLaunchQuery(STqGroupHandle*);
+int             tqSendLaunchQuery(STqGroupHandle*);
 
-int   tqSerializeGroupHandle(const TqGroupHandle* gHandle, TqSerializedHead** ppHead);
+int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead);
 
-const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle** gHandle);
+const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle);
 
 #ifdef __cplusplus
 }

include/libs/parser/parser.h

@@ -153,7 +153,7 @@ typedef struct SParseContext {
  * @param msg      extended error message if exists.
  * @return         error code
  */
-int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen);
+int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen);
 
 typedef enum {
   PAYLOAD_TYPE_KV = 0,

include/libs/planner/planner.h

@@ -84,8 +84,8 @@ typedef struct SProjectPhyNode {
 
 typedef struct SExchangePhyNode {
   SPhyNode    node;
-  uint64_t    templateId;
-  SArray     *pSourceEpSet;  // SEpSet
+  uint64_t    srcTemplateId; // template id of datasource suplans
+  SArray     *pSourceEpSet;  // SEpSet, scheduler fill by calling qSetSuplanExecutionNode
 } SExchangePhyNode;
 
 typedef struct SSubplanId {
@@ -113,6 +113,8 @@ typedef struct SQueryDag {
  */
 int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag);
 
+int32_t qSetSuplanExecutionNode(SArray* subplans, SArray* nodes);
+
 int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str);
 
 
@@ -126,7 +128,7 @@ int32_t qSubPlanToString(struct SSubplan *pPhyNode, char** str);
  * @param pQueryPhyNode
  * @return
  */
-void* qDestroyQueryDag(struct SQueryDag* pDag);
+void qDestroyQueryDag(struct SQueryDag* pDag);
 
 #ifdef __cplusplus
 }

include/libs/planner/plannerOp.h

@@ -27,7 +27,7 @@ OP_ENUM_MACRO(TableScan)
 OP_ENUM_MACRO(DataBlocksOptScan)
 OP_ENUM_MACRO(TableSeqScan)
 OP_ENUM_MACRO(TagScan)
-OP_ENUM_MACRO(TableBlockInfoScan)
+OP_ENUM_MACRO(SystemTableScan)
 OP_ENUM_MACRO(Aggregate)
 OP_ENUM_MACRO(Project)
 OP_ENUM_MACRO(Groupby)

include/libs/wal/wal.h

@@ -16,25 +16,55 @@
 #define _TD_WAL_H_
 
 #include "os.h"
+#include "tarray.h"
 #include "tdef.h"
 #include "tlog.h"
-#include "tarray.h"
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 extern int32_t wDebugFlag;
 
-#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
-#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
-#define wWarn(...)  { if (wDebugFlag & DEBUG_WARN)  { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
-#define wInfo(...)  { if (wDebugFlag & DEBUG_INFO)  { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
-#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
-#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
+#define wFatal(...)                                 \
+  {                                                 \
+    if (wDebugFlag & DEBUG_FATAL) {                 \
+      taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
+    }                                               \
+  }
+#define wError(...)                                 \
+  {                                                 \
+    if (wDebugFlag & DEBUG_ERROR) {                 \
+      taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \
+    }                                               \
+  }
+#define wWarn(...)                                 \
+  {                                                \
+    if (wDebugFlag & DEBUG_WARN) {                 \
+      taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \
+    }                                              \
+  }
+#define wInfo(...)                            \
+  {                                           \
+    if (wDebugFlag & DEBUG_INFO) {            \
+      taosPrintLog("WAL ", 255, __VA_ARGS__); \
+    }                                         \
+  }
+#define wDebug(...)                                  \
+  {                                                  \
+    if (wDebugFlag & DEBUG_DEBUG) {                  \
+      taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
+    }                                                \
+  }
+#define wTrace(...)                                  \
+  {                                                  \
+    if (wDebugFlag & DEBUG_TRACE) {                  \
+      taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
+    }                                                \
+  }
 
 #define WAL_HEAD_VER 0
 #define WAL_NOSUFFIX_LEN 20
-#define WAL_SUFFIX_AT    (WAL_NOSUFFIX_LEN+1)
+#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
 #define WAL_LOG_SUFFIX "log"
 #define WAL_INDEX_SUFFIX "idx"
 #define WAL_REFRESH_MS 1000
@@ -45,18 +75,14 @@ extern int32_t wDebugFlag;
 #define WAL_CUR_FAILED 1
 
 #pragma pack(push, 1)
-typedef enum {
-  TAOS_WAL_NOLOG = 0,
-  TAOS_WAL_WRITE = 1,
-  TAOS_WAL_FSYNC = 2
-} EWalType;
+typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType;
 
 typedef struct SWalReadHead {
   int8_t  headVer;
   uint8_t msgType;
   int8_t  reserved[2];
   int32_t len;
-  int64_t  ingestTs;  //not implemented
+  int64_t ingestTs;  // not implemented
   int64_t version;
   char    body[];
 } SWalReadHead;
@@ -89,37 +115,37 @@ typedef struct SWal {
   // cfg
   SWalCfg cfg;
   int32_t fsyncSeq;
-  //meta
+  // meta
   SWalVer vers;
   int64_t writeLogTfd;
   int64_t writeIdxTfd;
   int32_t writeCur;
-  SArray* fileInfoSet;
-  //status
+  SArray *fileInfoSet;
+  // status
   int64_t totSize;
   int64_t lastRollSeq;
-  //ctl
+  // ctl
   int64_t         refId;
   pthread_mutex_t mutex;
-  //path
+  // path
   char path[WAL_PATH_LEN];
-  //reusable write head
+  // reusable write head
   SWalHead writeHead;
 } SWal;  // WAL HANDLE
 
 typedef struct SWalReadHandle {
-  SWal*        pWal;
+  SWal     *pWal;
   int64_t   readLogTfd;
   int64_t   readIdxTfd;
   int64_t   curFileFirstVer;
   int64_t   curVersion;
   int64_t   capacity;
-  int64_t      status;  //if cursor valid
-  SWalHead*    pHead;
+  int64_t   status;  // if cursor valid
+  SWalHead *pHead;
 } SWalReadHandle;
 #pragma pack(pop)
 
-//typedef int32_t (*FWalWrite)(void *ahandle, void *pHead);
+// typedef int32_t (*FWalWrite)(void *ahandle, void *pHead);
 
 // module initialization
 int32_t walInit();
@@ -141,15 +167,15 @@ int32_t walRollback(SWal *, int64_t ver);
 // notify that previous logs can be pruned safely
 int32_t walBeginSnapshot(SWal *, int64_t ver);
 int32_t walEndSnapshot(SWal *);
-//int32_t  walDataCorrupted(SWal*);
+// int32_t  walDataCorrupted(SWal*);
 
 // read
-SWalReadHandle* walOpenReadHandle(SWal *);
+SWalReadHandle *walOpenReadHandle(SWal *);
 void            walCloseReadHandle(SWalReadHandle *);
 int32_t         walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
 
 int32_t walRead(SWal *, SWalHead **, int64_t ver);
-//int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
+// int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
 
 // lifecycle check
 int64_t walGetFirstVer(SWal *);

include/util/tdef.h

@@ -196,6 +196,7 @@ do { \
 
 #define TSDB_AUTH_LEN             16
 #define TSDB_PASSWORD_LEN         32
+#define TSDB_USET_PASSWORD_LEN    129
 #define TSDB_VERSION_LEN          12
 #define TSDB_LABEL_LEN            8 
 

source/client/CMakeLists.txt

@@ -9,7 +9,7 @@ target_link_libraries(
     taos
     PRIVATE common
     INTERFACE api
-    PRIVATE os util common transport parser
+    PRIVATE os util common transport parser catalog function query
 )
 
 ADD_SUBDIRECTORY(test)
\ No newline at end of file

source/client/inc/clientInt.h

@@ -20,14 +20,15 @@
 extern "C" {
 #endif
 
+#include <common.h>
 #include "taos.h"
 #include "taosmsg.h"
+#include "tdef.h"
+#include "tep.h"
 #include "thash.h"
 #include "tlist.h"
-#include "trpc.h"
-#include "tdef.h"
 #include "tmsgtype.h"
-#include "tep.h"
+#include "trpc.h"
 
 typedef struct SQueryExecMetric {
   int64_t      start;    // start timestamp
@@ -86,12 +87,22 @@ typedef struct STscObj {
   SAppInstInfo      *pAppInfo;
 } STscObj;
 
+typedef struct SClientResultInfo {
+  SSDataBlock *pData;
+  TAOS_FIELD  *resultFields;
+  int32_t      current;
+} SClientResultInfo;
+
 typedef struct SReqBody {
   tsem_t    rspSem;        // not used now
   void*     fp;
   void*     param;
+  int32_t   paramLen;
+  SClientResultInfo* pResInfo;
 } SRequestBody;
 
+#define ERROR_MSG_BUF_DEFAULT_SIZE  512
+
 typedef struct SRequestObj {
   uint64_t         requestId;
   int32_t          type;   // request type
@@ -118,7 +129,7 @@ extern int32_t    tscReqRef;
 extern void      *tscQhandle;
 extern int32_t    tscConnRef;
 
-extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg);
+extern int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody);
 extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen);
 
 int   taos_init();
@@ -129,8 +140,6 @@ void  destroyTscObj(void*pObj);
 void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
 void destroyRequest(SRequestObj* pRequest);
 
-TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
-
 void taos_init_imp(void);
 int taos_options_imp(TSDB_OPTION option, const char *str);
 
@@ -139,6 +148,11 @@ void* openTransporter(const char *user, const char *auth);
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
 void initMsgHandleFp();
 
+TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
+TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
+
+void* doFetchRow(SRequestObj* pRequest);
+
 #ifdef __cplusplus
 }
 #endif

source/client/inc/tscLog.h → source/client/inc/clientLog.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSCLOG_H
-#define TDENGINE_TSCLOG_H
+#ifndef TDENGINE_CLIENTLOG_H
+#define TDENGINE_CLIENTLOG_H
 
 #ifdef __cplusplus
 extern "C" {

source/client/src/client.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "os.h"
-#include "tdef.h"
-#include "tglobal.h"
-#include "clientInt.h"
-#include "tscLog.h"
-
-TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
-  int32_t p = (port != 0)? port:tsServerPort;
-
-  tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db);
-  if (user == NULL) {
-    user = TSDB_DEFAULT_USER;
-  }
-
-  if (pass == NULL) {
-    pass = TSDB_DEFAULT_PASS;
-  }
-
-  return taos_connect_internal(ip, user, pass, NULL, db, p);
-}
-
-TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) {
-  tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db);
-  if (user == NULL) {
-    user = TSDB_DEFAULT_USER;
-  }
-
-  if (auth == NULL) {
-    tscError("No auth info is given, failed to connect to server");
-    return NULL;
-  }
-
-  return taos_connect_internal(ip, user, NULL, auth, db, port);
-}
-
-TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) {
-  char ipStr[TSDB_EP_LEN]      = {0};
-  char dbStr[TSDB_DB_NAME_LEN] = {0};
-  char userStr[TSDB_USER_LEN]  = {0};
-  char passStr[TSDB_PASSWORD_LEN]   = {0};
-
-  strncpy(ipStr,   ip,   MIN(TSDB_EP_LEN - 1, ipLen));
-  strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen));
-  strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen));
-  strncpy(dbStr,   db,   MIN(TSDB_DB_NAME_LEN - 1, dbLen));
-  return taos_connect(ipStr, userStr, passStr, dbStr, port);
-}
-

source/client/src/clientImpl.c

-#include <tpagedfile.h>
+
 #include "clientInt.h"
+#include "clientLog.h"
 #include "tdef.h"
 #include "tep.h"
 #include "tglobal.h"
 #include "tmsgtype.h"
+#include "tnote.h"
+#include "tpagedfile.h"
 #include "tref.h"
-#include "tscLog.h"
+#include "parser.h"
 
 static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
 static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody);
@@ -109,6 +112,71 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
   return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst);
 }
 
+TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
+  STscObj *pTscObj = (STscObj *)taos;
+  if (sqlLen > (size_t) tsMaxSQLStringLen) {
+    tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
+    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
+    return NULL;
+  }
+
+  nPrintTsc("%s", sql)
+
+  SRequestObj* pRequest = createRequest(pTscObj, NULL, NULL, TSDB_SQL_SELECT);
+  if (pRequest == NULL) {
+    tscError("failed to malloc sqlObj");
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pRequest->sqlstr = malloc(sqlLen + 1);
+  if (pRequest->sqlstr == NULL) {
+    tscError("0x%"PRIx64" failed to prepare sql string buffer", pRequest->self);
+
+    pRequest->msgBuf = strdup("failed to prepare sql string buffer");
+    terrno = pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return pRequest;
+  }
+
+  strntolower(pRequest->sqlstr, sql, (int32_t)sqlLen);
+  pRequest->sqlstr[sqlLen] = 0;
+
+  tscDebugL("0x%"PRIx64" SQL: %s", pRequest->requestId, pRequest->sqlstr);
+
+  int32_t code = 0;
+  if (qIsInsertSql(pRequest->sqlstr, sqlLen)) {
+    // todo add
+  } else {
+    int32_t type = 0;
+    void*   output = NULL;
+    int32_t outputLen = 0;
+    code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE);
+    if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW) {
+      pRequest->type = type;
+      pRequest->body.param = output;
+      pRequest->body.paramLen = outputLen;
+
+      SRequestMsgBody body = {0};
+      buildRequestMsgFp[type](pRequest, &body);
+
+      int64_t transporterId = 0;
+      sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
+
+      tsem_wait(&pRequest->body.rspSem);
+      destroyConnectMsg(&body);
+    } else {
+      assert(0);
+    }
+  }
+
+  if (code != TSDB_CODE_SUCCESS) {
+    pRequest->code = code;
+    return pRequest;
+  }
+
+  return pRequest;
+}
+
 int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) {
   pEpSet->version = 0;
 
@@ -272,8 +340,8 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
       pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen);
     }
   } else {
-    tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType],
-             tstrerror(pMsg->code), pMsg->contLen);
+    tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%"PRId64" ms", pRequest->requestId, taosMsg[pMsg->msgType],
+             tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start);
   }
 
   taosReleaseRef(tscReqRef, requestRefId);
@@ -281,3 +349,47 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
 
   sem_post(&pRequest->body.rspSem);
 }
+
+TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) {
+  tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db);
+  if (user == NULL) {
+    user = TSDB_DEFAULT_USER;
+  }
+
+  if (auth == NULL) {
+    tscError("No auth info is given, failed to connect to server");
+    return NULL;
+  }
+
+  return taos_connect_internal(ip, user, NULL, auth, db, port);
+}
+
+TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) {
+  char ipStr[TSDB_EP_LEN]      = {0};
+  char dbStr[TSDB_DB_NAME_LEN] = {0};
+  char userStr[TSDB_USER_LEN]  = {0};
+  char passStr[TSDB_PASSWORD_LEN]   = {0};
+
+  strncpy(ipStr,   ip,   MIN(TSDB_EP_LEN - 1, ipLen));
+  strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen));
+  strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen));
+  strncpy(dbStr,   db,   MIN(TSDB_DB_NAME_LEN - 1, dbLen));
+  return taos_connect(ipStr, userStr, passStr, dbStr, port);
+}
+
+void* doFetchRow(SRequestObj* pRequest) {
+  assert(pRequest != NULL);
+  SClientResultInfo* pResultInfo = pRequest->body.pResInfo;
+
+  if (pResultInfo == NULL || pResultInfo->current >= pResultInfo->pData->info.rows) {
+    if (pResultInfo == NULL) {
+      pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo));
+//      pRequest->body.pResInfo.
+    }
+    // current data set are exhausted, fetch more result from node
+//    if (pRes->row >= pRes->numOfRows && needToFetchNewBlock(pSql)) {
+//      taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj);
+//      tsem_wait(&pSql->rspSem);
+//    }
+  }
+}
\ No newline at end of file

source/client/src/clientmain.c → source/client/src/clientMain.c

 #include "clientInt.h"
-#include "trpc.h"
+#include "clientLog.h"
 #include "os.h"
 #include "taosmsg.h"
 #include "tcache.h"
@@ -7,7 +7,7 @@
 #include "tglobal.h"
 #include "tnote.h"
 #include "tref.h"
-#include "tscLog.h"
+#include "trpc.h"
 #include "tsched.h"
 #include "ttime.h"
 #include "ttimezone.h"
@@ -16,9 +16,6 @@
 #define TSC_VAR_RELEASED    0
 
 static int32_t sentinel = TSC_VAR_NOT_RELEASE;
-static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
-
-extern int32_t tscInitRes;
 
 int taos_options(TSDB_OPTION option, const void *arg, ...) {
   static int32_t lock = 0;
@@ -36,11 +33,6 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
   return ret;
 }
 
-int taos_init() {
-  pthread_once(&tscinit, taos_init_imp);
-  return tscInitRes;
-}
-
 // this function may be called by user or system, or by both simultaneously.
 void taos_cleanup(void) {
   tscDebug("start to cleanup client environment");
@@ -65,6 +57,21 @@ void taos_cleanup(void) {
   taosCloseLog();
 }
 
+TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
+    int32_t p = (port != 0)? port:tsServerPort;
+
+    tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db);
+    if (user == NULL) {
+        user = TSDB_DEFAULT_USER;
+    }
+
+    if (pass == NULL) {
+        pass = TSDB_DEFAULT_PASS;
+    }
+
+    return taos_connect_internal(ip, user, pass, NULL, db, p);
+}
+
 void taos_close(TAOS* taos) {
   if (taos == NULL) {
     return;
@@ -76,10 +83,51 @@ void taos_close(TAOS* taos) {
   taosRemoveRef(tscConnRef, pTscObj->id);
 }
 
+int taos_errno(TAOS_RES *tres) {
+  if (tres == NULL) {
+    return terrno;
+  }
+
+  return ((SRequestObj*) tres)->code;
+}
+
 const char *taos_errstr(TAOS_RES *res) {
+  SRequestObj *pRequest = (SRequestObj *) res;
+
+  if (pRequest == NULL) {
+    return (const char*) tstrerror(terrno);
+  }
 
+  if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) {
+    return pRequest->msgBuf;
+  } else {
+    return (const char*)tstrerror(pRequest->code);
+  }
 }
 
 void taos_free_result(TAOS_RES *res) {
+  SRequestObj* pRequest = (SRequestObj*) res;
+  destroyRequest(pRequest);
+}
+
+TAOS_RES *taos_query(TAOS *taos, const char *sql) {
+  if (taos == NULL || sql == NULL) {
+    return NULL;
+  }
+
+  return taos_query_l(taos, sql, strlen(sql));
+}
+
+TAOS_ROW taos_fetch_row(TAOS_RES *pRes) {
+  if (pRes == NULL) {
+    return NULL;
+  }
+
+  SRequestObj *pRequest = (SRequestObj *) pRes;
+  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
+      pRequest->type == TSDB_SQL_INSERT) {
+    return NULL;
+  }
 
+  return doFetchRow(pRequest);
 }

source/client/src/clientMsgHandler.c

@@ -13,11 +13,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
+#include <catalog.h>
 #include "clientInt.h"
+#include "clientLog.h"
+#include "os.h"
 #include "tmsgtype.h"
 #include "trpc.h"
-#include "tscLog.h"
 
 int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0};
 int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen);
@@ -3091,6 +3092,36 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
 
 #endif
 
+int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
+  pMsgBody->msgType         = TSDB_MSG_TYPE_CONNECT;
+  pMsgBody->msgLen          = sizeof(SConnectMsg);
+  pMsgBody->requestObjRefId = pRequest->self;
+
+  SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg));
+  if (pConnect == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  // TODO refactor full_name
+  char *db;  // ugly code to move the space
+
+  STscObj *pObj = pRequest->pTscObj;
+  pthread_mutex_lock(&pObj->mutex);
+  db = strstr(pObj->db, TS_PATH_DELIMITER);
+
+  db = (db == NULL) ? pObj->db : db + 1;
+  tstrncpy(pConnect->db, db, sizeof(pConnect->db));
+  pthread_mutex_unlock(&pObj->mutex);
+
+  pConnect->pid = htonl(appInfo.pid);
+  pConnect->startTime = htobe64(appInfo.startTime);
+  tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app));
+
+  pMsgBody->pData = pConnect;
+  return 0;
+}
+
 int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
   STscObj *pTscObj = pRequest->pTscObj;
 
@@ -3099,6 +3130,11 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
   pConnect->connId    = htonl(pConnect->connId);
   pConnect->clusterId = htonl(pConnect->clusterId);
 
+  assert(pConnect->epSet.numOfEps > 0);
+  for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) {
+    pConnect->epSet.port[i] = htons(pConnect->epSet.port[i]);
+  }
+
   // TODO refactor
   pthread_mutex_lock(&pTscObj->mutex);
   char temp[TSDB_TABLE_FNAME_LEN * 2] = {0};
@@ -3108,13 +3144,12 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
   tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db));
   pthread_mutex_unlock(&pTscObj->mutex);
 
-  assert(pConnect->epSet.numOfEps > 0);
   if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) {
     updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet);
   }
 
   for (int i = 0; i < pConnect->epSet.numOfEps; ++i) {
-    tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id);
+    tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pConnect->epSet.port[i], pTscObj->id);
   }
 
   pTscObj->connId = pConnect->connId;
@@ -3124,10 +3159,81 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
   atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
 
   tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns);
-  //  createHbObj(pTscObj);
+  return 0;
+}
+
+int32_t buildCreateUserMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
+  pMsgBody->msgType         = TSDB_MSG_TYPE_CREATE_USER;
+  pMsgBody->msgLen          = sizeof(SCreateUserMsg);
+  pMsgBody->requestObjRefId = pRequest->self;
+  pMsgBody->pData           = pRequest->body.param;
+  return 0;
+}
+
+int32_t buildShowMsg(SRequestObj* pRequest, SRequestMsgBody* pMsgBody) {
+  pMsgBody->msgType         = TSDB_MSG_TYPE_SHOW;
+  pMsgBody->msgLen          = pRequest->body.paramLen;
+  pMsgBody->requestObjRefId = pRequest->self;
+  pMsgBody->pData           = pRequest->body.param;
+}
+
+STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
+  assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2);
+
+  size_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema);
+  STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize);
+
+  pTableMeta->tableType = pTableMetaMsg->tableType;
+  pTableMeta->vgId      = pTableMetaMsg->vgId;
+  pTableMeta->suid      = pTableMetaMsg->suid;
+  pTableMeta->uid       = pTableMetaMsg->tuid;
+
+  pTableMeta->tableInfo = (STableComInfo) {
+      .numOfTags    = pTableMetaMsg->numOfTags,
+      .precision    = pTableMetaMsg->precision,
+      .numOfColumns = pTableMetaMsg->numOfColumns,
+  };
+
+  pTableMeta->sversion = pTableMetaMsg->sversion;
+  pTableMeta->tversion = pTableMetaMsg->tversion;
+
+  memcpy(pTableMeta->schema, pTableMetaMsg->pSchema, schemaSize);
+
+  int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns;
+  for(int32_t i = 0; i < numOfTotalCols; ++i) {
+    pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
+  }
+
+  return pTableMeta;
+}
+
+int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
+  SShowRsp* pShow = (SShowRsp *)pMsg;
+  pShow->showId   = htonl(pShow->showId);
+
+  STableMetaMsg *pMetaMsg = &(pShow->tableMeta);
+  pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns);
+
+  SSchema* pSchema = pMetaMsg->pSchema;
+  pMetaMsg->tuid = htobe64(pMetaMsg->tuid);
+  for (int i = 0; i < pMetaMsg->numOfColumns; ++i) {
+    pSchema->bytes = htons(pSchema->bytes);
+    pSchema++;
+  }
+
+  STableMeta* pTableMeta = createTableMetaFromMsg(pMetaMsg);
+  SSchema *pTableSchema = pTableMeta->schema;
+
+  TAOS_FIELD* pFields = calloc(1, pTableMeta->tableInfo.numOfColumns);
+  for (int16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i, ++pSchema) {
+    tstrncpy(pFields[i].name, pTableSchema[i].name, tListLen(pFields[i].name));
+    pFields[i].type  = pTableSchema[i].type;
+    pFields[i].bytes = pTableSchema[i].bytes;
+  }
+
+//  pRequest->body.resultFields = pFields;
+//  pRequest->body.numOfFields = pTableMeta->tableInfo.numOfColumns;
 
-  // launch a timer to send heartbeat to maintain the connection and send status to mnode
-  //  taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer);
   return 0;
 }
 
@@ -3207,6 +3313,12 @@ void initMsgHandleFp() {
   tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp;
 #endif
 
-//  buildRequestMsgFp[TSDB_SQL_CONNECT]  = tscBuildConnectMsg;
+  buildRequestMsgFp[TSDB_SQL_CONNECT]  = buildConnectMsg;
   handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp;
+
+  buildRequestMsgFp[TSDB_SQL_CREATE_USER]  = buildCreateUserMsg;
+
+  buildRequestMsgFp[TSDB_SQL_SHOW]         = buildShowMsg;
+  handleRequestRspFp[TSDB_SQL_SHOW]        = processShowRsp;
+
 }
\ No newline at end of file

source/client/src/tscEnv.c

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "clientInt.h"
+#include "clientLog.h"
 #include "os.h"
 #include "taosmsg.h"
 #include "tcache.h"
@@ -20,12 +22,10 @@
 #include "tglobal.h"
 #include "tnote.h"
 #include "tref.h"
-#include "tscLog.h"
+#include "trpc.h"
 #include "tsched.h"
 #include "ttime.h"
-#include "trpc.h"
 #include "ttimezone.h"
-#include "clientInt.h"
 
 #define TSC_VAR_NOT_RELEASE 1
 #define TSC_VAR_RELEASED    0
@@ -35,6 +35,7 @@ int32_t    tscReqRef  = -1;
 int32_t    tscConnRef = -1;
 void      *tscQhandle = NULL;
 
+static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
 int32_t tsNumOfThreads = 1;
 volatile int32_t tscInitRes = 0;
 
@@ -172,6 +173,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
   pRequest->pTscObj    = pObj;
   pRequest->body.fp    = fp;
   pRequest->body.param = param;
+  pRequest->msgBuf     = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
   tsem_init(&pRequest->body.rspSem, 0, 0);
 
   registerRequest(pRequest);
@@ -250,6 +252,11 @@ void taos_init_imp(void) {
   tscDebug("client is initialized successfully");
 }
 
+int taos_init() {
+  pthread_once(&tscinit, taos_init_imp);
+  return tscInitRes;
+}
+
 int taos_options_imp(TSDB_OPTION option, const char *str) {
   SGlobalCfg *cfg = NULL;
 

source/client/test/clientTests.cpp

@@ -14,6 +14,7 @@
  */
 
 #include <gtest/gtest.h>
+#include <taoserror.h>
 #include <iostream>
 #include "tglobal.h"
 #pragma GCC diagnostic ignored "-Wwrite-strings"
@@ -36,5 +37,16 @@ TEST(testCase, driverInit_Test) {
   TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
   assert(pConn != NULL);
 
+//  TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'");
+//  if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
+//    printf("failed to create user, reason:%s\n", taos_errstr(pRes));
+//  }
+//
+//  taos_free_result(pRes);
+
+  TAOS_RES* pRes = taos_query(pConn, "show users");
+  TAOS_ROW pRow = taos_fetch_row(pRes);
+  assert(pRow != NULL);
+
   taos_close(pConn);
 }
\ No newline at end of file

source/dnode/mnode/impl/src/mndUser.c

@@ -431,7 +431,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
-  pMeta->numOfColumns = htons(cols);
+  pMeta->numOfColumns = htonl(cols);
   pShow->numOfColumns = cols;
 
   pShow->offset[0] = 0;

source/dnode/vnode/tq/inc/tqMetaStore.h

@@ -25,9 +25,9 @@ extern "C" {
 
 
 STqMetaStore*  tqStoreOpen(const char* path,
-    TqSerializeFun   pSerializer,
-    TqDeserializeFun pDeserializer,
-    TqDeleteFun      pDeleter,
+    FTqSerialize   pSerializer,
+    FTqDeserialize pDeserializer,
+    FTqDelete      pDeleter,
     int32_t          tqConfigFlag
   );
 int32_t       tqStoreClose(STqMetaStore*);

source/dnode/vnode/tq/src/tq.c

@@ -16,34 +16,34 @@
 #include "tqInt.h"
 #include "tqMetaStore.h"
 
-//static
-//read next version data
+// static
+// read next version data
 //
-//send to fetch queue
+// send to fetch queue
 //
-//handle management message
+// handle management message
 //
 
-int tqGetgHandleSSize(const TqGroupHandle *gHandle);
+int tqGetgHandleSSize(const STqGroupHandle* gHandle);
 int tqBufHandleSSize();
 int tqBufItemSSize();
 
-TqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
-  TqGroupHandle* gHandle;
+STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
+  STqGroupHandle* gHandle;
   return NULL;
 }
 
-void* tqSerializeListHandle(TqListHandle* listHandle, void* ptr);
-void* tqSerializeBufHandle(TqBufferHandle* bufHandle, void* ptr);
-void* tqSerializeBufItem(TqBufferItem* bufItem, void* ptr);
+void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr);
+void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr);
+void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr);
 
-const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle* bufHandle);
-const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem* bufItem);
+const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle);
+const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem);
 
-STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac) {
+STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) {
   STQ* pTq = malloc(sizeof(STQ));
-  if(pTq == NULL) {
-    //TODO: memory error
+  if (pTq == NULL) {
+    // TODO: memory error
     return NULL;
   }
   pTq->path = strdup(path);
@@ -51,40 +51,35 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAl
   pTq->tqLogReader = tqLogReader;
   pTq->tqMemRef.pAlloctorFactory = allocFac;
   pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
-  if(pTq->tqMemRef.pAllocator == NULL) {
-    //TODO
-  }
-  pTq->tqMeta = tqStoreOpen(path,
-                            (TqSerializeFun)tqSerializeGroupHandle,
-                            (TqDeserializeFun)tqDeserializeGroupHandle,
-                            free,
-                            0);
-  if(pTq->tqMeta == NULL) {
-    //TODO: free STQ
+  if (pTq->tqMemRef.pAllocator == NULL) {
+    // TODO
+  }
+  pTq->tqMeta =
+      tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0);
+  if (pTq->tqMeta == NULL) {
+    // TODO: free STQ
     return NULL;
   }
   return pTq;
 }
 
-static int tqProtoCheck(TmqMsgHead *pMsg) {
-  return pMsg->protoVer == 0;
-}
+static int tqProtoCheck(TmqMsgHead* pMsg) { return pMsg->protoVer == 0; }
 
-static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg** ppQuery) {
-  //clean old item and move forward
+static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) {
+  // clean old item and move forward
   int32_t consumeOffset = pAck->consumeOffset;
   int     idx = consumeOffset % TQ_BUFFER_SIZE;
   ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor);
   tfree(bHandle->buffer[idx].content);
-  if( 1 /* TODO: need to launch new query */) {
-    TqQueryMsg* pNewQuery = malloc(sizeof(TqQueryMsg));
-    if(pNewQuery == NULL) {
-      //TODO: memory insufficient
+  if (1 /* TODO: need to launch new query */) {
+    STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg));
+    if (pNewQuery == NULL) {
+      // TODO: memory insufficient
       return -1;
     }
-    //TODO: lock executor
+    // TODO: lock executor
     pNewQuery->exec->executor = bHandle->buffer[idx].executor;
-    //TODO: read from wal and assign to src
+    // TODO: read from wal and assign to src
     pNewQuery->exec->src = 0;
     pNewQuery->exec->dest = &bHandle->buffer[idx];
     pNewQuery->next = *ppQuery;
@@ -93,98 +88,94 @@ static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg**
   return 0;
 }
 
-static int tqAck(TqGroupHandle* gHandle, TmqAcks* pAcks) {
+static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) {
   int32_t    ackNum = pAcks->ackNum;
-  TmqOneAck *acks = pAcks->acks;
-  //double ptr for acks and list
+  TmqOneAck* acks = pAcks->acks;
+  // double ptr for acks and list
   int            i = 0;
-  TqListHandle* node = gHandle->head;
+  STqListHandle* node = gHandle->head;
   int            ackCnt = 0;
-  TqQueryMsg *pQuery = NULL;
-  while(i < ackNum && node->next) {
-    if(acks[i].topicId == node->next->bufHandle.topicId) {
+  STqQueryMsg*   pQuery = NULL;
+  while (i < ackNum && node->next) {
+    if (acks[i].topicId == node->next->bufHandle.topicId) {
       ackCnt++;
       tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery);
-    } else if(acks[i].topicId < node->next->bufHandle.topicId) {
+    } else if (acks[i].topicId < node->next->bufHandle.topicId) {
       i++;
     } else {
       node = node->next;
     }
   }
-  if(pQuery) {
-    //post message
+  if (pQuery) {
+    // post message
   }
   return ackCnt;
 }
 
-static int tqCommitTCGroup(TqGroupHandle* handle) {
-  //persist modification into disk
+static int tqCommitTCGroup(STqGroupHandle* handle) {
+  // persist modification into disk
   return 0;
 }
 
-int tqCreateTCGroup(STQ *pTq, int64_t topicId, int64_t cgId, int64_t cId, TqGroupHandle** handle) {
-  //create in disk
-  TqGroupHandle* gHandle = (TqGroupHandle*)malloc(sizeof(TqGroupHandle));
-  if(gHandle == NULL) {
-    //TODO
+int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) {
+  // create in disk
+  STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle));
+  if (gHandle == NULL) {
+    // TODO
     return -1;
   }
-  memset(gHandle, 0, sizeof(TqGroupHandle));
+  memset(gHandle, 0, sizeof(STqGroupHandle));
 
   return 0;
 }
 
-TqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
-  TqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId);
-  if(gHandle == NULL) {
+STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
+  STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId);
+  if (gHandle == NULL) {
     int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle);
-    if(code != 0) {
-      //TODO
+    if (code != 0) {
+      // TODO
       return NULL;
     }
   }
 
-  //create
-  //open
+  // create
+  // open
   return gHandle;
 }
 
-int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
-  return 0;
-}
+int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; }
 
 int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
-  //delete from disk
+  // delete from disk
   return 0;
 }
 
-static int tqFetch(TqGroupHandle* gHandle, void** msg) {
-  TqListHandle* head = gHandle->head;
-  TqListHandle* node = head;
+static int tqFetch(STqGroupHandle* gHandle, void** msg) {
+  STqListHandle* head = gHandle->head;
+  STqListHandle* node = head;
   int            totSize = 0;
-  //TODO: make it a macro
+  // TODO: make it a macro
   int            sizeLimit = 4 * 1024;
   TmqMsgContent* buffer = malloc(sizeLimit);
-  if(buffer == NULL) {
-    //TODO:memory insufficient
+  if (buffer == NULL) {
+    // TODO:memory insufficient
     return -1;
   }
-  //iterate the list to get msgs of all topics
-  //until all topic iterated or msgs over sizeLimit
-  while(node->next) {
+  // iterate the list to get msgs of all topics
+  // until all topic iterated or msgs over sizeLimit
+  while (node->next) {
     node = node->next;
-    TqBufferHandle* bufHandle = &node->bufHandle;
+    STqBufferHandle* bufHandle = &node->bufHandle;
     int              idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE;
-    if(bufHandle->buffer[idx].content != NULL &&
-        bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset
-        ) {
+    if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) {
       totSize += bufHandle->buffer[idx].size;
-      if(totSize > sizeLimit) {
-        void *ptr = realloc(buffer, totSize);
-        if(ptr == NULL) {
+      if (totSize > sizeLimit) {
+        void* ptr = realloc(buffer, totSize);
+        if (ptr == NULL) {
           totSize -= bufHandle->buffer[idx].size;
-          //TODO:memory insufficient
-          //return msgs already copied
+          // TODO:memory insufficient
+          // return msgs already copied
           break;
         }
       }
@@ -194,7 +185,7 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) {
       buffer = POINTER_SHIFT(buffer, sizeof(int64_t));
       memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size);
       buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size);
-      if(totSize > sizeLimit) {
+      if (totSize > sizeLimit) {
         break;
       }
     }
@@ -202,104 +193,98 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) {
   return totSize;
 }
 
-TqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) {
-  return NULL;
-}
+STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; }
 
-int tqLaunchQuery(TqGroupHandle* gHandle) {
-  return 0;
-}
+int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; }
 
-int tqSendLaunchQuery(TqGroupHandle* gHandle) {
-  return 0;
-}
+int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; }
 
 /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/
-  /*return 0;*/
+/*return 0;*/
 /*}*/
 
-int tqPushMsg(STQ* pTq , void* p, int64_t version) {
-  //add reference
-  //judge and launch new query
+int tqPushMsg(STQ* pTq, void* p, int64_t version) {
+  // add reference
+  // judge and launch new query
   return 0;
 }
 
 int tqCommit(STQ* pTq) {
-  //do nothing
+  // do nothing
   return 0;
 }
 
 int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) {
-  if(!tqProtoCheck((TmqMsgHead *)pMsg)) {
-    //proto version invalid
+  if (!tqProtoCheck((TmqMsgHead*)pMsg)) {
+    // proto version invalid
     return -1;
   }
   int64_t         clientId = pMsg->head.clientId;
-  TqGroupHandle *gHandle = tqGetGroupHandle(pTq, clientId);
-  if(gHandle == NULL) {
-    //client not connect
+  STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId);
+  if (gHandle == NULL) {
+    // client not connect
     return -1;
   }
-  if(pMsg->acks.ackNum != 0) {
-    if(tqAck(gHandle, &pMsg->acks) != 0) {
-      //ack not success
+  if (pMsg->acks.ackNum != 0) {
+    if (tqAck(gHandle, &pMsg->acks) != 0) {
+      // ack not success
       return -1;
     }
   }
 
-  TmqConsumeRsp *pRsp = (TmqConsumeRsp*) pMsg;
+  TmqConsumeRsp* pRsp = (TmqConsumeRsp*)pMsg;
 
-  if(tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) {
-    //fetch error
+  if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) {
+    // fetch error
     return -1;
   }
 
-  //judge and launch new query
-  if(tqLaunchQuery(gHandle)) {
-    //launch query error
+  // judge and launch new query
+  if (tqLaunchQuery(gHandle)) {
+    // launch query error
     return -1;
   }
   return 0;
 }
 
-int tqSerializeGroupHandle(const TqGroupHandle *gHandle, TqSerializedHead** ppHead) {
-  //calculate size
-  int sz = tqGetgHandleSSize(gHandle) + sizeof(TqSerializedHead);
-  if(sz > (*ppHead)->ssize) {
+int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) {
+  // calculate size
+  int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead);
+  if (sz > (*ppHead)->ssize) {
     void* tmpPtr = realloc(*ppHead, sz);
-    if(tmpPtr == NULL) {
+    if (tmpPtr == NULL) {
       free(*ppHead);
-      //TODO: memory err
+      // TODO: memory err
       return -1;
     }
     *ppHead = tmpPtr;
     (*ppHead)->ssize = sz;
   }
   void* ptr = (*ppHead)->content;
-  //do serialization
+  // do serialization
   *(int64_t*)ptr = gHandle->cId;
   ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
   *(int64_t*)ptr = gHandle->cgId;
   ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
   *(int32_t*)ptr = gHandle->topicNum;
   ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
-  if(gHandle->topicNum > 0) {
+  if (gHandle->topicNum > 0) {
     tqSerializeListHandle(gHandle->head, ptr);
   }
   return 0;
 }
 
-void* tqSerializeListHandle(TqListHandle *listHandle, void* ptr) {
-  TqListHandle *node = listHandle;
+void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) {
+  STqListHandle* node = listHandle;
   ASSERT(node != NULL);
-  while(node) {
+  while (node) {
     ptr = tqSerializeBufHandle(&node->bufHandle, ptr);
     node = node->next;
   }
   return ptr;
 }
 
-void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) {
+void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) {
   *(int64_t*)ptr = bufHandle->nextConsumeOffset;
   ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
   *(int64_t*)ptr = bufHandle->topicId;
@@ -308,20 +293,20 @@ void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) {
   ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
   *(int32_t*)ptr = bufHandle->tail;
   ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
-  for(int i = 0; i < TQ_BUFFER_SIZE; i++) {
+  for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
     ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr);
   }
   return ptr;
 }
 
-void* tqSerializeBufItem(TqBufferItem *bufItem, void* ptr) {
-  //TODO: do we need serialize this? 
-  //mainly for executor
+void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) {
+  // TODO: do we need serialize this?
+  // mainly for executor
   return ptr;
 }
 
-const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle **ppGHandle) {
-  TqGroupHandle *gHandle = *ppGHandle;
+const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) {
+  STqGroupHandle* gHandle = *ppGHandle;
   const void*     ptr = pHead->content;
   gHandle->cId = *(int64_t*)ptr;
   ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
@@ -331,20 +316,20 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl
   gHandle->topicNum = *(int32_t*)ptr;
   ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
   gHandle->head = NULL;
-  TqListHandle *node = gHandle->head;
-  for(int i = 0; i < gHandle->topicNum; i++) {
-    if(gHandle->head == NULL) {
-      if((node = malloc(sizeof(TqListHandle))) == NULL) {
-        //TODO: error
+  STqListHandle* node = gHandle->head;
+  for (int i = 0; i < gHandle->topicNum; i++) {
+    if (gHandle->head == NULL) {
+      if ((node = malloc(sizeof(STqListHandle))) == NULL) {
+        // TODO: error
         return NULL;
       }
-      node->next= NULL;
+      node->next = NULL;
       ptr = tqDeserializeBufHandle(ptr, &node->bufHandle);
       gHandle->head = node;
     } else {
-      node->next = malloc(sizeof(TqListHandle));
-      if(node->next == NULL) {
-        //TODO: error
+      node->next = malloc(sizeof(STqListHandle));
+      if (node->next == NULL) {
+        // TODO: error
         return NULL;
       }
       node->next->next = NULL;
@@ -355,7 +340,7 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl
   return ptr;
 }
 
-const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle) {
+const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) {
   const void* ptr = pBytes;
   bufHandle->nextConsumeOffset = *(int64_t*)ptr;
   ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
@@ -365,24 +350,22 @@ const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle
   ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
   bufHandle->tail = *(int32_t*)ptr;
   ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
-  for(int i = 0; i < TQ_BUFFER_SIZE; i++) {
+  for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
     ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]);
   }
   return ptr;
 }
 
-const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem *bufItem) {
-  return pBytes;
-}
+const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; }
 
-//TODO: make this a macro
-int tqGetgHandleSSize(const TqGroupHandle *gHandle) {
-  return sizeof(int64_t) * 2 //cId + cgId
-    + sizeof(int32_t)        //topicNum
+// TODO: make this a macro
+int tqGetgHandleSSize(const STqGroupHandle* gHandle) {
+  return sizeof(int64_t) * 2  // cId + cgId
+         + sizeof(int32_t)    // topicNum
          + gHandle->topicNum * tqBufHandleSSize();
 }
 
-//TODO: make this a macro
+// TODO: make this a macro
 int tqBufHandleSSize() {
   return sizeof(int64_t) * 2    // nextConsumeOffset + topicId
          + sizeof(int32_t) * 2  // head + tail
@@ -390,7 +373,7 @@ int tqBufHandleSSize() {
 }
 
 int tqBufItemSSize() {
-  //TODO: do this need serialization?
-  //mainly for executor
+  // TODO: do this need serialization?
+  // mainly for executor
   return 0;
 }

source/dnode/vnode/tq/src/tqMetaStore.c

@@ -13,11 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 #include "tqMetaStore.h"
-//TODO:replace by an abstract file layer
-#include "osDir.h"
+// TODO:replace by an abstract file layer
 #include <fcntl.h>
 #include <string.h>
 #include <unistd.h>
+#include "osDir.h"
 
 #define TQ_META_NAME "tq.meta"
 #define TQ_IDX_NAME "tq.idx"
@@ -25,8 +25,8 @@
 static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value);
 static void*   tqHandleGetUncommitted(STqMetaStore*, int64_t key);
 
-static inline void tqLinkUnpersist(STqMetaStore *pMeta, STqMetaList* pNode) {
-  if(pNode->unpersistNext == NULL) {
+static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
+  if (pNode->unpersistNext == NULL) {
     pNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
     pNode->unpersistPrev = pMeta->unpersistHead;
     pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode;
@@ -41,24 +41,24 @@ static inline int tqSeekLastPage(int fd) {
   return lseek(fd, curPageOffset, SEEK_SET);
 }
 
-//TODO: the struct is tightly coupled with index entry
-typedef struct TqIdxPageHead {
+// TODO: the struct is tightly coupled with index entry
+typedef struct STqIdxPageHead {
   int16_t writeOffset;
   int8_t  unused[14];
-} TqIdxPageHead;
+} STqIdxPageHead;
 
-typedef struct TqIdxPageBuf {
-  TqIdxPageHead head;
+typedef struct STqIdxPageBuf {
+  STqIdxPageHead head;
   char           buffer[TQ_IDX_PAGE_BODY_SIZE];
-} TqIdxPageBuf;
+} STqIdxPageBuf;
 
-static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) {
+static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
   int offset = tqSeekLastPage(fd);
   int nBytes;
-  if((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
+  if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
     return -1;
   }
-  if(nBytes == 0) {
+  if (nBytes == 0) {
     memset(pBuf, 0, TQ_PAGE_SIZE);
     pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
   }
@@ -67,28 +67,24 @@ static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) {
   return lseek(fd, offset, SEEK_SET);
 }
 
-STqMetaStore* tqStoreOpen(const char* path,
-    TqSerializeFun   serializer,
-    TqDeserializeFun deserializer,
-    TqDeleteFun      deleter,
-    int32_t          tqConfigFlag
-    ) {
+STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter,
+                          int32_t tqConfigFlag) {
   STqMetaStore* pMeta = malloc(sizeof(STqMetaStore));
-  if(pMeta == NULL) {
-    //close
+  if (pMeta == NULL) {
+    // close
     return NULL;
   }
   memset(pMeta, 0, sizeof(STqMetaStore));
 
-  //concat data file name and index file name
+  // concat data file name and index file name
   size_t pathLen = strlen(path);
-  pMeta->dirPath = malloc(pathLen+1);
-  if(pMeta->dirPath != NULL) {
-    //TODO: memory insufficient
+  pMeta->dirPath = malloc(pathLen + 1);
+  if (pMeta->dirPath != NULL) {
+    // TODO: memory insufficient
   }
   strcpy(pMeta->dirPath, path);
 
-  char name[pathLen+10];
+  char name[pathLen + 10];
 
   strcpy(name, path);
   if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
@@ -96,30 +92,28 @@ STqMetaStore* tqStoreOpen(const char* path,
   }
   strcat(name, "/" TQ_IDX_NAME);
   int idxFd = open(name, O_RDWR | O_CREAT, 0755);
-  if(idxFd < 0) {
+  if (idxFd < 0) {
     ASSERT(false);
-    //close file
-    //free memory
+    // close file
+    // free memory
     return NULL;
   }
 
   pMeta->idxFd = idxFd;
   pMeta->unpersistHead = malloc(sizeof(STqMetaList));
-  if(pMeta->unpersistHead == NULL) {
+  if (pMeta->unpersistHead == NULL) {
     ASSERT(false);
-    //close file
-    //free memory
+    // close file
+    // free memory
     return NULL;
   }
   memset(pMeta->unpersistHead, 0, sizeof(STqMetaList));
-  pMeta->unpersistHead->unpersistNext
-    = pMeta->unpersistHead->unpersistPrev
-    = pMeta->unpersistHead;
+  pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead;
 
   strcpy(name, path);
   strcat(name, "/" TQ_META_NAME);
   int fileFd = open(name, O_RDWR | O_CREAT, 0755);
-  if(fileFd < 0){
+  if (fileFd < 0) {
     ASSERT(false);
     return NULL;
   }
@@ -131,63 +125,63 @@ STqMetaStore* tqStoreOpen(const char* path,
   pMeta->pDeleter = deleter;
   pMeta->tqConfigFlag = tqConfigFlag;
 
-  //read idx file and load into memory
-  TqIdxPageBuf idxBuf;
-  TqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
-  if(serializedObj == NULL) {
-    //TODO:memory insufficient
+  // read idx file and load into memory
+  STqIdxPageBuf      idxBuf;
+  STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
+  if (serializedObj == NULL) {
+    // TODO:memory insufficient
   }
   int  idxRead;
   int  allocated = TQ_PAGE_SIZE;
   bool readEnd = false;
-  while((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
-    if(idxRead == -1) {
-      //TODO: handle error
+  while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
+    if (idxRead == -1) {
+      // TODO: handle error
       ASSERT(false);
     }
     ASSERT(idxBuf.head.writeOffset == idxRead);
-    //loop read every entry
-    for(int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
-      STqMetaList *pNode = malloc(sizeof(STqMetaList));
-      if(pNode == NULL) {
-        //TODO: free memory and return error
+    // loop read every entry
+    for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
+      STqMetaList* pNode = malloc(sizeof(STqMetaList));
+      if (pNode == NULL) {
+        // TODO: free memory and return error
       }
       memset(pNode, 0, sizeof(STqMetaList));
       memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
 
       lseek(fileFd, pNode->handle.offset, SEEK_SET);
-      if(allocated < pNode->handle.serializedSize) {
-        void *ptr = realloc(serializedObj, pNode->handle.serializedSize);
-        if(ptr == NULL) {
-          //TODO: memory insufficient
+      if (allocated < pNode->handle.serializedSize) {
+        void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
+        if (ptr == NULL) {
+          // TODO: memory insufficient
         }
         serializedObj = ptr;
         allocated = pNode->handle.serializedSize;
       }
       serializedObj->ssize = pNode->handle.serializedSize;
-      if(read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
-        //TODO: read error
+      if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
+        // TODO: read error
       }
-      if(serializedObj->action == TQ_ACTION_INUSE) {
-        if(serializedObj->ssize != sizeof(TqSerializedHead)) {
+      if (serializedObj->action == TQ_ACTION_INUSE) {
+        if (serializedObj->ssize != sizeof(STqSerializedHead)) {
           pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse);
         } else {
           pNode->handle.valueInUse = TQ_DELETE_TOKEN;
         }
-      } else if(serializedObj->action == TQ_ACTION_INTXN) {
-        if(serializedObj->ssize != sizeof(TqSerializedHead)) {
+      } else if (serializedObj->action == TQ_ACTION_INTXN) {
+        if (serializedObj->ssize != sizeof(STqSerializedHead)) {
           pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn);
         } else {
           pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
         }
-      } else if(serializedObj->action == TQ_ACTION_INUSE_CONT) {
-        if(serializedObj->ssize != sizeof(TqSerializedHead)) {
+      } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) {
+        if (serializedObj->ssize != sizeof(STqSerializedHead)) {
           pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse);
         } else {
           pNode->handle.valueInUse = TQ_DELETE_TOKEN;
         }
-        TqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
-        if(ptr->ssize != sizeof(TqSerializedHead)) {
+        STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
+        if (ptr->ssize != sizeof(STqSerializedHead)) {
           pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn);
         } else {
           pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
@@ -196,22 +190,21 @@ STqMetaStore* tqStoreOpen(const char* path,
         ASSERT(0);
       }
 
-      //put into list
+      // put into list
       int          bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
       STqMetaList* pBucketNode = pMeta->bucket[bucketKey];
-      if(pBucketNode == NULL) {
+      if (pBucketNode == NULL) {
         pMeta->bucket[bucketKey] = pNode;
-      } else if(pBucketNode->handle.key == pNode->handle.key) {
+      } else if (pBucketNode->handle.key == pNode->handle.key) {
         pNode->next = pBucketNode->next;
         pMeta->bucket[bucketKey] = pNode;
       } else {
-        while(pBucketNode->next &&
-            pBucketNode->next->handle.key != pNode->handle.key) {
+        while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) {
           pBucketNode = pBucketNode->next;
         }
-        if(pBucketNode->next) {
+        if (pBucketNode->next) {
           ASSERT(pBucketNode->next->handle.key == pNode->handle.key);
-          STqMetaList *pNodeFound = pBucketNode->next;
+          STqMetaList* pNodeFound = pBucketNode->next;
           pNode->next = pNodeFound->next;
           pBucketNode->next = pNode;
           pBucketNode = pNodeFound;
@@ -221,13 +214,11 @@ STqMetaStore* tqStoreOpen(const char* path,
           pBucketNode = NULL;
         }
       }
-      if(pBucketNode) {
-        if(pBucketNode->handle.valueInUse
-            && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
+      if (pBucketNode) {
+        if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
           pMeta->pDeleter(pBucketNode->handle.valueInUse);
         }
-        if(pBucketNode->handle.valueInTxn
-            && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
+        if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
           pMeta->pDeleter(pBucketNode->handle.valueInTxn);
         }
         free(pBucketNode);
@@ -239,23 +230,21 @@ STqMetaStore* tqStoreOpen(const char* path,
 }
 
 int32_t tqStoreClose(STqMetaStore* pMeta) {
-  //commit data and idx
+  // commit data and idx
   tqStorePersist(pMeta);
-  ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next==NULL);
+  ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
   close(pMeta->fileFd);
   close(pMeta->idxFd);
-  //free memory
-  for(int i = 0; i < TQ_BUCKET_SIZE; i++) {
+  // free memory
+  for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
     STqMetaList* pNode = pMeta->bucket[i];
-    while(pNode) {
+    while (pNode) {
       ASSERT(pNode->unpersistNext == NULL);
       ASSERT(pNode->unpersistPrev == NULL);
-      if(pNode->handle.valueInTxn
-          && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
+      if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
         pMeta->pDeleter(pNode->handle.valueInTxn);
       }
-      if(pNode->handle.valueInUse
-          && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
+      if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
         pMeta->pDeleter(pNode->handle.valueInUse);
       }
       STqMetaList* next = pNode->next;
@@ -272,17 +261,15 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
 int32_t tqStoreDelete(STqMetaStore* pMeta) {
   close(pMeta->fileFd);
   close(pMeta->idxFd);
-  //free memory
-  for(int i = 0; i < TQ_BUCKET_SIZE; i++) {
+  // free memory
+  for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
     STqMetaList* pNode = pMeta->bucket[i];
     pMeta->bucket[i] = NULL;
-    while(pNode) {
-      if(pNode->handle.valueInTxn
-          && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
+    while (pNode) {
+      if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
         pMeta->pDeleter(pNode->handle.valueInTxn);
       }
-      if(pNode->handle.valueInUse
-          && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
+      if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
         pMeta->pDeleter(pNode->handle.valueInUse);
       }
       STqMetaList* next = pNode->next;
@@ -297,26 +284,26 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) {
   return 0;
 }
 
-//TODO: wrap in tfile
+// TODO: wrap in tfile
 int32_t tqStorePersist(STqMetaStore* pMeta) {
-  TqIdxPageBuf idxBuf;
+  STqIdxPageBuf      idxBuf;
   int64_t*           bufPtr = (int64_t*)idxBuf.buffer;
-  STqMetaList *pHead = pMeta->unpersistHead;
-  STqMetaList *pNode = pHead->unpersistNext;
-  TqSerializedHead *pSHead = malloc(sizeof(TqSerializedHead));
-  if(pSHead == NULL) {
-    //TODO: memory error
+  STqMetaList*       pHead = pMeta->unpersistHead;
+  STqMetaList*       pNode = pHead->unpersistNext;
+  STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead));
+  if (pSHead == NULL) {
+    // TODO: memory error
     return -1;
   }
   pSHead->ver = TQ_SVER;
   pSHead->checksum = 0;
-  pSHead->ssize = sizeof(TqSerializedHead);
-  int allocatedSize = sizeof(TqSerializedHead);
+  pSHead->ssize = sizeof(STqSerializedHead);
+  int allocatedSize = sizeof(STqSerializedHead);
   int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
 
   tqReadLastPage(pMeta->idxFd, &idxBuf);
 
-  if(idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
+  if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
     lseek(pMeta->idxFd, 0, SEEK_END);
     memset(&idxBuf, 0, TQ_PAGE_SIZE);
     idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
@@ -324,18 +311,18 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
     bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset);
   }
 
-  while(pHead != pNode) {
+  while (pHead != pNode) {
     int nBytes = 0;
 
-    if(pNode->handle.valueInUse) {
-      if(pNode->handle.valueInTxn) {
+    if (pNode->handle.valueInUse) {
+      if (pNode->handle.valueInTxn) {
         pSHead->action = TQ_ACTION_INUSE_CONT;
       } else {
         pSHead->action = TQ_ACTION_INUSE;
       }
 
-      if(pNode->handle.valueInUse == TQ_DELETE_TOKEN) {
-        pSHead->ssize = sizeof(TqSerializedHead);
+      if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) {
+        pSHead->ssize = sizeof(STqSerializedHead);
       } else {
         pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
       }
@@ -343,10 +330,10 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
       ASSERT(nBytes == pSHead->ssize);
     }
 
-    if(pNode->handle.valueInTxn) {
+    if (pNode->handle.valueInTxn) {
       pSHead->action = TQ_ACTION_INTXN;
-      if(pNode->handle.valueInTxn == TQ_DELETE_TOKEN) {
-        pSHead->ssize = sizeof(TqSerializedHead);
+      if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) {
+        pSHead->ssize = sizeof(STqSerializedHead);
       } else {
         pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
       }
@@ -357,42 +344,39 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
     pNode->handle.offset = offset;
     offset += nBytes;
 
-    //write idx file
-    //TODO: endian check and convert
+    // write idx file
+    // TODO: endian check and convert
     *(bufPtr++) = pNode->handle.key;
     *(bufPtr++) = pNode->handle.offset;
     *(bufPtr++) = (int64_t)nBytes;
     idxBuf.head.writeOffset += TQ_IDX_SIZE;
 
-    if(idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
+    if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
       nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
-      //TODO: handle error with tfile
+      // TODO: handle error with tfile
       ASSERT(nBytes == TQ_PAGE_SIZE);
       memset(&idxBuf, 0, TQ_PAGE_SIZE);
       idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
       bufPtr = (int64_t*)&idxBuf.buffer;
     }
-    //remove from unpersist list
+    // remove from unpersist list
     pHead->unpersistNext = pNode->unpersistNext;
     pHead->unpersistNext->unpersistPrev = pHead;
     pNode->unpersistPrev = pNode->unpersistNext = NULL;
     pNode = pHead->unpersistNext;
 
-    //remove from bucket
-    if(pNode->handle.valueInUse == TQ_DELETE_TOKEN &&
-        pNode->handle.valueInTxn == NULL
-        ) {
+    // remove from bucket
+    if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) {
       int          bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
       STqMetaList* pBucketHead = pMeta->bucket[bucketKey];
-      if(pBucketHead == pNode) {
+      if (pBucketHead == pNode) {
         pMeta->bucket[bucketKey] = pNode->next;
       } else {
         STqMetaList* pBucketNode = pBucketHead;
-        while(pBucketNode->next != NULL
-            && pBucketNode->next != pNode) {
+        while (pBucketNode->next != NULL && pBucketNode->next != pNode) {
           pBucketNode = pBucketNode->next;
         }
-        //impossible for pBucket->next == NULL
+        // impossible for pBucket->next == NULL
         ASSERT(pBucketNode->next == pNode);
         pBucketNode->next = pNode->next;
       }
@@ -400,15 +384,15 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
     }
   }
 
-  //write left bytes
+  // write left bytes
   free(pSHead);
-  //TODO: write new version in tfile
-  if((char*)bufPtr != idxBuf.buffer) {
+  // TODO: write new version in tfile
+  if ((char*)bufPtr != idxBuf.buffer) {
     int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
-    //TODO: handle error in tfile
+    // TODO: handle error in tfile
     ASSERT(nBytes == idxBuf.head.writeOffset);
   }
-  //TODO: using fsync in tfile
+  // TODO: using fsync in tfile
   fsync(pMeta->idxFd);
   fsync(pMeta->fileFd);
   return 0;
@@ -417,29 +401,28 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
 static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) {
   int64_t      bucketKey = key & TQ_BUCKET_MASK;
   STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while(pNode) {
-    if(pNode->handle.key == key) {
-      //TODO: think about thread safety
-      if(pNode->handle.valueInUse
-          && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
+  while (pNode) {
+    if (pNode->handle.key == key) {
+      // TODO: think about thread safety
+      if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
         pMeta->pDeleter(pNode->handle.valueInUse);
       }
-      //change pointer ownership
+      // change pointer ownership
       pNode->handle.valueInUse = value;
       return 0;
     } else {
       pNode = pNode->next;
     }
   }
-  STqMetaList *pNewNode = malloc(sizeof(STqMetaList));
-  if(pNewNode == NULL) {
-    //TODO: memory error
+  STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
+  if (pNewNode == NULL) {
+    // TODO: memory error
     return -1;
   }
   memset(pNewNode, 0, sizeof(STqMetaList));
   pNewNode->handle.key = key;
   pNewNode->handle.valueInUse = value;
-  //put into unpersist list
+  // put into unpersist list
   pNewNode->unpersistPrev = pMeta->unpersistHead;
   pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
   pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode;
@@ -450,10 +433,9 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu
 void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
   int64_t      bucketKey = key & TQ_BUCKET_MASK;
   STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while(pNode) {
-    if(pNode->handle.key == key) {
-      if(pNode->handle.valueInUse != NULL
-          && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
+  while (pNode) {
+    if (pNode->handle.key == key) {
+      if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
         return pNode->handle.valueInUse;
       } else {
         return NULL;
@@ -468,10 +450,9 @@ void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
 void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
   int64_t      bucketKey = key & TQ_BUCKET_MASK;
   STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while(pNode) {
-    if(pNode->handle.key == key) {
-      if(pNode->handle.valueInUse != NULL
-          && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
+  while (pNode) {
+    if (pNode->handle.key == key) {
+      if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
         tqLinkUnpersist(pMeta, pNode);
         return pNode->handle.valueInUse;
       } else {
@@ -487,14 +468,14 @@ void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
 static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) {
   int64_t      bucketKey = key & TQ_BUCKET_MASK;
   STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while(pNode) {
-    if(pNode->handle.key == key) {
-      //TODO: think about thread safety
-      if(pNode->handle.valueInTxn) {
-        if(TqDupIntxnReject(pMeta->tqConfigFlag)) {
+  while (pNode) {
+    if (pNode->handle.key == key) {
+      // TODO: think about thread safety
+      if (pNode->handle.valueInTxn) {
+        if (TqDupIntxnReject(pMeta->tqConfigFlag)) {
           return -2;
         }
-        if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
+        if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
           pMeta->pDeleter(pNode->handle.valueInTxn);
         }
       }
@@ -505,9 +486,9 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
       pNode = pNode->next;
     }
   }
-  STqMetaList *pNewNode = malloc(sizeof(STqMetaList));
-  if(pNewNode == NULL) {
-    //TODO: memory error
+  STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
+  if (pNewNode == NULL) {
+    // TODO: memory error
     return -1;
   }
   memset(pNewNode, 0, sizeof(STqMetaList));
@@ -519,14 +500,12 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
   return 0;
 }
 
-int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) {
-  return tqHandlePutImpl(pMeta, key, value);
-}
+int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); }
 
 int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) {
-  void *vmem = malloc(vsize);
-  if(vmem == NULL) {
-    //TODO: memory error
+  void* vmem = malloc(vsize);
+  if (vmem == NULL) {
+    // TODO: memory error
     return -1;
   }
   memcpy(vmem, value, vsize);
@@ -536,10 +515,9 @@ int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vs
 static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
   int64_t      bucketKey = key & TQ_BUCKET_MASK;
   STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while(pNode) {
-    if(pNode->handle.key == key) {
-      if(pNode->handle.valueInTxn != NULL
-          && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
+  while (pNode) {
+    if (pNode->handle.key == key) {
+      if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
         return pNode->handle.valueInTxn;
       } else {
         return NULL;
@@ -554,13 +532,12 @@ static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
 int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
   int64_t      bucketKey = key & TQ_BUCKET_MASK;
   STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while(pNode) {
-    if(pNode->handle.key == key) {
-      if(pNode->handle.valueInTxn == NULL) {
+  while (pNode) {
+    if (pNode->handle.key == key) {
+      if (pNode->handle.valueInTxn == NULL) {
         return -1;
       }
-      if(pNode->handle.valueInUse
-          && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
+      if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
         pMeta->pDeleter(pNode->handle.valueInUse);
       }
       pNode->handle.valueInUse = pNode->handle.valueInTxn;
@@ -577,10 +554,10 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
 int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
   int64_t      bucketKey = key & TQ_BUCKET_MASK;
   STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while(pNode) {
-    if(pNode->handle.key == key) {
-      if(pNode->handle.valueInTxn) {
-        if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
+  while (pNode) {
+    if (pNode->handle.key == key) {
+      if (pNode->handle.valueInTxn) {
+        if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
           pMeta->pDeleter(pNode->handle.valueInTxn);
         }
         pNode->handle.valueInTxn = NULL;
@@ -598,11 +575,12 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
 int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
   int64_t      bucketKey = key & TQ_BUCKET_MASK;
   STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while(pNode) {
-    if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-      if(pNode->handle.valueInTxn) {
+  while (pNode) {
+    if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
+      if (pNode->handle.valueInTxn) {
         pMeta->pDeleter(pNode->handle.valueInTxn);
       }
+
       pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
       tqLinkUnpersist(pMeta, pNode);
       return 0;
@@ -610,11 +588,9 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
       pNode = pNode->next;
     }
   }
-  //no such key
+  // no such key
   return -1;
 }
 
-//TODO: clean deleted idx and data from persistent file
-int32_t tqStoreCompact(STqMetaStore *pMeta) {
-  return 0;
-}
+// TODO: clean deleted idx and data from persistent file
+int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; }

source/dnode/vnode/tq/test/tqMetaTest.cpp

@@ -9,17 +9,17 @@ struct Foo {
   int32_t a;
 };
 
-int FooSerializer(const void* pObj, TqSerializedHead** ppHead) {
+int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
   Foo* foo = (Foo*) pObj;
-  if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(TqSerializedHead) + sizeof(int32_t)) {
-    *ppHead = (TqSerializedHead*)realloc(*ppHead, sizeof(TqSerializedHead) + sizeof(int32_t));
-    (*ppHead)->ssize = sizeof(TqSerializedHead) + sizeof(int32_t);
+  if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
+    *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t));
+    (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t);
   }
   *(int32_t*)(*ppHead)->content = foo->a;
   return (*ppHead)->ssize;
 }
 
-const void* FooDeserializer(const TqSerializedHead* pHead, void** ppObj) {
+const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
   if(*ppObj == NULL) {
     *ppObj = realloc(*ppObj, sizeof(int32_t));
   }

source/libs/parser/inc/parserInt.h

@@ -53,6 +53,15 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t
  */
 int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg, int32_t msgLen);
 
+/**
+ * validate the ddl ast, and convert the ast to the corresponding message format
+ * @param pSqlInfo
+ * @param output
+ * @param type
+ * @return
+ */
+int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen);
+
 /**
  * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause.
  * @param pNode

source/libs/parser/inc/parserUtil.h

@@ -45,6 +45,8 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo);
 SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index);
 
 int32_t parserValidateIdToken(SToken* pToken);
+int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf);
+
 int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
 int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo,  const char* sourceStr);
 
@@ -63,6 +65,8 @@ int32_t    getExprFunctionId(SExprInfo *pExprInfo);
 
 STableMeta* tableMetaDup(const STableMeta* pTableMeta);
 
+bool isDclSqlStatement(SSqlInfo* pSqlInfo);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/parser/src/astValidate.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <function.h>
 #include "astGenerator.h"
 #include "function.h"
 #include "parserInt.h"
@@ -23,6 +22,7 @@
 #include "tglobal.h"
 #include "tmsgtype.h"
 #include "ttime.h"
+#include "astToMsg.h"
 
 #define TSQL_TBNAME_L "tbname"
 #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
@@ -3636,11 +3636,14 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf)
 }
 
 int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msgBuf, int32_t msgBufLen) {
-  //1. if it is a query, get the meta info and continue.
   assert(pCatalog != NULL && pInfo != NULL);
   int32_t code = 0;
-#if 0
+
+  SMsgBuf m = {.buf = msgBuf, .len = msgBufLen};
+  SMsgBuf *pMsgBuf = &m;
+
   switch (pInfo->type) {
+#if 0
     case TSDB_SQL_DROP_TABLE:
     case TSDB_SQL_DROP_USER:
     case TSDB_SQL_DROP_ACCT:
@@ -3651,14 +3654,14 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
 
       SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0);
       if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (parserValidateIdToken(pzName) != TSDB_CODE_SUCCESS)) {
-        return setInvalidOperatorMsg(pMsgBuf, msg2);
+        return buildInvalidOperationMsg(pMsgBuf, msg2);
       }
 
       if (pInfo->type == TSDB_SQL_DROP_DB) {
         assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1);
         code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName);
         if (code != TSDB_CODE_SUCCESS) {
-          return setInvalidOperatorMsg(pMsgBuf, msg2);
+          return buildInvalidOperationMsg(pMsgBuf, msg2);
         }
 
       } else if (pInfo->type == TSDB_SQL_DROP_TABLE) {
@@ -3675,7 +3678,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
         strncpy(pCmd->payload, pzName->z, pzName->n);
       } else {  // drop user/account
         if (pzName->n >= TSDB_USER_LEN) {
-          return setInvalidOperatorMsg(pMsgBuf, msg3);
+          return buildInvalidOperationMsg(pMsgBuf, msg3);
         }
 
         strncpy(pCmd->payload, pzName->z, pzName->n);
@@ -3689,12 +3692,12 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
 
       if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg);
+        return buildInvalidOperationMsg(pMsgBuf, msg);
       }
 
       int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken);
       if (ret != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg);
+        return buildInvalidOperationMsg(pMsgBuf, msg);
       }
 
       break;
@@ -3729,19 +3732,19 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
 
       SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt);
       if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) {
-        return setInvalidOperatorMsg(pMsgBuf, msg2);
+        return buildInvalidOperationMsg(pMsgBuf, msg2);
       }
 
       char buf[TSDB_DB_NAME_LEN] = {0};
       SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf));
 
       if (tscValidateName(&token) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg1);
+        return buildInvalidOperationMsg(pMsgBuf, msg1);
       }
 
       int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token);
       if (ret != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg2);
+        return buildInvalidOperationMsg(pMsgBuf, msg2);
       }
 
       if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) {
@@ -3755,7 +3758,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       const char* msg = "invalid host name (ip address)";
 
       if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) {
-        return setInvalidOperatorMsg(pMsgBuf, msg);
+        return buildInvalidOperationMsg(pMsgBuf, msg);
       }
 
       SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0);
@@ -3779,11 +3782,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       }
 
       if (pName->n >= TSDB_USER_LEN) {
-        return setInvalidOperatorMsg(pMsgBuf, msg3);
+        return buildInvalidOperationMsg(pMsgBuf, msg3);
       }
 
       if (tscValidateName(pName) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg2);
+        return buildInvalidOperationMsg(pMsgBuf, msg2);
       }
 
       SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt;
@@ -3793,7 +3796,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
         } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) {
         } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) {
         } else {
-          return setInvalidOperatorMsg(pMsgBuf, msg1);
+          return buildInvalidOperationMsg(pMsgBuf, msg1);
         }
       }
 
@@ -3805,7 +3808,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
 
       SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
       if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg1);
+        return buildInvalidOperationMsg(pMsgBuf, msg1);
       }
       // additional msg has been attached already
       code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql);
@@ -3821,7 +3824,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
 
       SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
       if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg1);
+        return buildInvalidOperationMsg(pMsgBuf, msg1);
       }
 
       code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql);
@@ -3836,11 +3839,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
 
       SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
       if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg1);
+        return buildInvalidOperationMsg(pMsgBuf, msg1);
       }
 
       if (pToken->n > TSDB_DB_NAME_LEN) {
-        return setInvalidOperatorMsg(pMsgBuf, msg1);
+        return buildInvalidOperationMsg(pMsgBuf, msg1);
       }
       return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken);
     }
@@ -3853,7 +3856,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
 
       /* validate the parameter names and options */
       if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg2);
+        return buildInvalidOperationMsg(pMsgBuf, msg2);
       }
 
       char* pMsg = pCmd->payload;
@@ -3867,7 +3870,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       strncpy(pCfg->ep, t0->z, t0->n);
 
       if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg3);
+        return buildInvalidOperationMsg(pMsgBuf, msg3);
       }
 
       strncpy(pCfg->config, t1->z, t1->n);
@@ -3882,65 +3885,13 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       break;
     }
 
-    case TSDB_SQL_CREATE_USER:
-    case TSDB_SQL_ALTER_USER: {
-      const char* msg2 = "invalid user/account name";
-      const char* msg3 = "name too long";
-      const char* msg5 = "invalid user rights";
-      const char* msg7 = "not support options";
-
-      pCmd->command = pInfo->type;
-
-      SUserInfo* pUser = &pInfo->pMiscInfo->user;
-      SToken* pName = &pUser->user;
-      SToken* pPwd = &pUser->passwd;
-
-      if (pName->n >= TSDB_USER_LEN) {
-        return setInvalidOperatorMsg(pMsgBuf, msg3);
-      }
-
-      if (tscValidateName(pName) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg2);
-      }
-
-      if (pCmd->command == TSDB_SQL_CREATE_USER) {
-        if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) {
-          return TSDB_CODE_TSC_INVALID_OPERATION;
-        }
-      } else {
-        if (pUser->type == TSDB_ALTER_USER_PASSWD) {
-          if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-        } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) {
-          assert(pPwd->type == TSDB_DATA_TYPE_NULL);
-
-          SToken* pPrivilege = &pUser->privilege;
-
-          if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) {
-            pCmd->count = 1;
-          } else if (strncasecmp(pPrivilege->z, "read", 4) == 0 && pPrivilege->n == 4) {
-            pCmd->count = 2;
-          } else if (strncasecmp(pPrivilege->z, "write", 5) == 0 && pPrivilege->n == 5) {
-            pCmd->count = 3;
-          } else {
-            return setInvalidOperatorMsg(pMsgBuf, msg5);
-          }
-        } else {
-          return setInvalidOperatorMsg(pMsgBuf, msg7);
-        }
-      }
-
-      break;
-    }
-
     case TSDB_SQL_CFG_LOCAL: {
       SMiscInfo  *pMiscInfo = pInfo->pMiscInfo;
       const char *msg = "invalid configure options or values";
 
       // validate the parameter names and options
       if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg);
+        return buildInvalidOperationMsg(pMsgBuf, msg);
       }
 
       int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a);
@@ -3995,7 +3946,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
         tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size);
 
         if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) {
-          return setInvalidOperatorMsg(pMsgBuf, msg1);
+          return buildInvalidOperationMsg(pMsgBuf, msg1);
         }
 
 //        normalizeSqlNode(pSqlNode); // normalize the column name in each function
@@ -4061,21 +4012,22 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1);
       code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName);
       if (code != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg1);
+        return buildInvalidOperationMsg(pMsgBuf, msg1);
       }
       break;
     }
     case TSDB_SQL_COMPACT_VNODE:{
       const char* msg = "invalid compact";
       if (setCompactVnodeInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) {
-        return setInvalidOperatorMsg(pMsgBuf, msg);
+        return buildInvalidOperationMsg(pMsgBuf, msg);
       }
       break;
     }
+    #endif
     default:
-      return setInvalidOperatorMsg(pMsgBuf, "not support sql expression");
+      return buildInvalidOperationMsg(pMsgBuf, "not support sql expression");
   }
-#endif
+
 
   SCatalogReq req = {0};
   SMetaData data = {0};
@@ -4114,5 +4066,159 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
   }
 
 
+  return code;
+}
+
+// todo remove it
+static int32_t setShowInfo(struct SSqlInfo* pInfo, void** output, int32_t* msgLen, SMsgBuf* pMsgBuf) {
+  const char* msg1 = "invalid name";
+  const char* msg2 = "wildcard string should be less than %d characters";
+  const char* msg3 = "database name too long";
+  const char* msg4 = "pattern is invalid";
+  const char* msg5 = "database name is empty";
+  const char* msg6 = "pattern string is empty";
+
+  /*
+   * database prefix in pInfo->pMiscInfo->a[0]
+   * wildcard in like clause in pInfo->pMiscInfo->a[1]
+   */
+  SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt;
+  int16_t    showType = pShowInfo->showType;
+  if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_VGROUP) {
+    SToken* pDbPrefixToken = &pShowInfo->prefix;
+    if (pDbPrefixToken->type != 0) {
+      if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) {  // db name is too long
+        return buildInvalidOperationMsg(pMsgBuf, msg3);
+      }
+
+      if (pDbPrefixToken->n <= 0) {
+        return buildInvalidOperationMsg(pMsgBuf, msg5);
+      }
+
+      if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) {
+        return buildInvalidOperationMsg(pMsgBuf, msg1);
+      }
+
+      //      int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken);
+      //      if (ret != TSDB_CODE_SUCCESS) {
+      //        return buildInvalidOperationMsg(pMsgBuf, msg1);
+      //      }
+    }
+
+    // show table/stable like 'xxxx', set the like pattern for show tables
+    SToken* pPattern = &pShowInfo->pattern;
+    if (pPattern->type != 0) {
+      if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) {
+        return buildInvalidOperationMsg(pMsgBuf, msg4);
+      }
+
+      pPattern->n = strdequote(pPattern->z);
+      if (pPattern->n <= 0) {
+        return buildInvalidOperationMsg(pMsgBuf, msg6);
+      }
+
+      if (pPattern->n > tsMaxWildCardsLen) {
+        char tmp[64] = {0};
+        sprintf(tmp, msg2, tsMaxWildCardsLen);
+        return buildInvalidOperationMsg(pMsgBuf, tmp);
+      }
+    }
+  } else if (showType == TSDB_MGMT_TABLE_VNODES) {
+    if (pShowInfo->prefix.type == 0) {
+      return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep");
+    }
+
+    if (pShowInfo->prefix.type == TK_STRING) {
+      pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z);
+    }
+  }
+
+  SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg));
+  pShowMsg->type = pShowInfo->showType;
+
+  if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
+    SToken* pPattern = &pShowInfo->pattern;
+    if (pPattern->type > 0) {  // only show tables support wildcard query
+      strncpy(pShowMsg->payload, pPattern->z, pPattern->n);
+      pShowMsg->payloadLen = htons(pPattern->n);
+    }
+  } else {
+    SToken* pEpAddr = &pShowInfo->prefix;
+    assert(pEpAddr->n > 0 && pEpAddr->type > 0);
+
+    strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n);
+    pShowMsg->payloadLen = htons(pEpAddr->n);
+  }
+
+  *output = pShowMsg;
+  *msgLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) {
+  int32_t code = 0;
+
+  SMsgBuf m = {.buf = msgBuf, .len = msgBufLen};
+  SMsgBuf *pMsgBuf = &m;
+
+  *type = pInfo->type;
+
+  switch (pInfo->type) {
+    case TSDB_SQL_CREATE_USER:
+    case TSDB_SQL_ALTER_USER: {
+      const char* msg1 = "not support options";
+      const char* msg2 = "invalid user/account name";
+      const char* msg3 = "name too long";
+      const char* msg4 = "invalid user rights";
+
+      SUserInfo* pUser = &pInfo->pMiscInfo->user;
+      SToken*    pName = &pUser->user;
+      SToken*    pPwd = &pUser->passwd;
+
+      if (pName->n >= TSDB_USER_LEN) {
+        return buildInvalidOperationMsg(pMsgBuf, msg3);
+      }
+
+      if (parserValidateIdToken(pName) != TSDB_CODE_SUCCESS) {
+        return buildInvalidOperationMsg(pMsgBuf, msg2);
+      }
+
+      if (pInfo->type == TSDB_SQL_CREATE_USER) {
+        if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+      } else {
+        if (pUser->type == TSDB_ALTER_USER_PASSWD) {
+          if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) {
+            return TSDB_CODE_TSC_INVALID_OPERATION;
+          }
+        } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) {
+          assert(pPwd->type == TSDB_DATA_TYPE_NULL);
+
+          SToken* pPrivilege = &pUser->privilege;
+          if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) {
+            //            pCmd->count = 1;
+          } else if (strncasecmp(pPrivilege->z, "normal", 4) == 0 && pPrivilege->n == 4) {
+            //            pCmd->count = 2;
+          } else {
+            return buildInvalidOperationMsg(pMsgBuf, msg4);
+          }
+        } else {
+          return buildInvalidOperationMsg(pMsgBuf, msg1);
+        }
+      }
+
+      *output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen);
+      break;
+    }
+    
+    case TSDB_SQL_SHOW: {
+      code = setShowInfo(pInfo, output, outputLen, pMsgBuf);
+      break;
+    }
+    default:
+      break;
+  }
+
   return code;
 }

source/libs/parser/src/parser.c

@@ -31,25 +31,35 @@ bool qIsInsertSql(const char* pStr, size_t length) {
   } while (1);
 }
 
-int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen) {
-  *pQueryInfo = calloc(1, sizeof(SQueryStmtInfo));
-  if (*pQueryInfo == NULL) {
-    return TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code.
+int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) {
+  SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo));
+  if (pQueryInfo == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code.
+    return terrno;
   }
 
   SSqlInfo info = doGenerateAST(pStr);
   if (!info.valid) {
     strncpy(msg, info.msg, msgLen);
-    return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
+    terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
+    return terrno;
   }
 
+  if (isDclSqlStatement(&info)) {
+    int32_t code = qParserValidateDclSqlNode(&info, id, pOutput, outputLen, type, msg, msgLen);
+    if (code == TSDB_CODE_SUCCESS) {
+      // do nothing
+    }
+  } else {
     struct SCatalog* pCatalog = NULL;
     int32_t code = catalogGetHandle(NULL, &pCatalog);
-  if (code) {
-    return code;
+    code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen);
+    if (code == TSDB_CODE_SUCCESS) {
+      *pOutput = pQueryInfo;
+    }
   }
 
-  return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen);
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {

source/libs/parser/src/parserUtil.c

@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include "parserUtil.h"
+
 
 #include "taosmsg.h"
 #include "parser.h"
@@ -22,6 +22,8 @@
 #include "thash.h"
 #include "tbuffer.h"
 #include "parserInt.h"
+#include "parserUtil.h"
+#include "tmsgtype.h"
 #include "queryInfoUtil.h"
 #include "function.h"
 
@@ -97,6 +99,29 @@ int32_t parserValidateIdToken(SToken* pToken) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) {
+  const char* msg1 = "password can not be empty";
+  const char* msg2 = "name or password too long";
+  const char* msg3 = "password needs single quote marks enclosed";
+
+  if (pToken->type != TK_STRING) {
+    return buildInvalidOperationMsg(pMsgBuf, msg3);
+  }
+
+  strdequote(pToken->z);
+
+  pToken->n = (uint32_t)strtrim(pToken->z);  // trim space before and after passwords
+  if (pToken->n <= 0) {
+    return buildInvalidOperationMsg(pMsgBuf, msg1);
+  }
+
+  if (pToken->n >= TSDB_USET_PASSWORD_LEN) {
+    return buildInvalidOperationMsg(pMsgBuf, msg2);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) {
   strncpy(pBuf->buf, msg, pBuf->len);
   return TSDB_CODE_TSC_INVALID_OPERATION;
@@ -584,21 +609,6 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo) {
   return pFieldInfo->numOfOutput;
 }
 
-int32_t getFirstInvisibleFieldPos(SQueryStmtInfo* pQueryInfo) {
-  if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) {
-    return 0;
-  }
-
-  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
-    SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
-    if (!pField->visible) {
-      return i;
-    }
-  }
-
-  return pQueryInfo->fieldsInfo.numOfOutput;
-}
-
 SInternalField* appendFieldInfo(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) {
   assert(pFieldInfo != NULL);
   pFieldInfo->numOfOutput++;
@@ -1583,6 +1593,10 @@ uint32_t convertRelationalOperator(SToken *pToken) {
   }
 }
 
+bool isDclSqlStatement(SSqlInfo* pSqlInfo) {
+  return (pSqlInfo->type != TSDB_SQL_SELECT);
+}
+
 #if 0
 int32_t tscCreateQueryFromQueryInfo(SQueryStmtInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) {
   memset(pQueryAttr, 0, sizeof(SQueryAttr));

source/libs/parser/test/parserTests.cpp

@@ -29,6 +29,7 @@
 #include "taos.h"
 #include "tdef.h"
 #include "tvariant.h"
+#include "parserUtil.h"
 
 namespace {
 void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_t colId) {
@@ -702,3 +703,45 @@ TEST(testCase, function_Test6) {
   qParserClearupMetaRequestInfo(&req);
   destroySqlInfo(&info1);
 }
+
+TEST(testCase, show_user_Test) {
+  char    msg[128] = {0};
+  SMsgBuf buf;
+  buf.len = 128;
+  buf.buf = msg;
+
+  char sql1[] = "show users";
+  SSqlInfo info1 = doGenerateAST(sql1);
+  ASSERT_EQ(info1.valid, true);
+
+  void* output = NULL;
+  int32_t type = 0;
+  int32_t len = 0;
+  int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len);
+  ASSERT_EQ(code, 0);
+
+  // convert the show command to be the select query
+  // select name, privilege, create_time, account from information_schema.users;
+}
+
+TEST(testCase, create_user_Test) {
+  char    msg[128] = {0};
+  SMsgBuf buf;
+  buf.len = 128;
+  buf.buf = msg;
+
+  char sql[] = {"create user abc pass 'abc'"};
+
+  SSqlInfo info1 = doGenerateAST(sql);
+  ASSERT_EQ(info1.valid, true);
+
+  ASSERT_EQ(isDclSqlStatement(&info1), true);
+
+  void* output = NULL;
+  int32_t type = 0;
+  int32_t len = 0;
+  int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len);
+  ASSERT_EQ(code, 0);
+
+  destroySqlInfo(&info1);
+}
\ No newline at end of file

source/libs/planner/inc/plannerInt.h

@@ -99,6 +99,8 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
  */
 int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
 
+int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag);
+
 /**
  * Convert to physical plan to string to enable to print it out in the shell.
  * @param pPhyNode
@@ -111,7 +113,7 @@ int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str);
  * Destroy the query plan object.
  * @return
  */
-void* destroyQueryPlan(struct SQueryPlanNode* pQueryNode);
+void destroyQueryPlan(struct SQueryPlanNode* pQueryNode);
 
 /**
  * Destroy the physical plan.

source/libs/planner/src/logicPlan.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "function.h"
+#include "os.h"
+#include "parser.h"
+#include "plannerInt.h"
+
+typedef struct SFillEssInfo {
+  int32_t  fillType;  // fill type
+  int64_t *val;       // fill value
+} SFillEssInfo;
+
+typedef struct SJoinCond {
+  bool     tagExists; // denote if tag condition exists or not
+  SColumn *tagCond[2];
+  SColumn *colCond[2];
+} SJoinCond;
+
+static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo);
+static void doDestroyQueryNode(SQueryPlanNode* pQueryNode);
+
+int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len);
+int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) {
+  return 0;
+}
+
+int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) {
+  SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo);
+  assert(taosArrayGetSize(upstream) == 1);
+
+  *pQueryNode = taosArrayGetP(upstream, 0);
+
+  taosArrayDestroy(upstream);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) {
+  return 0;
+}
+
+void destroyQueryPlan(SQueryPlanNode* pQueryNode) {
+  if (pQueryNode == NULL) {
+    return;
+  }
+
+  doDestroyQueryNode(pQueryNode);
+}
+
+//======================================================================================================================
+
+static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev,
+                                   SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) {
+  SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode));
+
+  pNode->info.type = type;
+  pNode->info.name = strdup(name);
+
+  pNode->numOfExpr = numOfOutput;
+  pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES);
+
+  for(int32_t i = 0; i < numOfOutput; ++i) {
+    taosArrayPush(pNode->pExpr, &pExpr[i]);
+  }
+
+  pNode->pChildren = taosArrayInit(4, POINTER_BYTES);
+  for(int32_t i = 0; i < numOfPrev; ++i) {
+    taosArrayPush(pNode->pChildren, &prev[i]);
+  }
+
+  switch(type) {
+    case QNODE_TAGSCAN:
+    case QNODE_TABLESCAN: {
+      SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo));
+      memcpy(info, pExtInfo, sizeof(SQueryTableInfo));
+      info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName);
+      pNode->pExtInfo = info;
+      break;
+    }
+
+    case QNODE_TIMEWINDOW: {
+      SInterval* pInterval = calloc(1, sizeof(SInterval));
+      pNode->pExtInfo = pInterval;
+      memcpy(pInterval, pExtInfo, sizeof(SInterval));
+      break;
+    }
+
+    case QNODE_STATEWINDOW: {
+      SColumn* psw = calloc(1, sizeof(SColumn));
+      pNode->pExtInfo = psw;
+      memcpy(psw, pExtInfo, sizeof(SColumn));
+      break;
+    }
+
+    case QNODE_SESSIONWINDOW: {
+      SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow));
+      pNode->pExtInfo = pSessionWindow;
+      memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow));
+      break;
+    }
+
+    case QNODE_GROUPBY: {
+      SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo;
+
+      SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr));
+      pGroupbyExpr->groupbyTag = p->groupbyTag;
+      pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo);
+
+      pNode->pExtInfo = pGroupbyExpr;
+      break;
+    }
+
+    case QNODE_FILL: { // todo !!
+      pNode->pExtInfo = pExtInfo;
+      break;
+    }
+
+    case QNODE_LIMIT: {
+      pNode->pExtInfo = calloc(1, sizeof(SLimit));
+      memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit));
+      break;
+    }
+
+    case QNODE_SORT: {
+      pNode->pExtInfo = taosArrayDup(pExtInfo);
+      break;
+    }
+
+    default:
+      break;
+  }
+  
+  return pNode;
+}
+
+static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info,
+                                        SArray* pExprs, SArray* tableCols) {
+  if (pQueryInfo->info.onlyTagQuery) {
+    int32_t num = (int32_t) taosArrayGetSize(pExprs);
+    SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info);
+
+    if (pQueryInfo->info.distinct) {
+      pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL);
+    }
+
+    return pNode;
+  }
+
+  SQueryPlanNode*  pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info);
+
+  if (pQueryInfo->info.projectionQuery) {
+    int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs);
+    pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL);
+  } else {
+    STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0);
+
+    // table source column projection, generate the projection expr
+    int32_t     numOfCols = (int32_t) taosArrayGetSize(tableCols);
+    SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES);
+    for (int32_t i = 0; i < numOfCols; ++i) {
+      SColumn* pCol = taosArrayGetP(tableCols, i);
+
+      SSourceParam param = {0};
+      addIntoSourceParam(&param, NULL, pCol);
+      SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name);
+      SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", &param, &s, 0);
+      pExpr[i] = p;
+    }
+
+    pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL);
+    tfree(pExpr);
+  }
+
+  return pNode;
+}
+
+static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) {
+  // group by column not by tag
+  size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo);
+
+  // check for aggregation
+  int32_t level = getExprFunctionLevel(pQueryInfo);
+
+  for(int32_t i = level - 1; i >= 0; --i) {
+    SArray* p = pQueryInfo->exprList[i];
+    size_t  num = taosArrayGetSize(p);
+
+    bool aggregateFunc = false;
+    for(int32_t j = 0; j < num; ++j) {
+      SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0);
+      if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) {
+        continue;
+      }
+
+      aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName);
+      if (aggregateFunc) {
+        break;
+      }
+    }
+
+    if (aggregateFunc) {
+      if (pQueryInfo->interval.interval > 0) {
+        pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval);
+      } else if (pQueryInfo->sessionWindow.gap > 0) {
+        pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow);
+      } else if (pQueryInfo->stateWindow.col.info.colId > 0) {
+        pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow);
+      } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) {
+          pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr);
+      } else {
+        pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL);
+      }
+    } else {
+      pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL);
+    }
+  }
+
+  if (pQueryInfo->havingFieldNum > 0) {
+//    int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1);
+//    pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL);
+  }
+
+  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
+    SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo));
+    pInfo->fillType = pQueryInfo->fillType;
+    pInfo->val      = calloc(pNode->numOfExpr, sizeof(int64_t));
+    memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr);
+
+    SArray* p = pQueryInfo->exprList[0];  // top expression in select clause
+    pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo);
+  }
+
+  if (pQueryInfo->order != NULL) {
+    SArray* pList = pQueryInfo->exprList[0];
+    pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order);
+  }
+
+  if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) {
+    pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit);
+  }
+
+  return pNode;
+}
+
+static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs,
+                                                SArray* tableCols) {
+  char name[TSDB_TABLE_FNAME_LEN] = {0};
+  tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN);
+
+  SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
+
+  // handle the only tag query
+  SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols);
+  if (pQueryInfo->info.onlyTagQuery) {
+    tfree(info.tableName);
+    return pNode;
+  }
+
+  SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info);
+  tfree(info.tableName);
+  return pNode1;
+}
+
+static bool isAllAggExpr(SArray* pList) {
+  assert(pList != NULL);
+
+  for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) {
+    SExprInfo* p = taosArrayGetP(pList, k);
+    if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) {
+  SArray* upstream = NULL;
+
+  if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) {  // subquery in the from clause
+    upstream = taosArrayInit(4, POINTER_BYTES);
+
+    size_t size = taosArrayGetSize(pQueryInfo->pUpstream);
+    for(int32_t i = 0; i < size; ++i) {
+      SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i);
+      SArray* p = createQueryPlanImpl(pq);
+      taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p));
+    }
+  }
+
+  if (pQueryInfo->numOfTables > 1) {  // it is a join query
+    // 1. separate the select clause according to table
+    taosArrayDestroy(upstream);
+    upstream = taosArrayInit(5, POINTER_BYTES);
+
+    for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+      STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i];
+      uint64_t        uid = pTableMetaInfo->pTableMeta->uid;
+
+      SArray* exprList = taosArrayInit(4, POINTER_BYTES);
+      if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) {
+        terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+//        dropAllExprInfo(exprList);
+        exit(-1);
+      }
+
+      // 2. create the query execution node
+      char name[TSDB_TABLE_FNAME_LEN] = {0};
+      tNameExtractFullName(&pTableMetaInfo->name, name);
+      SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
+
+      // 3. get the required table column list
+      SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn));
+      columnListCopy(tableColumnList, pQueryInfo->colList, uid);
+
+      // 4. add the projection query node
+      SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList);
+      columnListDestroy(tableColumnList);
+//      dropAllExprInfo(exprList);
+      taosArrayPush(upstream, &pNode);
+    }
+
+    // 3. add the join node here
+    SQueryTableInfo info = {0};
+    int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]);
+    SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables,
+                                        pQueryInfo->exprList[0]->pData, num, NULL);
+
+    // 4. add the aggregation or projection execution node
+    pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info);
+    upstream = taosArrayInit(5, POINTER_BYTES);
+    taosArrayPush(upstream, &pNode);
+  } else { // only one table, normal query process
+    STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
+    SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList);
+    upstream = taosArrayInit(5, POINTER_BYTES);
+    taosArrayPush(upstream, &pNode);
+  }
+
+  return upstream;
+}
+
+static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
+  tfree(pQueryNode->pExtInfo);
+  tfree(pQueryNode->pSchema);
+  tfree(pQueryNode->info.name);
+//  dropAllExprInfo(pQueryNode->pExpr);
+
+  if (pQueryNode->pChildren != NULL) {
+    int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren);
+    for(int32_t i = 0; i < size; ++i) {
+      SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i);
+      doDestroyQueryNode(p);
+    }
+
+    taosArrayDestroy(pQueryNode->pChildren);
+  }
+
+  tfree(pQueryNode);
+}
+
+static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
+  if (level > 0) {
+    sprintf(buf + totalLen, "%*c", level, ' ');
+    totalLen += level;
+  }
+
+  int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name);
+  int32_t len = len1 + totalLen;
+
+  switch(pQueryNode->info.type) {
+    case QNODE_TABLESCAN: {
+      SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo;
+      len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid,
+                     pInfo->window.skey, pInfo->window.ekey);
+      assert(len1 > 0);
+      len += len1;
+
+      for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
+        SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i);
+        len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId);
+
+        assert(len1 > 0);
+        len += len1;
+      }
+
+      len1 = sprintf(buf + len, "\n");
+      assert(len1 > 0);
+
+      len += len1;
+      break;
+    }
+
+    case QNODE_PROJECT: {
+      len1 = sprintf(buf + len, "cols:");
+      assert(len1 > 0);
+      len += len1;
+
+      len = printExprInfo(buf, pQueryNode, len);
+      len1 = sprintf(buf + len, ")");
+      len += len1;
+
+      // todo print filter info
+      len1 = sprintf(buf + len, " filters:(nil)\n");
+      len += len1;
+      break;
+    }
+
+    case QNODE_AGGREGATE: {
+      len = printExprInfo(buf, pQueryNode, len);
+      len1 = sprintf(buf + len, ")\n");
+      len += len1;
+
+      break;
+    }
+
+    case QNODE_TIMEWINDOW: {
+      len = printExprInfo(buf, pQueryNode, len);
+      len1 = sprintf(buf + len, ") ");
+      len += len1;
+
+      SInterval* pInterval = pQueryNode->pExtInfo;
+
+      // todo dynamic return the time precision
+      len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n",
+                     pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding,
+                     TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR);
+      len += len1;
+
+      break;
+    }
+
+    case QNODE_STATEWINDOW: {
+      len = printExprInfo(buf, pQueryNode, len);
+      len1 = sprintf(buf + len, ") ");
+      len += len1;
+
+      SColumn* pCol = pQueryNode->pExtInfo;
+      len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId);
+      len += len1;
+      break;
+    }
+
+    case QNODE_SESSIONWINDOW: {
+      len = printExprInfo(buf, pQueryNode, len);
+
+      len1 = sprintf(buf + len, ") ");
+      len += len1;
+
+      struct SSessionWindow* ps = pQueryNode->pExtInfo;
+      len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap);
+      len += len1;
+      break;
+    }
+
+    case QNODE_GROUPBY: {
+      len = printExprInfo(buf, pQueryNode, len);
+
+      SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo;
+      len1 = sprintf(buf + len, ") groupby_col: ");
+      len += len1;
+
+      for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) {
+        SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i);
+        len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId);
+        len += len1;
+      }
+
+      len += sprintf(buf + len, "\n");
+      break;
+    }
+
+    case QNODE_FILL: {
+      SFillEssInfo* pEssInfo = pQueryNode->pExtInfo;
+      len1 = sprintf(buf + len, "%d", pEssInfo->fillType);
+      len += len1;
+
+      if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) {
+        len1 = sprintf(buf + len, ", val:");
+        len += len1;
+
+        // todo get the correct fill data type
+        for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
+          len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]);
+          len += len1;
+
+          if (i < pQueryNode->numOfExpr - 1) {
+            len1 = sprintf(buf + len, ", ");
+            len += len1;
+          }
+        }
+      }
+
+      len1 = sprintf(buf + len, ")\n");
+      len += len1;
+      break;
+    }
+
+    case QNODE_LIMIT: {
+      SLimit* pVal = pQueryNode->pExtInfo;
+      len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset);
+      len += len1;
+      break;
+    }
+
+    case QNODE_DISTINCT:
+    case QNODE_TAGSCAN: {
+      len1 = sprintf(buf + len, "cols: ");
+      len += len1;
+
+      len = printExprInfo(buf, pQueryNode, len);
+
+      len1 = sprintf(buf + len, ")\n");
+      len += len1;
+
+      break;
+    }
+
+    case QNODE_SORT: {
+      len1 = sprintf(buf + len, "cols:");
+      len += len1;
+
+      SArray* pSort = pQueryNode->pExtInfo;
+      for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) {
+        SOrder* p = taosArrayGet(pSort, i);
+        len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC");
+
+        len += len1;
+      }
+
+      len1 = sprintf(buf + len, ")\n");
+      len += len1;
+      break;
+    }
+
+    case QNODE_JOIN: {
+      //  print join condition
+      len1 = sprintf(buf + len, ")\n");
+      len += len1;
+      break;
+    }
+  }
+
+  return len;
+}
+
+int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len) {
+  int32_t len1 = 0;
+
+  for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
+    SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
+
+    SSqlExpr* pExpr = &pExprInfo->base;
+    len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId);
+    assert(len1 > 0);
+
+    len += len1;
+    if (i < pQueryNode->numOfExpr - 1) {
+      len1 = sprintf(buf + len, ", ");
+      len += len1;
+    }
+  }
+
+  return len;
+}
+
+int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
+  int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen);
+
+  for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) {
+    SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i);
+    int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len);
+    len = len1;
+  }
+
+  return len;
+}
+
+int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) {
+  assert(pQueryNode);
+
+  *str = calloc(1, 4096);
+
+  int32_t len = sprintf(*str, "===== logic plan =====\n");
+  queryPlanToStringImpl(*str, pQueryNode, 0, len);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+SQueryPlanNode* queryPlanFromString() {
+  return NULL;
+}

source/libs/planner/src/physicalPlan.c

@@ -16,6 +16,9 @@
 #include "plannerInt.h"
 #include "parser.h"
 
+#define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan
+#define RECOVERY_CURRENT_SUBPLAN(cxt) cxt->pCurrentSubplan = _
+
 static const char* gOpName[] = {
   "Unknown",
 #define INCLUDE_AS_NAME
@@ -43,8 +46,56 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si
   toDataBlockSchema(pPlanNode, &(node->targetSchema));
 }
 
+static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) {
+  SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size);
+  node->uid = pTable->pMeta->pTableMeta->uid;
+  node->tableType = pTable->pMeta->pTableMeta->tableType;
+  return (SPhyNode*)node;
+}
+
+static SPhyNode* createPseudoScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) {
+  return initScanNode(pPlanNode, pTable, op, sizeof(SScanPhyNode));
+}
+
 static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) {
-  return initPhyNode(pPlanNode, OP_TagScan, sizeof(STagScanPhyNode));
+  SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo;
+  return createPseudoScanNode(pPlanNode, pTable, OP_TagScan);
+}
+
+static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
+  // todo
+  return MASTER_SCAN;
+}
+
+static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) {
+  STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode));
+  node->scanFlag = getScanFlag(pPlanNode, pTable);
+  node->window = pTable->window;
+  // todo tag cond
+  return (SPhyNode*)node;
+}
+
+static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
+  return createUserTableScanNode(pPlanNode, pTable, OP_TableScan);
+}
+
+static bool isSystemTable(SQueryTableInfo* pTable) {
+  // todo
+  return false;
+}
+
+static bool needSeqScan(SQueryPlanNode* pPlanNode) {
+  // todo
+  return false;
+}
+
+static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
+  if (isSystemTable(pTable)) {
+    return createPseudoScanNode(pPlanNode, pTable, OP_SystemTableScan);
+  } else if (needSeqScan(pPlanNode)) {
+    return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan);
+  }
+  return createUserTableScanNode(pPlanNode, pTable, OP_DataBlocksOptScan);
 }
 
 static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) {
@@ -58,53 +109,61 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) {
     if (NULL == pCxt->pCurrentSubplan->pChildern) {
       pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
     }
-    taosArrayPush(pCxt->pCurrentSubplan->pChildern, subplan);
+    taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan);
     subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
-    taosArrayPush(subplan->pParents, pCxt->pCurrentSubplan);
+    taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan);
+  }
+  SArray* currentLevel;
+  if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) {
+    currentLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
+    taosArrayPush(pCxt->pDag->pSubplans, &currentLevel);
+  } else {
+    currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level);
   }
+  taosArrayPush(currentLevel, &subplan);
   pCxt->pCurrentSubplan = subplan;
   return subplan;
 }
 
-static uint8_t getScanFlag(SQueryPlanNode* pPlanNode) {
-  // todo
-  return MASTER_SCAN;
-}
-
-static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
-  STableScanPhyNode* node = (STableScanPhyNode*)initPhyNode(pPlanNode, OP_TableScan, sizeof(STableScanPhyNode));
-  node->scan.uid = pTable->pMeta->pTableMeta->uid;
-  node->scan.tableType = pTable->pMeta->pTableMeta->tableType;
-  node->scanFlag = getScanFlag(pPlanNode);
-  node->window = pTable->window;
-  // todo tag cond
-}
-
 static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) {
-  // todo
+  epSet->inUse = 0; // todo
+  epSet->numOfEps = vg->numOfEps;
+  for (int8_t i = 0; i < vg->numOfEps; ++i) {
+    epSet->port[i] = vg->epAddr[i].port;
+    strcpy(epSet->fqdn[i], vg->epAddr[i].fqdn);
+  }
+  return;
 }
 
-static void splitSubplanBySTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
+static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
   SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList;
   for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) {
+    STORE_CURRENT_SUBPLAN(pCxt);
     SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN);
     vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet);
-    subplan->pNode = createTableScanNode(pCxt, pPlanNode, pTable);
-    // todo reset pCxt->pCurrentSubplan
+    subplan->pNode = createMultiTableScanNode(pPlanNode, pTable);
+    RECOVERY_CURRENT_SUBPLAN(pCxt);
   }
+  return pCxt->nextId.templateId++;
 }
 
-static SPhyNode* createExchangeNode() {
+static SPhyNode* createExchangeNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, uint64_t srcTemplateId) {
+  SExchangePhyNode* node = (SExchangePhyNode*)initPhyNode(pPlanNode, OP_Exchange, sizeof(SExchangePhyNode));
+  node->srcTemplateId = srcTemplateId;
+  return (SPhyNode*)node;
+}
 
+static bool needMultiNodeScan(SQueryTableInfo* pTable) {
+  // todo system table, for instance, user_tables
+  return (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType);
 }
 
-static SPhyNode* createScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
+static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
   SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo;
-  if (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType) {
-    splitSubplanBySTable(pCxt, pPlanNode, pTable);
-    return createExchangeNode(pCxt, pTable);
+  if (needMultiNodeScan(pTable)) {
+    return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable));
   }
-  return createTableScanNode(pCxt, pPlanNode, pTable);
+  return createSingleTableScanNode(pPlanNode, pTable);
 }
 
 static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
@@ -114,7 +173,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
       node = createTagScanNode(pPlanNode);
       break;
     case QNODE_TABLESCAN:
-      node = createScanNode(pCxt, pPlanNode);
+      node = createTableScanNode(pCxt, pPlanNode);
       break;
     default:
       assert(false);
@@ -133,6 +192,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
 
 static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) {
   SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE);
+  ++(pCxt->nextId.templateId);
   subplan->pNode = createPhyNode(pCxt, pRoot);
   SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
   taosArrayPush(l0, &subplan);
@@ -144,7 +204,8 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD
   SPlanContext context = {
     .pCatalog = pCatalog,
     .pDag = calloc(1, sizeof(SQueryDag)),
-    .pCurrentSubplan = NULL
+    .pCurrentSubplan = NULL,
+    .nextId = {0} // todo queryid
   };
   if (NULL == context.pDag) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;

source/libs/planner/src/planner.c

@@ -13,606 +13,31 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "function.h"
-#include "os.h"
 #include "parser.h"
 #include "plannerInt.h"
 
-typedef struct SFillEssInfo {
-  int32_t  fillType;  // fill type
-  int64_t *val;       // fill value
-} SFillEssInfo;
-
-typedef struct SJoinCond {
-  bool     tagExists; // denote if tag condition exists or not
-  SColumn *tagCond[2];
-  SColumn *colCond[2];
-} SJoinCond;
-
-static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo);
-static void doDestroyQueryNode(SQueryPlanNode* pQueryNode);
-
-int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len);
-int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) {
-  return 0;
-}
-
-int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) {
-  SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo);
-  assert(taosArrayGetSize(upstream) == 1);
-
-  *pQueryNode = taosArrayGetP(upstream, 0);
-
-  taosArrayDestroy(upstream);
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) {
-  return 0;
-}
-
-int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) {
-
-  return 0;
-}
-
-int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str) {
-  return 0;
-}
-
-void* destroyQueryPlan(SQueryPlanNode* pQueryNode) {
-  if (pQueryNode == NULL) {
-    return NULL;
-  }
-
-  doDestroyQueryNode(pQueryNode);
-  return NULL;
-}
-
-void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode) {
-  return NULL;
-}
-
-//======================================================================================================================
-
-static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev,
-                                   SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) {
-  SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode));
-
-  pNode->info.type = type;
-  pNode->info.name = strdup(name);
-
-  pNode->numOfExpr = numOfOutput;
-  pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES);
-
-  for(int32_t i = 0; i < numOfOutput; ++i) {
-    taosArrayPush(pNode->pExpr, &pExpr[i]);
-  }
-
-  pNode->pChildren = taosArrayInit(4, POINTER_BYTES);
-  for(int32_t i = 0; i < numOfPrev; ++i) {
-    taosArrayPush(pNode->pChildren, &prev[i]);
-  }
-
-  switch(type) {
-    case QNODE_TAGSCAN:
-    case QNODE_TABLESCAN: {
-      SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo));
-      memcpy(info, pExtInfo, sizeof(SQueryTableInfo));
-      info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName);
-      pNode->pExtInfo = info;
-      break;
-    }
-
-    case QNODE_TIMEWINDOW: {
-      SInterval* pInterval = calloc(1, sizeof(SInterval));
-      pNode->pExtInfo = pInterval;
-      memcpy(pInterval, pExtInfo, sizeof(SInterval));
-      break;
-    }
-
-    case QNODE_STATEWINDOW: {
-      SColumn* psw = calloc(1, sizeof(SColumn));
-      pNode->pExtInfo = psw;
-      memcpy(psw, pExtInfo, sizeof(SColumn));
-      break;
-    }
-
-    case QNODE_SESSIONWINDOW: {
-      SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow));
-      pNode->pExtInfo = pSessionWindow;
-      memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow));
-      break;
-    }
-
-    case QNODE_GROUPBY: {
-      SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo;
-
-      SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr));
-      pGroupbyExpr->groupbyTag = p->groupbyTag;
-      pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo);
-
-      pNode->pExtInfo = pGroupbyExpr;
-      break;
-    }
-
-    case QNODE_FILL: { // todo !!
-      pNode->pExtInfo = pExtInfo;
-      break;
-    }
-
-    case QNODE_LIMIT: {
-      pNode->pExtInfo = calloc(1, sizeof(SLimit));
-      memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit));
-      break;
-    }
-
-    case QNODE_SORT: {
-      pNode->pExtInfo = taosArrayDup(pExtInfo);
-      break;
-    }
-
-    default:
-      break;
-  }
-  
-  return pNode;
-}
-
-static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info,
-                                        SArray* pExprs, SArray* tableCols) {
-  if (pQueryInfo->info.onlyTagQuery) {
-    int32_t num = (int32_t) taosArrayGetSize(pExprs);
-    SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info);
-
-    if (pQueryInfo->info.distinct) {
-      pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL);
-    }
-
-    return pNode;
-  }
-
-  SQueryPlanNode*  pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info);
-
-  if (pQueryInfo->info.projectionQuery) {
-    int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs);
-    pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL);
-  } else {
-    STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0);
-
-    // table source column projection, generate the projection expr
-    int32_t     numOfCols = (int32_t) taosArrayGetSize(tableCols);
-    SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES);
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      SColumn* pCol = taosArrayGetP(tableCols, i);
-
-      SSourceParam param = {0};
-      addIntoSourceParam(&param, NULL, pCol);
-      SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name);
-      SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", &param, &s, 0);
-      pExpr[i] = p;
-    }
-
-    pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL);
-    tfree(pExpr);
-  }
-
-  return pNode;
-}
-
-static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) {
-  // group by column not by tag
-  size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo);
-
-  // check for aggregation
-  int32_t level = getExprFunctionLevel(pQueryInfo);
-
-  for(int32_t i = level - 1; i >= 0; --i) {
-    SArray* p = pQueryInfo->exprList[i];
-    size_t  num = taosArrayGetSize(p);
-
-    bool aggregateFunc = false;
-    for(int32_t j = 0; j < num; ++j) {
-      SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0);
-      if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) {
-        continue;
-      }
-
-      aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName);
-      if (aggregateFunc) {
-        break;
-      }
-    }
-
-    if (aggregateFunc) {
-      if (pQueryInfo->interval.interval > 0) {
-        pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval);
-      } else if (pQueryInfo->sessionWindow.gap > 0) {
-        pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow);
-      } else if (pQueryInfo->stateWindow.col.info.colId > 0) {
-        pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow);
-      } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) {
-          pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr);
-      } else {
-        pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL);
-      }
-    } else {
-      pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL);
-    }
-  }
-
-  if (pQueryInfo->havingFieldNum > 0) {
-//    int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1);
-//    pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL);
-  }
-
-  if (pQueryInfo->fillType != TSDB_FILL_NONE) {
-    SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo));
-    pInfo->fillType = pQueryInfo->fillType;
-    pInfo->val      = calloc(pNode->numOfExpr, sizeof(int64_t));
-    memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr);
-
-    SArray* p = pQueryInfo->exprList[0];  // top expression in select clause
-    pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo);
-  }
-
-  if (pQueryInfo->order != NULL) {
-    SArray* pList = pQueryInfo->exprList[0];
-    pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order);
-  }
-
-  if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) {
-    pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit);
-  }
-
-  return pNode;
-}
-
-static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs,
-                                                SArray* tableCols) {
-  char name[TSDB_TABLE_FNAME_LEN] = {0};
-  tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN);
-
-  SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
-
-  // handle the only tag query
-  SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols);
-  if (pQueryInfo->info.onlyTagQuery) {
-    tfree(info.tableName);
-    return pNode;
-  }
-
-  SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info);
-  tfree(info.tableName);
-  return pNode1;
-}
-
-static bool isAllAggExpr(SArray* pList) {
-  assert(pList != NULL);
-
-  for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) {
-    SExprInfo* p = taosArrayGetP(pList, k);
-    if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) {
-  SArray* upstream = NULL;
-
-  if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) {  // subquery in the from clause
-    upstream = taosArrayInit(4, POINTER_BYTES);
-
-    size_t size = taosArrayGetSize(pQueryInfo->pUpstream);
-    for(int32_t i = 0; i < size; ++i) {
-      SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i);
-      SArray* p = createQueryPlanImpl(pq);
-      taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p));
-    }
-  }
-
-  if (pQueryInfo->numOfTables > 1) {  // it is a join query
-    // 1. separate the select clause according to table
-    taosArrayDestroy(upstream);
-    upstream = taosArrayInit(5, POINTER_BYTES);
-
-    for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
-      STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i];
-      uint64_t        uid = pTableMetaInfo->pTableMeta->uid;
-
-      SArray* exprList = taosArrayInit(4, POINTER_BYTES);
-      if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) {
-        terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-//        dropAllExprInfo(exprList);
-        exit(-1);
-      }
-
-      // 2. create the query execution node
-      char name[TSDB_TABLE_FNAME_LEN] = {0};
-      tNameExtractFullName(&pTableMetaInfo->name, name);
-      SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
-
-      // 3. get the required table column list
-      SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn));
-      columnListCopy(tableColumnList, pQueryInfo->colList, uid);
-
-      // 4. add the projection query node
-      SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList);
-      columnListDestroy(tableColumnList);
-//      dropAllExprInfo(exprList);
-      taosArrayPush(upstream, &pNode);
-    }
-
-    // 3. add the join node here
-    SQueryTableInfo info = {0};
-    int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]);
-    SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables,
-                                        pQueryInfo->exprList[0]->pData, num, NULL);
-
-    // 4. add the aggregation or projection execution node
-    pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info);
-    upstream = taosArrayInit(5, POINTER_BYTES);
-    taosArrayPush(upstream, &pNode);
-  } else { // only one table, normal query process
-    STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
-    SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList);
-    upstream = taosArrayInit(5, POINTER_BYTES);
-    taosArrayPush(upstream, &pNode);
-  }
-
-  return upstream;
+void qDestroyQueryDag(struct SQueryDag* pDag) {
+  // todo
 }
 
-static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
-  tfree(pQueryNode->pExtInfo);
-  tfree(pQueryNode->pSchema);
-  tfree(pQueryNode->info.name);
-//  dropAllExprInfo(pQueryNode->pExpr);
-
-  if (pQueryNode->pChildren != NULL) {
-    int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren);
-    for(int32_t i = 0; i < size; ++i) {
-      SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i);
-      doDestroyQueryNode(p);
+int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag) {
+  SQueryPlanNode* logicPlan;
+  int32_t code = createQueryPlan(pQueryInfo, &logicPlan);
+  if (TSDB_CODE_SUCCESS != code) {
+    destroyQueryPlan(logicPlan);
+    return code;
   }
-
-    taosArrayDestroy(pQueryNode->pChildren);
+  code = optimizeQueryPlan(logicPlan);
+  if (TSDB_CODE_SUCCESS != code) {
+    destroyQueryPlan(logicPlan);
+    return code;
   }
-
-  tfree(pQueryNode);
-}
-
-static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
-  if (level > 0) {
-    sprintf(buf + totalLen, "%*c", level, ' ');
-    totalLen += level;
+  code = createDag(logicPlan, NULL, pDag);
+  if (TSDB_CODE_SUCCESS != code) {
+    destroyQueryPlan(logicPlan);
+    qDestroyQueryDag(*pDag);
+    return code;
   }
-
-  int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name);
-  int32_t len = len1 + totalLen;
-
-  switch(pQueryNode->info.type) {
-    case QNODE_TABLESCAN: {
-      SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo;
-      len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid,
-                     pInfo->window.skey, pInfo->window.ekey);
-      assert(len1 > 0);
-      len += len1;
-
-      for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
-        SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i);
-        len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId);
-
-        assert(len1 > 0);
-        len += len1;
-      }
-
-      len1 = sprintf(buf + len, "\n");
-      assert(len1 > 0);
-
-      len += len1;
-      break;
-    }
-
-    case QNODE_PROJECT: {
-      len1 = sprintf(buf + len, "cols:");
-      assert(len1 > 0);
-      len += len1;
-
-      len = printExprInfo(buf, pQueryNode, len);
-      len1 = sprintf(buf + len, ")");
-      len += len1;
-
-      // todo print filter info
-      len1 = sprintf(buf + len, " filters:(nil)\n");
-      len += len1;
-      break;
-    }
-
-    case QNODE_AGGREGATE: {
-      len = printExprInfo(buf, pQueryNode, len);
-      len1 = sprintf(buf + len, ")\n");
-      len += len1;
-
-      break;
-    }
-
-    case QNODE_TIMEWINDOW: {
-      len = printExprInfo(buf, pQueryNode, len);
-      len1 = sprintf(buf + len, ") ");
-      len += len1;
-
-      SInterval* pInterval = pQueryNode->pExtInfo;
-
-      // todo dynamic return the time precision
-      len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n",
-                     pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding,
-                     TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR);
-      len += len1;
-
-      break;
-    }
-
-    case QNODE_STATEWINDOW: {
-      len = printExprInfo(buf, pQueryNode, len);
-      len1 = sprintf(buf + len, ") ");
-      len += len1;
-
-      SColumn* pCol = pQueryNode->pExtInfo;
-      len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId);
-      len += len1;
-      break;
-    }
-
-    case QNODE_SESSIONWINDOW: {
-      len = printExprInfo(buf, pQueryNode, len);
-
-      len1 = sprintf(buf + len, ") ");
-      len += len1;
-
-      struct SSessionWindow* ps = pQueryNode->pExtInfo;
-      len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap);
-      len += len1;
-      break;
-    }
-
-    case QNODE_GROUPBY: {
-      len = printExprInfo(buf, pQueryNode, len);
-
-      SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo;
-      len1 = sprintf(buf + len, ") groupby_col: ");
-      len += len1;
-
-      for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) {
-        SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i);
-        len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId);
-        len += len1;
-      }
-
-      len += sprintf(buf + len, "\n");
-      break;
-    }
-
-    case QNODE_FILL: {
-      SFillEssInfo* pEssInfo = pQueryNode->pExtInfo;
-      len1 = sprintf(buf + len, "%d", pEssInfo->fillType);
-      len += len1;
-
-      if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) {
-        len1 = sprintf(buf + len, ", val:");
-        len += len1;
-
-        // todo get the correct fill data type
-        for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
-          len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]);
-          len += len1;
-
-          if (i < pQueryNode->numOfExpr - 1) {
-            len1 = sprintf(buf + len, ", ");
-            len += len1;
-          }
-        }
-      }
-
-      len1 = sprintf(buf + len, ")\n");
-      len += len1;
-      break;
-    }
-
-    case QNODE_LIMIT: {
-      SLimit* pVal = pQueryNode->pExtInfo;
-      len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset);
-      len += len1;
-      break;
-    }
-
-    case QNODE_DISTINCT:
-    case QNODE_TAGSCAN: {
-      len1 = sprintf(buf + len, "cols: ");
-      len += len1;
-
-      len = printExprInfo(buf, pQueryNode, len);
-
-      len1 = sprintf(buf + len, ")\n");
-      len += len1;
-
-      break;
-    }
-
-    case QNODE_SORT: {
-      len1 = sprintf(buf + len, "cols:");
-      len += len1;
-
-      SArray* pSort = pQueryNode->pExtInfo;
-      for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) {
-        SOrder* p = taosArrayGet(pSort, i);
-        len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC");
-
-        len += len1;
-      }
-
-      len1 = sprintf(buf + len, ")\n");
-      len += len1;
-      break;
-    }
-
-    case QNODE_JOIN: {
-      //  print join condition
-      len1 = sprintf(buf + len, ")\n");
-      len += len1;
-      break;
-    }
-  }
-
-  return len;
-}
-
-int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len) {
-  int32_t len1 = 0;
-
-  for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
-    SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
-
-    SSqlExpr* pExpr = &pExprInfo->base;
-    len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId);
-    assert(len1 > 0);
-
-    len += len1;
-    if (i < pQueryNode->numOfExpr - 1) {
-      len1 = sprintf(buf + len, ", ");
-      len += len1;
-    }
-  }
-
-  return len;
-}
-
-int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
-  int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen);
-
-  for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) {
-    SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i);
-    int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len);
-    len = len1;
-  }
-
-  return len;
-}
-
-int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) {
-  assert(pQueryNode);
-
-  *str = calloc(1, 4096);
-
-  int32_t len = sprintf(*str, "===== logic plan =====\n");
-  queryPlanToStringImpl(*str, pQueryNode, 0, len);
-
+  destroyQueryPlan(logicPlan);
   return TSDB_CODE_SUCCESS;
 }
-
-SQueryPlanNode* queryPlanFromString() {
-  return NULL;
-}

source/libs/wal/src/walMeta.c

@@ -14,27 +14,21 @@
  */
 
 #define _DEFAULT_SOURCE
+#include "cJSON.h"
 #include "os.h"
 #include "taoserror.h"
-#include "tref.h"
 #include "tfile.h"
-#include "cJSON.h"
+#include "tref.h"
 #include "walInt.h"
 
 #include <libgen.h>
 #include <regex.h>
 
-int64_t inline walGetFirstVer(SWal *pWal) {
-  return pWal->vers.firstVer;
-}
+int64_t inline walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }
 
-int64_t inline walGetSnaphostVer(SWal *pWal) {
-  return pWal->vers.snapshotVer;
-}
+int64_t inline walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; }
 
-int64_t inline walGetLastVer(SWal *pWal) {
-  return pWal->vers.lastVer;
-}
+int64_t inline walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; }
 
 static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
   return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
@@ -51,24 +45,23 @@ int walCheckAndRepairMeta(SWal* pWal) {
   regcomp(&logRegPattern, logPattern, REG_EXTENDED);
   regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
 
-  DIR *dir = opendir(pWal->path); 
-  if(dir == NULL) {
+  DIR* dir = opendir(pWal->path);
+  if (dir == NULL) {
     wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
     return -1;
   }
 
   struct dirent* ent;
-  while((ent = readdir(dir)) != NULL) {
-    char *name = basename(ent->d_name);
+  while ((ent = readdir(dir)) != NULL) {
+    char* name = basename(ent->d_name);
     int   code = regexec(&logRegPattern, name, 0, NULL, 0);
-    if(code == 0) {
+    if (code == 0) {
       int64_t firstVer;
       sscanf(name, "%" PRId64 ".log", &firstVer);
       taosArrayPush(pLogArray, &firstVer);
     }
   }
 
-  
   // load meta
   // if not match, or meta missing
   // rebuild meta
@@ -85,15 +78,15 @@ int walRollFileInfo(SWal* pWal) {
   int64_t ts = taosGetTimestampSec();
 
   SArray* pArray = pWal->fileInfoSet;
-  if(taosArrayGetSize(pArray) != 0) {
-    WalFileInfo *pInfo = taosArrayGetLast(pArray);
+  if (taosArrayGetSize(pArray) != 0) {
+    WalFileInfo* pInfo = taosArrayGetLast(pArray);
     pInfo->lastVer = pWal->vers.lastVer;
     pInfo->closeTs = ts;
   }
 
-  //TODO: change to emplace back
-  WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo));
-  if(pNewInfo == NULL) {
+  // TODO: change to emplace back
+  WalFileInfo* pNewInfo = malloc(sizeof(WalFileInfo));
+  if (pNewInfo == NULL) {
     return -1;
   }
   pNewInfo->firstVer = pWal->vers.lastVer + 1;
@@ -114,8 +107,8 @@ char* walMetaSerialize(SWal* pWal) {
   cJSON* pMeta = cJSON_CreateObject();
   cJSON* pFiles = cJSON_CreateArray();
   cJSON* pField;
-  if(pRoot == NULL || pMeta == NULL || pFiles == NULL) {
-    //TODO
+  if (pRoot == NULL || pMeta == NULL || pFiles == NULL) {
+    // TODO
     return NULL;
   }
   cJSON_AddItemToObject(pRoot, "meta", pMeta);
@@ -130,15 +123,15 @@ char* walMetaSerialize(SWal* pWal) {
 
   cJSON_AddItemToObject(pRoot, "files", pFiles);
   WalFileInfo* pData = pWal->fileInfoSet->pData;
-  for(int i = 0; i < sz; i++) {
+  for (int i = 0; i < sz; i++) {
     WalFileInfo* pInfo = &pData[i];
     cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject());
-    if(pField == NULL) {
+    if (pField == NULL) {
       cJSON_Delete(pRoot);
       return NULL;
     }
-    //cjson only support int32_t or double
-    //string are used to prohibit the loss of precision
+    // cjson only support int32_t or double
+    // string are used to prohibit the loss of precision
     sprintf(buf, "%" PRId64, pInfo->firstVer);
     cJSON_AddStringToObject(pField, "firstVer", buf);
     sprintf(buf, "%" PRId64, pInfo->lastVer);
@@ -171,11 +164,11 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
 
   pFiles = cJSON_GetObjectItem(pRoot, "files");
   int sz = cJSON_GetArraySize(pFiles);
-  //deserialize
+  // deserialize
   SArray* pArray = pWal->fileInfoSet;
   taosArrayEnsureCap(pArray, sz);
-  WalFileInfo *pData = pArray->pData;
-  for(int i = 0; i < sz; i++) {
+  WalFileInfo* pData = pArray->pData;
+  for (int i = 0; i < sz; i++) {
     cJSON*       pInfoJson = cJSON_GetArrayItem(pFiles, i);
     WalFileInfo* pInfo = &pData[i];
     pField = cJSON_GetObjectItem(pInfoJson, "firstVer");
@@ -196,24 +189,24 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
 }
 
 static int walFindCurMetaVer(SWal* pWal) {
-  const char * pattern = "^meta-ver[0-9]+$";
+  const char* pattern = "^meta-ver[0-9]+$";
   regex_t     walMetaRegexPattern;
   regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED);
 
-  DIR *dir = opendir(pWal->path); 
-  if(dir == NULL) {
+  DIR* dir = opendir(pWal->path);
+  if (dir == NULL) {
     wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
     return -1;
   }
 
   struct dirent* ent;
 
-  //find existing meta-ver[x].json
+  // find existing meta-ver[x].json
   int metaVer = -1;
-  while((ent = readdir(dir)) != NULL) {
-    char *name = basename(ent->d_name);
+  while ((ent = readdir(dir)) != NULL) {
+    char* name = basename(ent->d_name);
     int   code = regexec(&walMetaRegexPattern, name, 0, NULL, 0);
-    if(code == 0) {
+    if (code == 0) {
       sscanf(name, "meta-ver%d", &metaVer);
       break;
     }
@@ -226,21 +219,21 @@ static int walFindCurMetaVer(SWal* pWal) {
 int walSaveMeta(SWal* pWal) {
   int  metaVer = walFindCurMetaVer(pWal);
   char fnameStr[WAL_FILE_LEN];
-  walBuildMetaName(pWal, metaVer+1, fnameStr);
+  walBuildMetaName(pWal, metaVer + 1, fnameStr);
   int metaTfd = tfOpenCreateWrite(fnameStr);
-  if(metaTfd < 0) {
+  if (metaTfd < 0) {
     return -1;
   }
   char* serialized = walMetaSerialize(pWal);
   int   len = strlen(serialized);
-  if(len != tfWrite(metaTfd, serialized, len)) {
-    //TODO:clean file
+  if (len != tfWrite(metaTfd, serialized, len)) {
+    // TODO:clean file
     return -1;
   }
 
   tfClose(metaTfd);
-  //delete old file
-  if(metaVer > -1) {
+  // delete old file
+  if (metaVer > -1) {
     walBuildMetaName(pWal, metaVer, fnameStr);
     remove(fnameStr);
   }
@@ -250,30 +243,30 @@ int walSaveMeta(SWal* pWal) {
 
 int walLoadMeta(SWal* pWal) {
   ASSERT(pWal->fileInfoSet->size == 0);
-  //find existing meta file
+  // find existing meta file
   int metaVer = walFindCurMetaVer(pWal);
-  if(metaVer == -1) {
+  if (metaVer == -1) {
     return 0;
   }
   char fnameStr[WAL_FILE_LEN];
   walBuildMetaName(pWal, metaVer, fnameStr);
-  //read metafile
+  // read metafile
   struct stat statbuf;
   stat(fnameStr, &statbuf);
   int   size = statbuf.st_size;
   char* buf = malloc(size + 5);
-  if(buf == NULL) {
+  if (buf == NULL) {
     return -1;
   }
-  memset(buf, 0, size+5);
+  memset(buf, 0, size + 5);
   int tfd = tfOpenRead(fnameStr);
-  if(tfRead(tfd, buf, size) != size) {
+  if (tfRead(tfd, buf, size) != size) {
     free(buf);
     return -1;
   }
-  //load into fileInfoSet
+  // load into fileInfoSet
   int code = walMetaDeserialize(pWal, buf);
-  if(code != 0) {
+  if (code != 0) {
     free(buf);
     return -1;
   }

source/libs/wal/src/walMgmt.c

@@ -14,11 +14,11 @@
  */
 
 #define _DEFAULT_SOURCE
+#include "compare.h"
 #include "os.h"
 #include "taoserror.h"
-#include "tref.h"
 #include "tfile.h"
-#include "compare.h"
+#include "tref.h"
 #include "walInt.h"
 
 typedef struct {
@@ -34,16 +34,14 @@ static int32_t  walCreateThread();
 static void     walStopThread();
 static void     walFreeObj(void *pWal);
 
-int64_t walGetSeq() {
-  return (int64_t)atomic_load_32(&tsWal.seq);
-}
+int64_t walGetSeq() { return (int64_t)atomic_load_32(&tsWal.seq); }
 
 int32_t walInit() {
   int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1);
-  if(old == 1) return 0;
+  if (old == 1) return 0;
 
   int code = tfInit();
-  if(code != 0) {
+  if (code != 0) {
     wError("failed to init tfile since %s", tstrerror(code));
     atomic_store_8(&tsWal.inited, 0);
     return code;
@@ -63,7 +61,7 @@ int32_t walInit() {
 
 void walCleanUp() {
   int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0);
-  if(old == 0) {
+  if (old == 0) {
     return;
   }
   walStopThread();
@@ -78,52 +76,52 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
     return NULL;
   }
 
-  //set config
+  // set config
   memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg));
   pWal->fsyncSeq = pCfg->fsyncPeriod / 1000;
-  if(pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
+  if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
 
   tstrncpy(pWal->path, path, sizeof(pWal->path));
-  if(taosMkDir(pWal->path) != 0) {
+  if (taosMkDir(pWal->path) != 0) {
     wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
     return NULL;
   }
 
-  //open meta
+  // open meta
   walResetVer(&pWal->vers);
   pWal->writeLogTfd = -1;
   pWal->writeIdxTfd = -1;
   pWal->writeCur = -1;
   pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo));
-  if(pWal->fileInfoSet == NULL) {
+  if (pWal->fileInfoSet == NULL) {
     wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno));
     free(pWal);
     return NULL;
   }
 
-  //init status
+  // init status
   pWal->totSize = 0;
   pWal->lastRollSeq = -1;
 
-  //init write buffer
+  // init write buffer
   memset(&pWal->writeHead, 0, sizeof(SWalHead));
   pWal->writeHead.head.headVer = WAL_HEAD_VER;
 
-  if(pthread_mutex_init(&pWal->mutex, NULL) < 0) {
+  if (pthread_mutex_init(&pWal->mutex, NULL) < 0) {
     taosArrayDestroy(pWal->fileInfoSet);
     free(pWal);
     return NULL;
   }
 
   pWal->refId = taosAddRef(tsWal.refSetId, pWal);
-  if(pWal->refId < 0) {
+  if (pWal->refId < 0) {
     pthread_mutex_destroy(&pWal->mutex);
     taosArrayDestroy(pWal->fileInfoSet);
     free(pWal);
     return NULL;
   }
 
-  if(walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) {
+  if (walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) {
     taosRemoveRef(tsWal.refSetId, pWal->refId);
     pthread_mutex_destroy(&pWal->mutex);
     taosArrayDestroy(pWal->fileInfoSet);
@@ -131,11 +129,11 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
     return NULL;
   }
 
-  if(walCheckAndRepairIdx(pWal) < 0) {
-
+  if (walCheckAndRepairIdx(pWal) < 0) {
   }
 
-  wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod);
+  wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level,
+         pWal->cfg.fsyncPeriod);
 
   return pWal;
 }
@@ -203,10 +201,12 @@ static void walFsyncAll() {
   SWal *pWal = taosIterateRef(tsWal.refSetId, 0);
   while (pWal) {
     if (walNeedFsync(pWal)) {
-      wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq));
+      wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq,
+             atomic_load_32(&tsWal.seq));
       int32_t code = tfFsync(pWal->writeLogTfd);
       if (code != 0) {
-        wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code));
+        wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
+               strerror(code));
       }
     }
     pWal = taosIterateRef(tsWal.refSetId, pWal->refId);

source/libs/wal/src/walRead.c

@@ -13,12 +13,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "walInt.h"
 #include "tfile.h"
+#include "walInt.h"
 
-SWalReadHandle* walOpenReadHandle(SWal* pWal) {
+SWalReadHandle *walOpenReadHandle(SWal *pWal) {
   SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle));
-  if(pRead == NULL) {
+  if (pRead == NULL) {
     return NULL;
   }
   pRead->pWal = pWal;
@@ -29,7 +29,7 @@ SWalReadHandle* walOpenReadHandle(SWal* pWal) {
   pRead->capacity = 0;
   pRead->status = 0;
   pRead->pHead = malloc(sizeof(SWalHead));
-  if(pRead->pHead == NULL) {
+  if (pRead->pHead == NULL) {
     free(pRead);
     return NULL;
   }
@@ -43,9 +43,7 @@ void walCloseReadHandle(SWalReadHandle *pRead) {
   free(pRead);
 }
 
-int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) {
-  return 0;
-}
+int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; }
 
 static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
   int code = 0;
@@ -53,17 +51,17 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
   int64_t idxTfd = pRead->readIdxTfd;
   int64_t logTfd = pRead->readLogTfd;
 
-  //seek position
+  // seek position
   int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry);
   code = tfLseek(idxTfd, offset, SEEK_SET);
-  if(code < 0) {
+  if (code < 0) {
     return -1;
   }
   WalIdxEntry entry;
-  if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
+  if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
     return -1;
   }
-  //TODO:deserialize
+  // TODO:deserialize
   ASSERT(entry.ver == ver);
   code = tfLseek(logTfd, entry.offset, SEEK_SET);
   if (code < 0) {
@@ -80,13 +78,13 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
 
   walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
   int64_t logTfd = tfOpenRead(fnameStr);
-  if(logTfd < 0) {
+  if (logTfd < 0) {
     return -1;
   }
 
   walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
   int64_t idxTfd = tfOpenRead(fnameStr);
-  if(idxTfd < 0) {
+  if (idxTfd < 0) {
     return -1;
   }
 
@@ -98,31 +96,30 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
 static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
   int   code;
   SWal *pWal = pRead->pWal;
-  if(ver == pRead->curVersion) {
+  if (ver == pRead->curVersion) {
     return 0;
   }
-  if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
+  if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
     return -1;
   }
-  if(ver < pWal->vers.snapshotVer) {
-
+  if (ver < pWal->vers.snapshotVer) {
   }
 
   WalFileInfo tmpInfo;
   tmpInfo.firstVer = ver;
-  //bsearch in fileSet
-  WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
+  // bsearch in fileSet
+  WalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
   ASSERT(pRet != NULL);
-  if(pRead->curFileFirstVer != pRet->firstVer) {
+  if (pRead->curFileFirstVer != pRet->firstVer) {
     code = walReadChangeFile(pRead, pRet->firstVer);
-    if(code < 0) {
-      //TODO: set error flag
+    if (code < 0) {
+      // TODO: set error flag
       return -1;
     }
   }
 
   code = walReadSeekFilePos(pRead, pRet->firstVer, ver);
-  if(code < 0) {
+  if (code < 0) {
     return -1;
   }
   pRead->curVersion = ver;
@@ -132,40 +129,40 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
 
 int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
   int code;
-  //TODO: check wal life
-  if(pRead->curVersion != ver) {
+  // TODO: check wal life
+  if (pRead->curVersion != ver) {
     code = walReadSeekVer(pRead, ver);
-    if(code != 0) {
+    if (code != 0) {
       return -1;
     }
   }
 
-  if(!tfValid(pRead->readLogTfd)) return -1;
+  if (!tfValid(pRead->readLogTfd)) return -1;
 
   code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead));
-  if(code != sizeof(SWalHead)) {
+  if (code != sizeof(SWalHead)) {
     return -1;
   }
   code = walValidHeadCksum(pRead->pHead);
-  if(code != 0) {
+  if (code != 0) {
     return -1;
   }
-  if(pRead->capacity < pRead->pHead->head.len) {
-    void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len);
-    if(ptr == NULL) {
+  if (pRead->capacity < pRead->pHead->head.len) {
+    void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len);
+    if (ptr == NULL) {
       return -1;
     }
     pRead->pHead = ptr;
     pRead->capacity = pRead->pHead->head.len;
   }
-  if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
+  if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
     return -1;
   }
 
   /*code = walValidBodyCksum(pRead->pHead);*/
   ASSERT(pRead->pHead->head.version == ver);
 
-  if(code != 0) {
+  if (code != 0) {
     return -1;
   }
   pRead->curVersion++;
@@ -176,34 +173,34 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
 int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
   int code;
   code = walSeekVer(pWal, ver);
-  if(code != 0) {
+  if (code != 0) {
     return code;
   }
-  if(*ppHead == NULL) {
-    void* ptr = realloc(*ppHead, sizeof(SWalHead));
-    if(ptr == NULL) {
+  if (*ppHead == NULL) {
+    void *ptr = realloc(*ppHead, sizeof(SWalHead));
+    if (ptr == NULL) {
       return -1;
     }
     *ppHead = ptr;
   }
-  if(tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
+  if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
     return -1;
   }
-  //TODO: endian compatibility processing after read
-  if(walValidHeadCksum(*ppHead) != 0) {
+  // TODO: endian compatibility processing after read
+  if (walValidHeadCksum(*ppHead) != 0) {
     return -1;
   }
-  void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len);
-  if(ptr == NULL) {
+  void *ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len);
+  if (ptr == NULL) {
     free(*ppHead);
     *ppHead = NULL;
     return -1;
   }
-  if(tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
+  if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
     return -1;
   }
-  //TODO: endian compatibility processing after read
-  if(walValidBodyCksum(*ppHead) != 0) {
+  // TODO: endian compatibility processing after read
+  if (walValidBodyCksum(*ppHead) != 0) {
     return -1;
   }
 
@@ -211,5 +208,5 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
 }
 
 /*int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {*/
-  /*return 0;*/
+/*return 0;*/
 /*}*/

source/libs/wal/src/walSeek.c

@@ -16,8 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
-#include "tref.h"
 #include "tfile.h"
+#include "tref.h"
 #include "walInt.h"
 
 static int walSeekFilePos(SWal* pWal, int64_t ver) {
@@ -26,16 +26,16 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
   int64_t idxTfd = pWal->writeIdxTfd;
   int64_t logTfd = pWal->writeLogTfd;
 
-  //seek position
+  // seek position
   int64_t idxOff = walGetVerIdxOffset(pWal, ver);
   code = tfLseek(idxTfd, idxOff, SEEK_SET);
-  if(code != 0) {
+  if (code != 0) {
     return -1;
   }
   WalIdxEntry entry;
-  //TODO:deserialize
+  // TODO:deserialize
   code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry));
-  if(code != 0) {
+  if (code != 0) {
     return -1;
   }
   ASSERT(entry.ver == ver);
@@ -46,7 +46,7 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
   return code;
 }
 
-int walChangeFileToLast(SWal *pWal) {
+int walChangeFileToLast(SWal* pWal) {
   int64_t      idxTfd, logTfd;
   WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
   ASSERT(pRet != NULL);
@@ -55,42 +55,42 @@ int walChangeFileToLast(SWal *pWal) {
   char fnameStr[WAL_FILE_LEN];
   walBuildIdxName(pWal, fileFirstVer, fnameStr);
   idxTfd = tfOpenReadWrite(fnameStr);
-  if(idxTfd < 0) {
+  if (idxTfd < 0) {
     return -1;
   }
   walBuildLogName(pWal, fileFirstVer, fnameStr);
   logTfd = tfOpenReadWrite(fnameStr);
-  if(logTfd < 0) {
+  if (logTfd < 0) {
     return -1;
   }
-  //switch file
+  // switch file
   pWal->writeIdxTfd = idxTfd;
   pWal->writeLogTfd = logTfd;
   return 0;
 }
 
-int walChangeFile(SWal *pWal, int64_t ver) {
+int walChangeFile(SWal* pWal, int64_t ver) {
   int     code = 0;
   int64_t idxTfd, logTfd;
   char    fnameStr[WAL_FILE_LEN];
   code = tfClose(pWal->writeLogTfd);
-  if(code != 0) {
-   //TODO 
+  if (code != 0) {
+    // TODO
     return -1;
   }
   code = tfClose(pWal->writeIdxTfd);
-  if(code != 0) {
-   //TODO 
+  if (code != 0) {
+    // TODO
     return -1;
   }
   WalFileInfo tmpInfo;
   tmpInfo.firstVer = ver;
-  //bsearch in fileSet
+  // bsearch in fileSet
   WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
   ASSERT(pRet != NULL);
   int64_t fileFirstVer = pRet->firstVer;
-  //closed
-  if(taosArrayGetLast(pWal->fileInfoSet) != pRet) {
+  // closed
+  if (taosArrayGetLast(pWal->fileInfoSet) != pRet) {
     walBuildIdxName(pWal, fileFirstVer, fnameStr);
     idxTfd = tfOpenRead(fnameStr);
     walBuildLogName(pWal, fileFirstVer, fnameStr);
@@ -107,25 +107,24 @@ int walChangeFile(SWal *pWal, int64_t ver) {
   return fileFirstVer;
 }
 
-int walSeekVer(SWal *pWal, int64_t ver) {
+int walSeekVer(SWal* pWal, int64_t ver) {
   int code;
-  if(ver == pWal->vers.lastVer) {
+  if (ver == pWal->vers.lastVer) {
     return 0;
   }
-  if(ver > pWal->vers.lastVer|| ver < pWal->vers.firstVer) {
+  if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
     return -1;
   }
-  if(ver < pWal->vers.snapshotVer) {
-
+  if (ver < pWal->vers.snapshotVer) {
   }
-  if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) {
+  if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) {
     code = walChangeFile(pWal, ver);
-    if(code != 0) {
+    if (code != 0) {
       return -1;
     }
   }
   code = walSeekFilePos(pWal, ver);
-  if(code != 0) {
+  if (code != 0) {
     return -1;
   }
 

source/libs/wal/src/walWrite.c

@@ -24,7 +24,7 @@
 int32_t walCommit(SWal *pWal, int64_t ver) {
   ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer);
   ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer);
-  if(ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) {
+  if (ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) {
     return -1;
   }
   pWal->vers.commitVer = ver;
@@ -34,55 +34,55 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
 int32_t walRollback(SWal *pWal, int64_t ver) {
   int  code;
   char fnameStr[WAL_FILE_LEN];
-  if(ver == pWal->vers.lastVer) {
+  if (ver == pWal->vers.lastVer) {
     return 0;
   }
-  if(ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
+  if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
     return -1;
   }
   pthread_mutex_lock(&pWal->mutex);
 
-  //find correct file
-  if(ver < walGetLastFileFirstVer(pWal)) {
-    //close current files
+  // find correct file
+  if (ver < walGetLastFileFirstVer(pWal)) {
+    // close current files
     tfClose(pWal->writeIdxTfd);
     tfClose(pWal->writeLogTfd);
-    //open old files
+    // open old files
     code = walChangeFile(pWal, ver);
-    if(code != 0) {
+    if (code != 0) {
       return -1;
     }
 
-    //delete files
+    // delete files
     int fileSetSize = taosArrayGetSize(pWal->fileInfoSet);
-    for(int i = pWal->writeCur; i < fileSetSize; i++) {
-      walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
+    for (int i = pWal->writeCur; i < fileSetSize; i++) {
+      walBuildLogName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
       remove(fnameStr);
-      walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
+      walBuildIdxName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
       remove(fnameStr);
     }
-    //pop from fileInfoSet
+    // pop from fileInfoSet
     taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1);
   }
 
   walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
   int64_t idxTfd = tfOpenReadWrite(fnameStr);
 
-  //TODO:change to deserialize function
-  if(idxTfd < 0) {
+  // TODO:change to deserialize function
+  if (idxTfd < 0) {
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
   }
   int64_t idxOff = walGetVerIdxOffset(pWal, ver);
   code = tfLseek(idxTfd, idxOff, SEEK_SET);
-  if(code < 0) {
+  if (code < 0) {
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
   }
-  //read idx file and get log file pos
-  //TODO:change to deserialize function
+  // read idx file and get log file pos
+  // TODO:change to deserialize function
   WalIdxEntry entry;
-  if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
+  if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
   }
@@ -90,56 +90,56 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
 
   walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
   int64_t logTfd = tfOpenReadWrite(fnameStr);
-  if(logTfd < 0) {
-    //TODO
+  if (logTfd < 0) {
+    // TODO
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
   }
   code = tfLseek(logTfd, entry.offset, SEEK_SET);
-  if(code < 0) {
-    //TODO
+  if (code < 0) {
+    // TODO
     pthread_mutex_unlock(&pWal->mutex);
     return -1;
   }
-  //validate offset
+  // validate offset
   SWalHead head;
   ASSERT(tfValid(logTfd));
   int size = tfRead(logTfd, &head, sizeof(SWalHead));
-  if(size != sizeof(SWalHead)) {
+  if (size != sizeof(SWalHead)) {
     return -1;
   }
   code = walValidHeadCksum(&head);
 
   ASSERT(code == 0);
-  if(code != 0) {
+  if (code != 0) {
     return -1;
   }
-  if(head.head.version != ver) {
-    //TODO
+  if (head.head.version != ver) {
+    // TODO
     return -1;
   }
-  //truncate old files
+  // truncate old files
   code = tfFtruncate(logTfd, entry.offset);
-  if(code < 0) {
+  if (code < 0) {
     return -1;
   }
   code = tfFtruncate(idxTfd, idxOff);
-  if(code < 0) {
+  if (code < 0) {
     return -1;
   }
   pWal->vers.lastVer = ver - 1;
-  ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
-  ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
+  ((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
+  ((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
 
-  //unlock
+  // unlock
   pthread_mutex_unlock(&pWal->mutex);
   return 0;
 }
 
-int32_t walBeginSnapshot(SWal* pWal, int64_t ver) {
+int32_t walBeginSnapshot(SWal *pWal, int64_t ver) {
   pWal->vers.verInSnapshotting = ver;
-  //check file rolling
-  if(pWal->cfg.retentionPeriod == 0) {
+  // check file rolling
+  if (pWal->cfg.retentionPeriod == 0) {
     walRoll(pWal);
   }
 
@@ -148,7 +148,7 @@ int32_t walBeginSnapshot(SWal* pWal, int64_t ver) {
 
 int32_t walEndSnapshot(SWal *pWal) {
   int64_t ver = pWal->vers.verInSnapshotting;
-  if(ver == -1) return -1;
+  if (ver == -1) return -1;
 
   pWal->vers.snapshotVer = ver;
   int ts = taosGetTimestampSec();
@@ -157,45 +157,45 @@ int32_t walEndSnapshot(SWal *pWal) {
   int64_t     newTotSize = pWal->totSize;
   WalFileInfo tmp;
   tmp.firstVer = ver;
-  //find files safe to delete
-  WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
-  if(ver >= pInfo->lastVer) {
+  // find files safe to delete
+  WalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
+  if (ver >= pInfo->lastVer) {
     pInfo++;
   }
-  //iterate files, until the searched result
-  for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
-    if(pWal->totSize > pWal->cfg.retentionSize ||
-        iter->closeTs + pWal->cfg.retentionPeriod > ts) {
-      //delete according to file size or close time
+  // iterate files, until the searched result
+  for (WalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
+    if (pWal->totSize > pWal->cfg.retentionSize || iter->closeTs + pWal->cfg.retentionPeriod > ts) {
+      // delete according to file size or close time
       deleteCnt++;
       newTotSize -= iter->fileSize;
     }
   }
   char fnameStr[WAL_FILE_LEN];
-  //remove file
-  for(int i = 0; i < deleteCnt; i++) {
-    WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i);
+  // remove file
+  for (int i = 0; i < deleteCnt; i++) {
+    WalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i);
     walBuildLogName(pWal, pInfo->firstVer, fnameStr);
     remove(fnameStr);
     walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
     remove(fnameStr);
   }
 
-  //make new array, remove files
+  // make new array, remove files
   taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt);
-  if(taosArrayGetSize(pWal->fileInfoSet) == 0) {
+  if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
     pWal->writeCur = -1;
     pWal->vers.firstVer = -1;
   } else {
-    pWal->vers.firstVer = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
+    pWal->vers.firstVer = ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
   }
-  pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;;
+  pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
+  ;
   pWal->totSize = newTotSize;
   pWal->vers.verInSnapshotting = -1;
 
-  //save snapshot ver, commit ver
+  // save snapshot ver, commit ver
   int code = walSaveMeta(pWal);
-  if(code != 0) {
+  if (code != 0) {
     return -1;
   }
 
@@ -204,41 +204,41 @@ int32_t walEndSnapshot(SWal *pWal) {
 
 int walRoll(SWal *pWal) {
   int code = 0;
-  if(pWal->writeIdxTfd != -1) {
+  if (pWal->writeIdxTfd != -1) {
     code = tfClose(pWal->writeIdxTfd);
-    if(code != 0) {
+    if (code != 0) {
       return -1;
     }
   }
-  if(pWal->writeLogTfd != -1) {
+  if (pWal->writeLogTfd != -1) {
     code = tfClose(pWal->writeLogTfd);
-    if(code != 0) {
+    if (code != 0) {
       return -1;
     }
   }
   int64_t idxTfd, logTfd;
-  //create new file
+  // create new file
   int64_t newFileFirstVersion = pWal->vers.lastVer + 1;
   char    fnameStr[WAL_FILE_LEN];
   walBuildIdxName(pWal, newFileFirstVersion, fnameStr);
   idxTfd = tfOpenCreateWriteAppend(fnameStr);
-  if(idxTfd < 0) {
+  if (idxTfd < 0) {
     ASSERT(0);
     return -1;
   }
   walBuildLogName(pWal, newFileFirstVersion, fnameStr);
   logTfd = tfOpenCreateWriteAppend(fnameStr);
-  if(logTfd < 0) {
+  if (logTfd < 0) {
     ASSERT(0);
     return -1;
   }
   code = walRollFileInfo(pWal);
-  if(code != 0) {
+  if (code != 0) {
     ASSERT(0);
     return -1;
   }
 
-  //switch file
+  // switch file
   pWal->writeIdxTfd = idxTfd;
   pWal->writeLogTfd = logTfd;
   pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
@@ -248,10 +248,10 @@ int walRoll(SWal *pWal) {
 }
 
 static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
-  WalIdxEntry entry = { .ver = ver, .offset = offset };
+  WalIdxEntry entry = {.ver = ver, .offset = offset};
   int         size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry));
-  if(size != sizeof(WalIdxEntry)) {
-    //TODO truncate
+  if (size != sizeof(WalIdxEntry)) {
+    // TODO truncate
     return -1;
   }
   return 0;
@@ -265,21 +265,21 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
   if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0;
 
   if (index == pWal->vers.lastVer + 1) {
-    if(taosArrayGetSize(pWal->fileInfoSet) == 0) {
+    if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
       pWal->vers.firstVer = index;
       code = walRoll(pWal);
       ASSERT(code == 0);
     } else {
       int64_t passed = walGetSeq() - pWal->lastRollSeq;
-      if(pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
+      if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
         walRoll(pWal);
-      } else if(pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
+      } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
         walRoll(pWal);
       }
     }
   } else {
-    //reject skip log or rewrite log
-    //must truncate explicitly first
+    // reject skip log or rewrite log
+    // must truncate explicitly first
     return -1;
   }
   /*if (!tfValid(pWal->writeLogTfd)) return -1;*/
@@ -294,23 +294,25 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
   pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
 
   if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
-    //ftruncate
+    // ftruncate
     code = TAOS_SYSTEM_ERROR(errno);
-    wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno));
+    wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
+           strerror(errno));
   }
 
-  if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) {
-    //ftruncate
+  if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) {
+    // ftruncate
     code = TAOS_SYSTEM_ERROR(errno);
-    wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno));
+    wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
+           strerror(errno));
   }
   code = walWriteIndex(pWal, index, offset);
-  if(code != 0) {
-    //TODO
+  if (code != 0) {
+    // TODO
     return -1;
   }
 
-  //set status
+  // set status
   pWal->vers.lastVer = index;
   pWal->totSize += sizeof(SWalHead) + bodyLen;
   walGetCurFileInfo(pWal)->lastVer = index;
@@ -323,33 +325,34 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
 
 void walFsync(SWal *pWal, bool forceFsync) {
   if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
-    wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
+    wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
     if (tfFsync(pWal->writeLogTfd) < 0) {
-      wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno));
+      wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
+             strerror(errno));
     }
   }
 }
 
 /*static int walValidateOffset(SWal* pWal, int64_t ver) {*/
-  /*int code = 0;*/
-  /*SWalHead *pHead = NULL;*/
-  /*code = (int)walRead(pWal, &pHead, ver);*/
-  /*if(pHead->head.version != ver) {*/
-    /*return -1;*/
-  /*}*/
-  /*return 0;*/
+/*int code = 0;*/
+/*SWalHead *pHead = NULL;*/
+/*code = (int)walRead(pWal, &pHead, ver);*/
+/*if(pHead->head.version != ver) {*/
+/*return -1;*/
+/*}*/
+/*return 0;*/
 /*}*/
 
 /*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/
-  /*int code = walSeekVer(pWal, ver);*/
-  /*if(code != 0) {*/
-    /*return -1;*/
-  /*}*/
+/*int code = walSeekVer(pWal, ver);*/
+/*if(code != 0) {*/
+/*return -1;*/
+/*}*/
 
-  /*code = walValidateOffset(pWal, ver);*/
-  /*if(code != 0) {*/
-    /*return -1;*/
-  /*}*/
+/*code = walValidateOffset(pWal, ver);*/
+/*if(code != 0) {*/
+/*return -1;*/
+/*}*/
 
-  /*return 0;*/
+/*return 0;*/
 /*}*/

source/libs/wal/test/walMetaTest.cpp

@@ -15,9 +15,7 @@ class WalCleanEnv : public ::testing::Test {
     ASSERT(code == 0);
   }
 
-    static void TearDownTestCase() {
-      walCleanUp();
-    }
+  static void TearDownTestCase() { walCleanUp(); }
 
   void SetUp() override {
     taosRemoveDir(pathName);
@@ -49,9 +47,7 @@ class WalCleanDeleteEnv : public ::testing::Test {
     ASSERT(code == 0);
   }
 
-    static void TearDownTestCase() {
-      walCleanUp();
-    }
+  static void TearDownTestCase() { walCleanUp(); }
 
   void SetUp() override {
     taosRemoveDir(pathName);
@@ -81,9 +77,7 @@ class WalKeepEnv : public ::testing::Test {
     ASSERT(code == 0);
   }
 
-    static void TearDownTestCase() {
-      walCleanUp();
-    }
+  static void TearDownTestCase() { walCleanUp(); }
 
   void walResetEnv() {
     TearDown();
@@ -139,7 +133,7 @@ TEST_F(WalCleanEnv, serialize) {
   ASSERT(code == 0);
   code = walRollFileInfo(pWal);
   ASSERT(code == 0);
-  char*ss = walMetaSerialize(pWal);
+  char* ss = walMetaSerialize(pWal);
   printf("%s\n", ss);
   free(ss);
   code = walSaveMeta(pWal);
@@ -162,11 +156,11 @@ TEST_F(WalKeepEnv, readOldMeta) {
   walResetEnv();
   int code;
 
-  for(int i = 0; i < 10; i++) {
-    code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); 
+  for (int i = 0; i < 10; i++) {
+    code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
     ASSERT_EQ(code, 0);
     ASSERT_EQ(pWal->vers.lastVer, i);
-    code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen);
+    code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen);
     ASSERT_EQ(code, -1);
     ASSERT_EQ(pWal->vers.lastVer, i);
   }
@@ -182,7 +176,7 @@ TEST_F(WalKeepEnv, readOldMeta) {
 
   int len = strlen(oldss);
   ASSERT_EQ(len, strlen(newss));
-  for(int i = 0; i < len; i++) {
+  for (int i = 0; i < len; i++) {
     EXPECT_EQ(oldss[i], newss[i]);
   }
   free(oldss);
@@ -191,11 +185,11 @@ TEST_F(WalKeepEnv, readOldMeta) {
 
 TEST_F(WalCleanEnv, write) {
   int code;
-  for(int i = 0; i < 10; i++) {
-    code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); 
+  for (int i = 0; i < 10; i++) {
+    code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
     ASSERT_EQ(code, 0);
     ASSERT_EQ(pWal->vers.lastVer, i);
-    code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen);
+    code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen);
     ASSERT_EQ(code, -1);
     ASSERT_EQ(pWal->vers.lastVer, i);
   }
@@ -205,8 +199,8 @@ TEST_F(WalCleanEnv, write) {
 
 TEST_F(WalCleanEnv, rollback) {
   int code;
-  for(int i = 0; i < 10; i++) {
-    code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); 
+  for (int i = 0; i < 10; i++) {
+    code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
     ASSERT_EQ(code, 0);
     ASSERT_EQ(pWal->vers.lastVer, i);
   }
@@ -223,7 +217,7 @@ TEST_F(WalCleanEnv, rollback) {
 TEST_F(WalCleanDeleteEnv, roll) {
   int code;
   int i;
-  for(i = 0; i < 100; i++) {
+  for (i = 0; i < 100; i++) {
     code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen);
     ASSERT_EQ(code, 0);
     ASSERT_EQ(pWal->vers.lastVer, i);
@@ -231,16 +225,16 @@ TEST_F(WalCleanDeleteEnv, roll) {
     ASSERT_EQ(pWal->vers.commitVer, i);
   }
 
-  walBeginSnapshot(pWal, i-1);
-  ASSERT_EQ(pWal->vers.verInSnapshotting, i-1);
+  walBeginSnapshot(pWal, i - 1);
+  ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1);
   walEndSnapshot(pWal);
-  ASSERT_EQ(pWal->vers.snapshotVer, i-1);
+  ASSERT_EQ(pWal->vers.snapshotVer, i - 1);
   ASSERT_EQ(pWal->vers.verInSnapshotting, -1);
 
   code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen);
   ASSERT_NE(code, 0);
 
-  for(; i < 200; i++) {
+  for (; i < 200; i++) {
     code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen);
     ASSERT_EQ(code, 0);
     code = walCommit(pWal, i);
@@ -259,32 +253,32 @@ TEST_F(WalKeepEnv, readHandleRead) {
   SWalReadHandle* pRead = walOpenReadHandle(pWal);
   ASSERT(pRead != NULL);
 
-  int i ;
-  for(i = 0; i < 100; i++) {
+  int i;
+  for (i = 0; i < 100; i++) {
     char newStr[100];
     sprintf(newStr, "%s-%d", ranStr, i);
     int len = strlen(newStr);
     code = walWrite(pWal, i, 0, newStr, len);
     ASSERT_EQ(code, 0);
   }
-  for(int i = 0; i < 1000; i++) {
+  for (int i = 0; i < 1000; i++) {
     int ver = rand() % 100;
     code = walReadWithHandle(pRead, ver);
     ASSERT_EQ(code, 0);
 
-    //printf("rrbody: \n");
-    //for(int i = 0; i < pRead->pHead->head.len; i++) {
-      //printf("%d ", pRead->pHead->head.body[i]);
+    // printf("rrbody: \n");
+    // for(int i = 0; i < pRead->pHead->head.len; i++) {
+    // printf("%d ", pRead->pHead->head.body[i]);
     //}
-    //printf("\n");
+    // printf("\n");
 
     ASSERT_EQ(pRead->pHead->head.version, ver);
-    ASSERT_EQ(pRead->curVersion, ver+1);
+    ASSERT_EQ(pRead->curVersion, ver + 1);
     char newStr[100];
     sprintf(newStr, "%s-%d", ranStr, ver);
     int len = strlen(newStr);
     ASSERT_EQ(pRead->pHead->head.len, len);
-    for(int j = 0; j < len; j++) {
+    for (int j = 0; j < len; j++) {
       EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
     }
   }