merge from 3.0

include/libs/nodes/nodes.h

@@ -62,6 +62,7 @@ typedef enum ENodeType {
   QUERY_NODE_NODE_LIST,
   QUERY_NODE_FILL,
   QUERY_NODE_COLUMN_REF,
+  QUERY_NODE_TARGET,
 
   // Only be used in parser module.
   QUERY_NODE_RAW_EXPR,
@@ -93,7 +94,7 @@ typedef struct SListCell {
 } SListCell;
 
 typedef struct SNodeList {
-  int16_t length;
+  int32_t length;
   SListCell* pHead;
   SListCell* pTail;
 } SNodeList;
@@ -103,6 +104,7 @@ void nodesDestroyNode(SNode* pNode);
 
 SNodeList* nodesMakeList();
 int32_t nodesListAppend(SNodeList* pList, SNode* pNode);
+int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
 SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
 SNode* nodesListGetNode(SNodeList* pList, int32_t index);
 void nodesDestroyList(SNodeList* pList);

include/libs/nodes/plannodes.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_PLANN_NODES_H_
+#define _TD_PLANN_NODES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "querynodes.h"
+
+typedef struct SLogicNode {
+  ENodeType type;
+  int32_t id;
+  SNodeList* pTargets; // SColumnNode
+  SNode* pConditions;
+  SNodeList* pChildren;
+  struct SLogicNode* pParent;
+} SLogicNode;
+
+typedef struct SScanLogicNode {
+  SLogicNode node;
+  SNodeList* pScanCols;
+  struct STableMeta* pMeta;
+} SScanLogicNode;
+
+typedef struct SJoinLogicNode {
+  SLogicNode node;
+  EJoinType joinType;
+  SNode* pOnConditions;
+} SJoinLogicNode;
+
+typedef struct SFilterLogicNode {
+  SLogicNode node;
+} SFilterLogicNode;
+
+typedef struct SAggLogicNode {
+  SLogicNode node;
+  SNodeList* pGroupKeys;
+  SNodeList* pAggFuncs;
+} SAggLogicNode;
+
+typedef struct SProjectLogicNode {
+  SLogicNode node;
+  SNodeList* pProjections;
+} SProjectLogicNode;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_PLANN_NODES_H_*/

include/libs/nodes/querynodes.h

@@ -62,8 +62,10 @@ typedef struct SColumnNode {
 
 typedef struct SColumnRefNode {
   ENodeType type;
-  int32_t tupleId;
-  int32_t slotId;
+  SDataType dataType;
+  int16_t tupleId;
+  int16_t slotId;
+  int16_t columnId;
 } SColumnRefNode;
 
 typedef struct SValueNode {
@@ -106,6 +108,12 @@ typedef enum EOperatorType {
   OP_TYPE_NMATCH,
   OP_TYPE_IS_NULL,
   OP_TYPE_IS_NOT_NULL,
+  OP_TYPE_IS_TRUE,
+  OP_TYPE_IS_FALSE,
+  OP_TYPE_IS_UNKNOWN,
+  OP_TYPE_IS_NOT_TRUE,
+  OP_TYPE_IS_NOT_FALSE,
+  OP_TYPE_IS_NOT_UNKNOWN,
 
   // json operator
   OP_TYPE_JSON_GET_VALUE,
@@ -285,7 +293,7 @@ typedef enum ESqlClause {
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 
-int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, uint64_t tableId, bool realCol, SNodeList** pCols);
+int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols);
 
 typedef bool (*FFuncClassifier)(int32_t funcId);
 int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs);

source/libs/tdb/src/inc/tdb_db.h → include/libs/parser/newParser.h

@@ -13,33 +13,31 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TDB_DB_H_
-#define _TD_TDB_DB_H_
-
-#include "tdb_mpool.h"
+#ifndef _TD_NEW_PARSER_H_
+#define _TD_NEW_PARSER_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct TDB TDB;
+#include "parser.h"
+
+typedef enum EStmtType {
+  STMT_TYPE_CMD = 1,
+  STMT_TYPE_QUERY
+} EStmtType;
 
-struct TDB {
-  char *      fname;
-  char *      dbname;
-  TDB_MPFILE *mpf;
-  // union {
-  //   TDB_BTREE *btree;
-  //   TDB_HASH * hash;
-  //   TDB_HEAP * heap;
-  // } dbam;  // db access method
-};
+typedef struct SQuery {
+  EStmtType stmtType;
+  SNode* pRoot;
+  int32_t numOfResCols;
+  SSchema* pResSchema;
+} SQuery;
 
-int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags);
-int tdbClose(TDB *dbp, uint32_t flags);
+int32_t parser(SParseContext* pParseCxt, SQuery* pQuery);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TDB_DB_H_*/
\ No newline at end of file
+#endif /*_TD_NEW_PARSER_H_*/

include/util/tdef.h

@@ -94,21 +94,20 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_TIME_PRECISION_MICRO_STR "us"
 #define TSDB_TIME_PRECISION_NANO_STR  "ns"
 
-#define TSDB_TICK_PER_SECOND(precision)                      \
-  ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \
-                                                      : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L)))
+#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L)))
 
 #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member)
 #define T_APPEND_MEMBER(dst, ptr, type, member) \
-  do {                                                                              \
-    memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member)); \
-    dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));                    \
-  } while (0)
+do {\
+  memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\
+  dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\
+} while(0)
 #define T_READ_MEMBER(src, type, target) \
-  do {                                            \
+do { \
   (target) = *(type *)(src); \
-    (src) = (void *)((char *)src + sizeof(type)); \
-  } while (0)
+  (src) = (void *)((char *)src + sizeof(type));\
+} while(0)
+
 
 // TODO: check if below is necessary
 #define TSDB_RELATION_INVALID     0
@@ -160,7 +159,7 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_ACCT_ID_LEN          11
 
 #define TSDB_MAX_COLUMNS          4096
-#define TSDB_MIN_COLUMNS 2  // PRIMARY COLUMN(timestamp) + other columns
+#define TSDB_MIN_COLUMNS          2       //PRIMARY COLUMN(timestamp) + other columns
 
 #define TSDB_NODE_NAME_LEN        64
 #define TSDB_TABLE_NAME_LEN       193     // it is a null-terminated string
@@ -185,11 +184,11 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
 #define TSDB_MAX_SQL_SHOW_LEN     1024
-#define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u)  // sql length should be less than 1mb
+#define TSDB_MAX_ALLOWED_SQL_LEN  (1*1024*1024u)          // sql length should be less than 1mb
 
 #define TSDB_APP_NAME_LEN         TSDB_UNI_LEN
 #define TSDB_STB_COMMENT_LEN      1024
-/**
+  /**
    *  In some scenarios uint16_t (0~65535) is used to store the row len.
    *  - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
    *  - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus
@@ -216,8 +215,8 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_SLOW_QUERY_SQL_LEN   512
 
 #define TSDB_TRANS_STAGE_LEN      12
-#define TSDB_TRANS_DESC_LEN  16
-#define TSDB_TRANS_ERROR_LEN 128
+#define TSDB_TRANS_TYPE_LEN       16
+#define TSDB_TRANS_ERROR_LEN      64
 
 #define TSDB_STEP_NAME_LEN        32
 #define TSDB_STEP_DESC_LEN        128
@@ -236,7 +235,7 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_DB_TYPE_DEFAULT      0
 #define TSDB_DB_TYPE_TOPIC        1
 
-#define TSDB_DEFAULT_PKT_SIZE 65480  // same as RPC_MAX_UDP_SIZE
+#define TSDB_DEFAULT_PKT_SIZE     65480  //same as RPC_MAX_UDP_SIZE
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120   // default payload size, greater than PATH_MAX value
@@ -324,8 +323,8 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_MAX_UNION_CLAUSE           5
 
 #define TSDB_MAX_FIELD_LEN              16384
-#define TSDB_MAX_BINARY_LEN            (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE)  // keep 16384
-#define TSDB_MAX_NCHAR_LEN             (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE)  // keep 16384
+#define TSDB_MAX_BINARY_LEN            (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
+#define TSDB_MAX_NCHAR_LEN             (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
 #define PRIMARYKEY_TIMESTAMP_COL_ID     1
 #define COL_REACH_END(colId, maxColId)  ((colId) > (maxColId))
 
@@ -334,8 +333,13 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_QUERY_TYPE_NON_TYPE        0x00u     // none type
 #define TSDB_QUERY_TYPE_FREE_RESOURCE   0x01u     // free qhandle at vnode
 
+#define TSDB_QUERY_TYPE_NON_TYPE      0x00u  // none type
+#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u  // free qhandle at vnode
+
 #define TSDB_META_COMPACT_RATIO         0       // disable tsdb meta compact by default
 
+
+
 /*
  * 1. ordinary sub query for select * from super_table
  * 2. all sqlobj generated by createSubqueryObj with this flag
@@ -374,7 +378,7 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_DEFAULT_STABLES_HASH_SIZE         100
 #define TSDB_DEFAULT_CTABLES_HASH_SIZE         20000
 
-#define TSDB_MAX_WAL_SIZE (1024 * 1024 * 3)
+#define TSDB_MAX_WAL_SIZE    (1024*1024*3)
 
 #define TSDB_ARB_DUMMY_TIME                    4765104000000 // 2121-01-01 00:00:00.000, :P
 

include/util/tjson.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_JSON_H_
+#define _TD_UTIL_JSON_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+typedef void SJson;
+
+SJson* tjsonCreateObject();
+void tjsonDelete(SJson* pJson);
+
+SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName);
+
+int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number);
+int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal);
+int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem);
+int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
+
+typedef int32_t (*FToJson)(const void* pObj, SJson* pJson);
+
+int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj);
+int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj);
+
+typedef int32_t (*FFromJson)(const SJson* pJson, void* pObj);
+
+char* tjsonToString(const SJson* pJson);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_JSON_H_*/

source/dnode/mgmt/impl/src/dndTransport.c

@@ -104,6 +104,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessMgmtMsg;
+  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_TRANS)] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT)] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GRANT_RSP)] = dndProcessMgmtMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_AUTH)] = dndProcessMnodeReadMsg;

source/dnode/mnode/impl/src/mndSync.c

@@ -85,6 +85,8 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
   mDebug("restore sdb wal finished, sdb ver:%" PRId64, sdbVer);
 
   mndTransPullup(pMnode);
+  sdbVer = sdbUpdateVer(pSdb, 0);
+  mDebug("pullup trans finished, sdb ver:%" PRId64, sdbVer);
 
   if (sdbVer != lastSdbVer) {
     mInfo("sdb restored from %" PRId64 " to %" PRId64 ", write file", lastSdbVer, sdbVer);

source/dnode/mnode/impl/src/mndTrans.c

@@ -221,7 +221,6 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
   pTrans = sdbGetRowObj(pRow);
   if (pTrans == NULL) goto TRANS_DECODE_OVER;
 
-
   SDB_GET_INT32(pRaw, dataPos, &pTrans->id, TRANS_DECODE_OVER)
 
   int16_t type = 0;
@@ -353,8 +352,60 @@ static const char *mndTransStr(ETrnStage stage) {
 
 static const char *mndTransType(ETrnType type) {
   switch (type) {
+    case TRN_TYPE_CREATE_USER:
+      return "create-user";
+    case TRN_TYPE_ALTER_USER:
+      return "alter-user";
+    case TRN_TYPE_DROP_USER:
+      return "drop-user";
+    case TRN_TYPE_CREATE_FUNC:
+      return "create-func";
+    case TRN_TYPE_DROP_FUNC:
+      return "drop-func";
+    case TRN_TYPE_CREATE_SNODE:
+      return "create-snode";
+    case TRN_TYPE_DROP_SNODE:
+      return "drop-snode";
+    case TRN_TYPE_CREATE_QNODE:
+      return "create-qnode";
+    case TRN_TYPE_DROP_QNODE:
+      return "drop-qnode";
+    case TRN_TYPE_CREATE_BNODE:
+      return "create-bnode";
+    case TRN_TYPE_DROP_BNODE:
+      return "drop-bnode";
+    case TRN_TYPE_CREATE_MNODE:
+      return "create-mnode";
+    case TRN_TYPE_DROP_MNODE:
+      return "drop-mnode";
+    case TRN_TYPE_CREATE_TOPIC:
+      return "create-topic";
+    case TRN_TYPE_DROP_TOPIC:
+      return "drop-topic";
+    case TRN_TYPE_SUBSCRIBE:
+      return "subscribe";
+    case TRN_TYPE_REBALANCE:
+      return "rebalance";
+    case TRN_TYPE_CREATE_DNODE:
+      return "create-qnode";
+    case TRN_TYPE_DROP_DNODE:
+      return "drop-qnode";
     case TRN_TYPE_CREATE_DB:
       return "create-db";
+    case TRN_TYPE_ALTER_DB:
+      return "alter-db";
+    case TRN_TYPE_DROP_DB:
+      return "drop-db";
+    case TRN_TYPE_SPLIT_VGROUP:
+      return "split-vgroup";
+    case TRN_TYPE_MERGE_VGROUP:
+      return "merge-vgroup";
+    case TRN_TYPE_CREATE_STB:
+      return "create-stb";
+    case TRN_TYPE_ALTER_STB:
+      return "alter-stb";
+    case TRN_TYPE_DROP_STB:
+      return "drop-stb";
     default:
       return "invalid";
   }
@@ -391,6 +442,11 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
     mTrace("trans:%d, stage from %s to %s", pNew->id, mndTransStr(TRN_STAGE_COMMIT), mndTransStr(TRN_STAGE_COMMIT_LOG));
   }
 
+  if (pNew->stage == TRN_STAGE_ROLLBACK) {
+    pNew->stage = TRN_STAGE_FINISHED;
+    mTrace("trans:%d, stage from %s to %s", pNew->id, mndTransStr(TRN_STAGE_ROLLBACK), mndTransStr(TRN_STAGE_FINISHED));
+  }
+
   mTrace("trans:%d, perform update action, old row:%p stage:%s, new row:%p stage:%s", pOld->id, pOld,
          mndTransStr(pOld->stage), pNew, mndTransStr(pNew->stage));
   pOld->stage = pNew->stage;
@@ -423,6 +479,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S
   pTrans->stage = TRN_STAGE_PREPARE;
   pTrans->policy = policy;
   pTrans->transType = type;
+  pTrans->createdTime = taosGetTimestampMs();
   pTrans->rpcHandle = pReq->handle;
   pTrans->rpcAHandle = pReq->ahandle;
   pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *));
@@ -754,6 +811,9 @@ void mndTransProcessRsp(SMnodeMsg *pRsp) {
   if (pAction != NULL) {
     pAction->msgReceived = 1;
     pAction->errCode = pRsp->rpcMsg.code;
+    if (pAction->errCode != 0) {
+      tstrncpy(pTrans->lastError, tstrerror(pAction->errCode), TSDB_TRANS_ERROR_LEN);
+    }
   }
 
   mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%x", transId, action, pRsp->rpcMsg.code,
@@ -1059,6 +1119,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
   bool continueExec = true;
 
   while (continueExec) {
+    pTrans->lastExecTime = taosGetTimestampMs();
     switch (pTrans->stage) {
       case TRN_STAGE_PREPARE:
         continueExec = mndTransPerformPrepareStage(pMnode, pTrans);
@@ -1170,10 +1231,9 @@ static int32_t mndProcessKillTransReq(SMnodeMsg *pReq) {
   }
 
   code = mndKillTrans(pMnode, pTrans);
-  if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
 KILL_OVER:
-  if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
+  if (code != 0) {
     mError("trans:%d, failed to kill since %s", killReq.transId, terrstr());
     return -1;
   }
@@ -1228,7 +1288,7 @@ static int32_t mndGetTransMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
   pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
 
-  pShow->bytes[cols] = (TSDB_TRANS_DESC_LEN - 1) + VARSTR_HEADER_SIZE;
+  pShow->bytes[cols] = TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "type");
   pSchema[cols].bytes = pShow->bytes[cols];
@@ -1288,11 +1348,7 @@ static int32_t mndRetrieveTrans(SMnodeMsg *pReq, SShowObj *pShow, char *data, in
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     char *name = mnGetDbStr(pTrans->dbname);
-    if (name != NULL) {
     STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]);
-    } else {
-      STR_TO_VARSTR(pWrite, "-");
-    }
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

source/dnode/mnode/impl/test/trans/trans.cpp

@@ -28,7 +28,7 @@ class MndTestTrans : public ::testing::Test {
   static void KillThenRestartServer() {
     char    file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data";
     FileFd  fd = taosOpenFileRead(file);
-    int32_t size = 1024 * 1024;
+    int32_t size = 3 * 1024 * 1024;
     void*   buffer = malloc(size);
     int32_t readLen = taosReadFile(fd, buffer, size);
     if (readLen < 0 || readLen == size) {
@@ -61,6 +61,37 @@ class MndTestTrans : public ::testing::Test {
 Testbase   MndTestTrans::test;
 TestServer MndTestTrans::server2;
 
+TEST_F(MndTestTrans, 00_Create_User_Crash) {
+  {
+    test.SendShowMetaReq(TSDB_MGMT_TABLE_TRANS, "");
+    CHECK_META("show trans", 7);
+
+    CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "id");
+    CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time");
+    CHECK_SCHEMA(2, TSDB_DATA_TYPE_BINARY, TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, "stage");
+    CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "db");
+    CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE, "type");
+    CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "last_exec_time");
+    CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, TSDB_TRANS_ERROR_LEN - 1 + VARSTR_HEADER_SIZE, "last_error");
+
+    test.SendShowRetrieveReq();
+    EXPECT_EQ(test.GetShowRows(), 0);
+  }
+
+  {
+    SKillTransReq killReq = {0};
+    killReq.transId = 3;
+
+    int32_t contLen = tSerializeSKillTransReq(NULL, 0, &killReq);
+    void*   pReq = rpcMallocCont(contLen);
+    tSerializeSKillTransReq(pReq, contLen, &killReq);
+
+    SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_TRANS, pReq, contLen);
+    ASSERT_NE(pRsp, nullptr);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TRANS_NOT_EXIST);
+  }
+}
+
 TEST_F(MndTestTrans, 01_Create_User_Crash) {
   {
     SCreateUserReq createReq = {0};
@@ -171,6 +202,57 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) {
     ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL);
   }
 
+  {
+    // show trans
+    test.SendShowMetaReq(TSDB_MGMT_TABLE_TRANS, "");
+    CHECK_META("show trans", 7);
+    test.SendShowRetrieveReq();
+
+    EXPECT_EQ(test.GetShowRows(), 1);
+    CheckInt32(4);
+    CheckTimestamp();
+    CheckBinary("undoAction", TSDB_TRANS_STAGE_LEN);
+    CheckBinary("", TSDB_DB_NAME_LEN - 1);
+    CheckBinary("create-qnode", TSDB_TRANS_TYPE_LEN);
+    CheckTimestamp();
+    CheckBinary("Unable to establish connection", TSDB_TRANS_ERROR_LEN - 1);
+  }
+
+  // kill trans
+  {
+    SKillTransReq killReq = {0};
+    killReq.transId = 4;
+
+    int32_t contLen = tSerializeSKillTransReq(NULL, 0, &killReq);
+    void*   pReq = rpcMallocCont(contLen);
+    tSerializeSKillTransReq(pReq, contLen, &killReq);
+
+    SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_TRANS, pReq, contLen);
+    ASSERT_NE(pRsp, nullptr);
+    ASSERT_EQ(pRsp->code, 0);
+  }
+
+  // show trans
+  {
+    test.SendShowMetaReq(TSDB_MGMT_TABLE_TRANS, "");
+    test.SendShowRetrieveReq();
+    EXPECT_EQ(test.GetShowRows(), 0);
+  }
+
+  // re-create trans
+  {
+    SMCreateQnodeReq createReq = {0};
+    createReq.dnodeId = 2;
+
+    int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq);
+    void*   pReq = rpcMallocCont(contLen);
+    tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq);
+
+    SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen);
+    ASSERT_NE(pRsp, nullptr);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+  }
+
   KillThenRestartServer();
 
   server2.DoStart();
@@ -201,3 +283,11 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) {
     EXPECT_EQ(test.GetShowRows(), 2);
   }
 }
+
+// create db
+// partial create stb
+// drop db failed
+// create stb failed
+// start
+// create stb success
+// drop db success

source/dnode/vnode/CMakeLists.txt

+set(META_DB_IMPL_LIST "BDB" "TDB")
+set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation")
+set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST})
+
+if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)
+    message(STATUS "META DB Impl: ${META_DB_IMPL}==============")
+else()
+    message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============")
+endif()
+
 aux_source_directory(src/meta META_SRC)
+if(${META_DB_IMPL} STREQUAL "BDB")
+    list(REMOVE_ITEM META_SRC "src/meta/metaTDBImpl.c")
+elseif(${META_DB_IMPL} STREQUAL "TDB")
+    list(REMOVE_ITEM META_SRC "src/meta/metaBDBImpl.c")
+endif()
+
 aux_source_directory(src/tq TQ_SRC)
 aux_source_directory(src/tsdb TSDB_SRC)
 aux_source_directory(src/vnd VND_SRC)
+
 list(APPEND
     VNODE_SRC
     ${META_SRC}
@@ -22,7 +39,6 @@ target_link_libraries(
     PUBLIC util
     PUBLIC common
     PUBLIC transport
-    PUBLIC bdb
     PUBLIC tfs
     PUBLIC wal
     PUBLIC scheduler
@@ -31,6 +47,12 @@ target_link_libraries(
     PUBLIC sync
 )
 
+if(${META_DB_IMPL} STREQUAL "BDB")
+    target_link_libraries(vnode PUBLIC bdb)
+elseif(${META_DB_IMPL} STREQUAL "TDB")
+    target_link_libraries(vnode PUBLIC tdb)
+endif()
+
 if(${BUILD_TEST})
     # add_subdirectory(test)
 endif(${BUILD_TEST})

source/dnode/vnode/src/inc/vnd.h

@@ -23,8 +23,8 @@
 #include "tlist.h"
 #include "tlockfree.h"
 #include "tmacro.h"
-#include "wal.h"
 #include "tq.h"
+#include "wal.h"
 
 #include "vnode.h"
 
@@ -175,7 +175,6 @@ void*           vmaMalloc(SVMemAllocator* pVMA, uint64_t size);
 void            vmaFree(SVMemAllocator* pVMA, void* ptr);
 bool            vmaIsFull(SVMemAllocator* pVMA);
 
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/meta/metaTDBImpl.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaDef.h"
+
+#include "tdb.h"
+
+struct SMetaDB {
+  TENV *pEnv;
+  TDB * pTbDB;
+  TDB * pSchemaDB;
+  TDB * pNameIdx;
+  TDB * pStbIdx;
+  TDB * pNtbIdx;
+  TDB * pCtbIdx;
+  // tag index hash table
+  // suid+colid --> TDB *
+  struct {
+  } tagIdxHt;
+};
+
+#define A(op, flag)                 \
+  do {                              \
+    if ((ret = op) != 0) goto flag; \
+  } while (0)
+
+int metaOpenDB(SMeta *pMeta) {
+  SMetaDB *pDb;
+  TENV *   pEnv;
+  TDB *    pTbDB;
+  TDB *    pSchemaDB;
+  TDB *    pNameIdx;
+  TDB *    pStbIdx;
+  TDB *    pNtbIdx;
+  TDB *    pCtbIdx;
+  int      ret;
+
+  pDb = (SMetaDB *)calloc(1, sizeof(*pDb));
+  if (pDb == NULL) {
+    return -1;
+  }
+
+  // Create and open the ENV
+  A((tdbEnvCreate(&pEnv)), _err);
+#if 0
+  // Set options of the environment
+  A(tdbEnvSetPageSize(pEnv, 8192), _err);
+  A(tdbEnvSetCacheSize(pEnv, 16 * 1024 * 1024), _err);
+#endif
+  A((tdbEnvOpen(&pEnv)), _err);
+
+  // Create and open each DB
+  A(tdbCreate(&pTbDB), _err);
+  A(tdbOpen(&pTbDB, "table.db", NULL, pEnv), _err);
+
+  A(tdbCreate(&pSchemaDB), _err);
+  A(tdbOpen(&pSchemaDB, "schema.db", NULL, pEnv), _err);
+
+  A(tdbCreate(&pNameIdx), _err);
+  A(tdbOpen(&pNameIdx, "name.db", NULL, pEnv), _err);
+  // tdbAssociate();
+
+  pDb->pEnv = pEnv;
+  pDb->pTbDB = pTbDB;
+  pDb->pSchemaDB = pSchemaDB;
+  pMeta->pDB = pDb;
+  return 0;
+
+_err:
+  return -1;
+}
+
+void metaCloseDB(SMeta *pMeta) {
+  // TODO
+}
+
+int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
+  // TODO
+  return 0;
+}
+
+int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
+  // TODO
+  return 0;
+}
+
+STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
+  // TODO
+  return NULL;
+}
+
+STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
+  // TODO
+  return NULL;
+}
+
+SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
+  // TODO
+  return NULL;
+}
+
+STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
+  // TODO
+  return NULL;
+}
+
+SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
+  // TODO
+  return NULL;
+}
+
+void metaCloseTbCursor(SMTbCursor *pTbCur) {
+  // TODO
+}
+
+char *metaTbCursorNext(SMTbCursor *pTbCur) {
+  // TODO
+  return NULL;
+}
+
+SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
+  // TODO
+  return NULL;
+}
+
+void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
+  // TODO
+}
+
+tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/libs/nodes/CMakeLists.txt

@@ -7,7 +7,7 @@ target_include_directories(
 )
 target_link_libraries(
     nodes
-    PRIVATE os util
+    PRIVATE os util common qcom
 )
 
 if(${BUILD_TEST})

source/libs/nodes/src/nodesCodeFuncs.c

@@ -13,11 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "nodes.h"
+#include "plannodes.h"
+#include "querynodes.h"
+#include "query.h"
+#include "taoserror.h"
+#include "tjson.h"
 
-int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) {
-  switch (nodeType(pNode)) {
+static int32_t nodeToJson(const void* pObj, SJson* pJson);
+
+static char* nodeName(ENodeType type) {
+  switch (type) {
     case QUERY_NODE_COLUMN:
+      return "Column";    
     case QUERY_NODE_VALUE:
     case QUERY_NODE_OPERATOR:
     case QUERY_NODE_LOGIC_CONDITION:
@@ -31,14 +38,230 @@ int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) {
     case QUERY_NODE_STATE_WINDOW:
     case QUERY_NODE_SESSION_WINDOW:
     case QUERY_NODE_INTERVAL_WINDOW:
+    case QUERY_NODE_NODE_LIST:
+    case QUERY_NODE_FILL:
+    case QUERY_NODE_COLUMN_REF:
+    case QUERY_NODE_TARGET:
+    case QUERY_NODE_RAW_EXPR:
     case QUERY_NODE_SET_OPERATOR:
     case QUERY_NODE_SELECT_STMT:
     case QUERY_NODE_SHOW_STMT:
+      break;
+    case QUERY_NODE_LOGIC_PLAN_SCAN:
+      return "LogicScan";
+    case QUERY_NODE_LOGIC_PLAN_JOIN:
+      return "LogicJoin";
+    case QUERY_NODE_LOGIC_PLAN_FILTER:
+      return "LogicFilter";
+    case QUERY_NODE_LOGIC_PLAN_AGG:
+      return "LogicAgg";
+    case QUERY_NODE_LOGIC_PLAN_PROJECT:
+      return "LogicProject";
     default:
       break;
   }
+  return "Unknown";
 }
 
-int32_t nodesStringToNode(const char* pStr, SNode** pNode) {
+static int32_t addNodeList(SJson* pJson, const char* pName, FToJson func, const SNodeList* pList) {
+  if (LIST_LENGTH(pList) > 0) {
+    SJson* jList = tjsonAddArrayToObject(pJson, pName);
+    if (NULL == jList) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    SNode* pNode;
+    FOREACH(pNode, pList) {
+      int32_t code = tjsonAddItem(jList, func, pNode);
+      if (TSDB_CODE_SUCCESS != code) {
+        return code;
+      }
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static const char* jkTableMetaUid = "TableMetaUid";
+static const char* jkTableMetaSuid = "TableMetaSuid";
+
+static int32_t tableMetaToJson(const void* pObj, SJson* pJson) {
+  const STableMeta* pNode = (const STableMeta*)pObj;
+
+  int32_t code = tjsonAddIntegerToObject(pJson, jkTableMetaUid, pNode->uid);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkTableMetaSuid, pNode->suid);
+  }
+
+  return code;
+}
+
+static const char* jkLogicPlanId = "Id";
+static const char* jkLogicPlanTargets = "Targets";
+static const char* jkLogicPlanConditions = "Conditions";
+static const char* jkLogicPlanChildren = "Children";
+
+static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
+  const SLogicNode* pNode = (const SLogicNode*)pObj;
+
+  int32_t code = tjsonAddIntegerToObject(pJson, jkLogicPlanId, pNode->id);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addNodeList(pJson, jkLogicPlanTargets, nodeToJson, pNode->pTargets);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addNodeList(pJson, jkLogicPlanChildren, nodeToJson, pNode->pChildren);
+  }
+
+  return code;
+}
+
+static const char* jkScanLogicPlanScanCols = "ScanCols";
+static const char* jkScanLogicPlanTableMeta = "TableMeta";
+
+static int32_t logicScanToJson(const void* pObj, SJson* pJson) {
+  const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
+
+  int32_t code = logicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addNodeList(pJson, jkScanLogicPlanScanCols, nodeToJson, pNode->pScanCols);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta);
+  }
+
+  return code;
+}
+
+static const char* jkProjectLogicPlanProjections = "Projections";
+
+static int32_t logicProjectToJson(const void* pObj, SJson* pJson) {
+  const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj;
+
+  int32_t code = logicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addNodeList(pJson, jkProjectLogicPlanProjections, nodeToJson, pNode->pProjections);
+  }
+
+  return code;
+}
+
+static const char* jkJoinLogicPlanJoinType = "JoinType";
+static const char* jkJoinLogicPlanOnConditions = "OnConditions";
+
+static int32_t logicJoinToJson(const void* pObj, SJson* pJson) {
+  const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj;
+
+  int32_t code = logicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkJoinLogicPlanJoinType, pNode->joinType);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions);
+  }
+
+  return code;
+}
+
+static int32_t logicFilterToJson(const void* pObj, SJson* pJson) {
+  return logicPlanNodeToJson(pObj, pJson);
+}
+
+static const char* jkAggLogicPlanGroupKeys = "GroupKeys";
+static const char* jkAggLogicPlanAggFuncs = "AggFuncs";
 
+static int32_t logicAggToJson(const void* pObj, SJson* pJson) {
+  const SAggLogicNode* pNode = (const SAggLogicNode*)pObj;
+
+  int32_t code = logicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addNodeList(pJson, jkAggLogicPlanGroupKeys, nodeToJson, pNode->pGroupKeys);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addNodeList(pJson, jkAggLogicPlanAggFuncs, nodeToJson, pNode->pAggFuncs);
+  }
+
+  return code;
+}
+
+static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
+  switch (nodeType(pObj)) {
+    case QUERY_NODE_COLUMN:
+    case QUERY_NODE_VALUE:
+    case QUERY_NODE_OPERATOR:
+    case QUERY_NODE_LOGIC_CONDITION:
+    case QUERY_NODE_FUNCTION:
+    case QUERY_NODE_REAL_TABLE:
+    case QUERY_NODE_TEMP_TABLE:
+    case QUERY_NODE_JOIN_TABLE:
+    case QUERY_NODE_GROUPING_SET:
+    case QUERY_NODE_ORDER_BY_EXPR:
+    case QUERY_NODE_LIMIT:
+    case QUERY_NODE_STATE_WINDOW:
+    case QUERY_NODE_SESSION_WINDOW:
+    case QUERY_NODE_INTERVAL_WINDOW:
+    case QUERY_NODE_NODE_LIST:
+    case QUERY_NODE_FILL:
+    case QUERY_NODE_COLUMN_REF:
+    case QUERY_NODE_TARGET:
+    case QUERY_NODE_RAW_EXPR:
+    case QUERY_NODE_SET_OPERATOR:
+    case QUERY_NODE_SELECT_STMT:
+    case QUERY_NODE_SHOW_STMT:
+      break;
+    case QUERY_NODE_LOGIC_PLAN_SCAN:
+      return logicScanToJson(pObj, pJson);
+    case QUERY_NODE_LOGIC_PLAN_JOIN:
+      return logicJoinToJson(pObj, pJson);
+    case QUERY_NODE_LOGIC_PLAN_FILTER:
+      return logicFilterToJson(pObj, pJson);
+    case QUERY_NODE_LOGIC_PLAN_AGG:
+      return logicAggToJson(pObj, pJson);
+    case QUERY_NODE_LOGIC_PLAN_PROJECT:
+      return logicProjectToJson(pObj, pJson);
+    default:
+      break;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static const char* jkNodeType = "Type";
+static int32_t nodeToJson(const void* pObj, SJson* pJson) {
+  const SNode* pNode = (const SNode*)pObj;
+
+  char* pNodeName = nodeName(nodeType(pNode));
+  int32_t code = tjsonAddStringToObject(pJson, jkNodeType, pNodeName);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, pNodeName, specificNodeToJson, pNode);
+  }
+
+  return code;
+}
+
+int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) {
+  if (NULL == pNode || NULL == pStr || NULL == pLen) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SJson* pJson = tjsonCreateObject();
+  if (NULL == pJson) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  int32_t code = nodeToJson(pNode, pJson);
+  if (TSDB_CODE_SUCCESS != code) {
+    terrno = code;
+    return code;
+  }
+
+  *pStr = tjsonToString(pJson);
+  tjsonDelete(pJson);
+
+  *pLen = strlen(*pStr) + 1;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t nodesStringToNode(const char* pStr, SNode** pNode) {
+  return TSDB_CODE_SUCCESS;
 }

source/libs/nodes/src/nodesUtilFuncs.c

@@ -14,7 +14,7 @@
  */
 
 #include "querynodes.h"
-#include "nodesShowStmts.h"
+#include "plannodes.h"
 #include "taos.h"
 #include "taoserror.h"
 #include "thash.h"
@@ -68,8 +68,18 @@ SNode* nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SSetOperator));
     case QUERY_NODE_SELECT_STMT:
       return makeNode(type, sizeof(SSelectStmt));
-    case QUERY_NODE_SHOW_STMT:
-      return makeNode(type, sizeof(SShowStmt));
+    // case QUERY_NODE_SHOW_STMT:
+    //   return makeNode(type, sizeof(SShowStmt));
+    case QUERY_NODE_LOGIC_PLAN_SCAN:
+      return makeNode(type, sizeof(SScanLogicNode));
+    case QUERY_NODE_LOGIC_PLAN_JOIN:
+      return makeNode(type, sizeof(SJoinLogicNode));
+    case QUERY_NODE_LOGIC_PLAN_FILTER:
+      return makeNode(type, sizeof(SFilterLogicNode));
+    case QUERY_NODE_LOGIC_PLAN_AGG:
+      return makeNode(type, sizeof(SAggLogicNode));
+    case QUERY_NODE_LOGIC_PLAN_PROJECT:
+      return makeNode(type, sizeof(SProjectLogicNode));
     default:
       break;
   }
@@ -121,6 +131,15 @@ int32_t nodesListAppend(SNodeList* pList, SNode* pNode) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) {
+  pTarget->pTail->pNext = pSrc->pHead;
+  pSrc->pHead->pPrev = pTarget->pTail;
+  pTarget->pTail = pSrc->pTail;
+  pTarget->length += pSrc->length;
+  tfree(pSrc);
+  return TSDB_CODE_SUCCESS;
+}
+
 SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) {
   if (NULL == pCell->pPrev) {
     pList->pHead = pCell->pNext;
@@ -129,6 +148,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) {
     pCell->pNext->pPrev = pCell->pPrev;
   }
   SListCell* pNext = pCell->pNext;
+  nodesDestroyNode(pCell->pNode);
   tfree(pCell);
   --(pList->length);
   return pNext;
@@ -185,6 +205,14 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
     case OP_TYPE_NOT_LIKE:
     case OP_TYPE_MATCH:
     case OP_TYPE_NMATCH:
+    case OP_TYPE_IS_NULL:
+    case OP_TYPE_IS_NOT_NULL:
+    case OP_TYPE_IS_TRUE:
+    case OP_TYPE_IS_FALSE:
+    case OP_TYPE_IS_UNKNOWN:
+    case OP_TYPE_IS_NOT_TRUE:
+    case OP_TYPE_IS_NOT_FALSE:
+    case OP_TYPE_IS_NOT_UNKNOWN:
       return true;
     default:
       break;
@@ -213,8 +241,7 @@ bool nodesIsTimelineQuery(const SNode* pQuery) {
 
 typedef struct SCollectColumnsCxt {
   int32_t errCode;
-  uint64_t tableId;
-  bool realCol;
+  const char* pTableAlias;
   SNodeList* pCols;
   SHashObj* pColIdHash;
 } SCollectColumnsCxt;
@@ -232,27 +259,24 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, int32_t id, SNode* pNode) {
 
 static EDealRes collectColumns(SNode* pNode, void* pContext) {
   SCollectColumnsCxt* pCxt = (SCollectColumnsCxt*)pContext;
-
-  if (pCxt->realCol && QUERY_NODE_COLUMN == nodeType(pNode)) {
+  if (QUERY_NODE_COLUMN == nodeType(pNode)) {
     SColumnNode* pCol = (SColumnNode*)pNode;
     int32_t colId = pCol->colId;
-    if (pCxt->tableId == pCol->tableId && colId > 0) {
+    if (0 == strcmp(pCxt->pTableAlias, pCol->tableAlias)) {
       return doCollect(pCxt, colId, pNode);
     }
-  } else if (!pCxt->realCol && QUERY_NODE_COLUMN_REF == nodeType(pNode)) {
-    return doCollect(pCxt, ((SColumnRefNode*)pNode)->slotId, pNode);
   }
   return DEAL_RES_CONTINUE;
 }
 
-int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, uint64_t tableId, bool realCol, SNodeList** pCols) {
+int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols) {
   if (NULL == pSelect || NULL == pCols) {
     return TSDB_CODE_SUCCESS;
   }
 
   SCollectColumnsCxt cxt = {
     .errCode = TSDB_CODE_SUCCESS,
-    .realCol = realCol,
+    .pTableAlias = pTableAlias,
     .pCols = nodesMakeList(),
     .pColIdHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK)
   };
@@ -303,6 +327,12 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod
     nodesDestroyList(cxt.pFuncs);
     return cxt.errCode;
   }
+  if (LIST_LENGTH(cxt.pFuncs) > 0) {
     *pFuncs = cxt.pFuncs;
+  } else {
+    nodesDestroyList(cxt.pFuncs);
+    *pFuncs = NULL;
+  }
+  
   return TSDB_CODE_SUCCESS;
 }

source/libs/parser/inc/parserImpl.h

@@ -13,27 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_AST_CREATE_FUNCS_H_
-#define _TD_AST_CREATE_FUNCS_H_
+#ifndef _TD_PARSER_IMPL_H_
+#define _TD_PARSER_IMPL_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #include "querynodes.h"
-#include "parser.h"
-
-typedef enum EStmtType {
-  STMT_TYPE_CMD = 1,
-  STMT_TYPE_QUERY
-} EStmtType;
-
-typedef struct SQuery {
-  EStmtType stmtType;
-  SNode* pRoot;
-  int32_t numOfResCols;
-  SSchema* pResSchema;
-} SQuery;
+#include "newParser.h"
 
 int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery);
 int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery);
@@ -42,4 +30,4 @@ int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery);
 }
 #endif
 
-#endif /*_TD_AST_CREATE_FUNCS_H_*/
+#endif /*_TD_PARSER_IMPL_H_*/

source/libs/parser/src/parserImpl.c

@@ -880,7 +880,6 @@ static int32_t translateOrderByPosition(STranslateContext* pCxt, SNodeList* pPro
       int32_t pos = getPositionValue(pVal);
       if (pos < 0) {
         ERASE_NODE(pOrderByList);
-        nodesDestroyNode(pNode);
         continue;
       } else if (0 == pos || pos > LIST_LENGTH(pProjectionList)) {
         return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT);
@@ -1058,3 +1057,11 @@ int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) {
   }
   return code;
 }
+
+int32_t parser(SParseContext* pParseCxt, SQuery* pQuery) {
+  int32_t code = doParse(pParseCxt, pQuery);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = doTranslate(pParseCxt, pQuery);
+  }
+  return code;
+}

source/libs/planner/inc/plannerImpl.h

@@ -20,43 +20,10 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
+#include "plannodes.h"
 #include "planner.h"
 
-typedef struct SLogicNode {
-  ENodeType type;
-  int32_t id;
-  SNodeList* pTargets;
-  SNode* pConditions;
-  SNodeList* pChildren;
-  struct SLogicNode* pParent;
-} SLogicNode;
-
-typedef struct SScanLogicNode {
-  SLogicNode node;
-  SNodeList* pScanCols;
-  struct STableMeta* pMeta;
-} SScanLogicNode;
-
-typedef struct SJoinLogicNode {
-  SLogicNode node;
-  EJoinType joinType;
-  SNode* pOnConditions;
-} SJoinLogicNode;
-
-typedef struct SFilterLogicNode {
-  SLogicNode node;
-} SFilterLogicNode;
-
-typedef struct SAggLogicNode {
-  SLogicNode node;
-  SNodeList* pGroupKeys;
-  SNodeList* pAggFuncs;
-} SAggLogicNode;
-
-typedef struct SProjectLogicNode {
-  SLogicNode node;
-} SProjectLogicNode;
+int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode);
 
 #ifdef __cplusplus
 }

source/libs/planner/src/plannerImpl.c

@@ -19,6 +19,7 @@
 #define CHECK_ALLOC(p, res) \
   do { \
     if (NULL == p) { \
+      printf("%s : %d\n", __FUNCTION__, __LINE__); \
       pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \
       return res; \
     } \
@@ -28,6 +29,7 @@
   do { \
     int32_t code = exec; \
     if (TSDB_CODE_SUCCESS != code) { \
+      printf("%s : %d\n", __FUNCTION__, __LINE__); \
       pCxt->errCode = code; \
       return res; \
     } \
@@ -44,8 +46,7 @@ static SLogicNode* createLogicNodeByTable(SPlanContext* pCxt, SSelectStmt* pSele
 
 typedef struct SRewriteExprCxt {
   int32_t errCode;
-  int32_t planNodeId;
-  SNodeList* pTargets;
+  SNodeList* pExprs;
 } SRewriteExprCxt;
 
 static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
@@ -53,34 +54,44 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
     case QUERY_NODE_OPERATOR:
     case QUERY_NODE_LOGIC_CONDITION:
     case QUERY_NODE_FUNCTION: {
-      break;
-    }
-    default:
-      break;
-  }
       SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext;
-  SNode* pTarget;
+      SNode* pExpr;
       int32_t index = 0;
-  FOREACH(pTarget, pCxt->pTargets) {
-    if (nodesEqualNode(pTarget, *pNode)) {
-      SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF);
+      FOREACH(pExpr, pCxt->pExprs) {
+        if (nodesEqualNode(pExpr, *pNode)) {
+          SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
           if (NULL == pCol) {
             pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
             return DEAL_RES_ERROR;
           }
-      pCol->tupleId = pCxt->planNodeId;
-      pCol->slotId = index;
+          SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode);
+          pCol->node.resType = pToBeRewrittenExpr->resType;
+          strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName);
+          strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName);
           nodesDestroyNode(*pNode);
           *pNode = (SNode*)pCol;
           return DEAL_RES_IGNORE_CHILD;
         }
         ++index;
       }
+      break;
+    }
+    default:
+      break;
+  }
+
   return DEAL_RES_CONTINUE;
 }
 
-static int32_t rewriteExpr(int32_t planNodeId, SNodeList* pTargets, SSelectStmt* pSelect, ESqlClause clause) {
-  SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = planNodeId, .pTargets = pTargets };
+static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
+  SNode* pNode;
+  FOREACH(pNode, pExprs) {
+    if (QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_VALUE == nodeType(pNode)) {
+      continue;
+    }
+    sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", planNodeId, rewriteId);
+  }
+  SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
   nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
   return cxt.errCode;
 }
@@ -114,23 +125,6 @@ error:
   return pRoot;
 }
 
-static SNodeList* createScanTargets(int32_t planNodeId, int32_t numOfScanCols) {
-  SNodeList* pTargets = nodesMakeList();
-  if (NULL == pTargets) {
-    return NULL;
-  }
-  for (int32_t i = 0; i < numOfScanCols; ++i) {
-    SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF);
-    if (NULL == pCol || TSDB_CODE_SUCCESS != nodesListAppend(pTargets, (SNode*)pCol)) {
-      nodesDestroyList(pTargets);
-      return NULL;
-    }
-    pCol->tupleId = planNodeId;
-    pCol->slotId = i;
-  }
-  return pTargets;
-}
-
 static SLogicNode* createScanLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable) {
   SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
   CHECK_ALLOC(pScan, NULL);
@@ -140,22 +134,25 @@ static SLogicNode* createScanLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect,
 
   // set columns to scan
   SNodeList* pCols = NULL;
-  CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pScan->pMeta->uid, true, &pCols), (SLogicNode*)pScan);
+  CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan);
   pScan->pScanCols = nodesCloneList(pCols);
   CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan);
 
-  // pScanCols of SScanLogicNode is equivalent to pTargets of other logic nodes
-  CHECK_CODE(rewriteExpr(pScan->node.id, pScan->pScanCols, pSelect, SQL_CLAUSE_FROM), (SLogicNode*)pScan);
-
   // set output
-  pScan->node.pTargets = createScanTargets(pScan->node.id, LIST_LENGTH(pScan->pScanCols));
+  pScan->node.pTargets = nodesCloneList(pCols);
   CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan);
 
   return (SLogicNode*)pScan;
 }
 
 static SLogicNode* createSubqueryLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable) {
-  return createQueryLogicNode(pCxt, pTable->pSubquery);
+  SLogicNode* pRoot = createQueryLogicNode(pCxt, pTable->pSubquery);
+  CHECK_ALLOC(pRoot, NULL);
+  SNode* pNode;
+  FOREACH(pNode, pRoot->pTargets) {
+    strcpy(((SColumnNode*)pNode)->tableAlias, pTable->table.tableAlias);
+  }
+  return pRoot;
 }
 
 static SLogicNode* createJoinLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect, SJoinTableNode* pJoinTable) {
@@ -179,12 +176,12 @@ static SLogicNode* createJoinLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect,
   pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond);
   CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin);
 
-  // set the output and rewrite the expression in subsequent clauses with the output
-  SNodeList* pCols = NULL;
-  CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, 0, false, &pCols), (SLogicNode*)pJoin);
-  pJoin->node.pTargets = nodesCloneList(pCols);
+  // set the output
+  pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
   CHECK_ALLOC(pJoin->node.pTargets, (SLogicNode*)pJoin);
-  CHECK_CODE(rewriteExpr(pJoin->node.id, pJoin->node.pTargets, pSelect, SQL_CLAUSE_FROM), (SLogicNode*)pJoin);
+  SNodeList* pTargets = nodesCloneList(pRight->pTargets);
+  CHECK_ALLOC(pTargets, (SLogicNode*)pJoin);
+  nodesListAppendList(pJoin->node.pTargets, pTargets);
 
   return (SLogicNode*)pJoin;
 }
@@ -203,7 +200,7 @@ static SLogicNode* createLogicNodeByTable(SPlanContext* pCxt, SSelectStmt* pSele
   return NULL;
 }
 
-static SLogicNode* createWhereFilterLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) {
+static SLogicNode* createWhereFilterLogicNode(SPlanContext* pCxt, SLogicNode* pChild, SSelectStmt* pSelect) {
   if (NULL == pSelect->pWhere) {
     return NULL;
   }
@@ -216,16 +213,44 @@ static SLogicNode* createWhereFilterLogicNode(SPlanContext* pCxt, SSelectStmt* p
   pFilter->node.pConditions = nodesCloneNode(pSelect->pWhere);
   CHECK_ALLOC(pFilter->node.pConditions, (SLogicNode*)pFilter);
 
-  // set the output and rewrite the expression in subsequent clauses with the output
-  SNodeList* pCols = NULL;
-  CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_WHERE, 0, false, &pCols), (SLogicNode*)pFilter);
-  pFilter->node.pTargets = nodesCloneList(pCols);
+  // set the output
+  pFilter->node.pTargets = nodesCloneList(pChild->pTargets);
   CHECK_ALLOC(pFilter->node.pTargets, (SLogicNode*)pFilter);
-  CHECK_CODE(rewriteExpr(pFilter->node.id, pFilter->node.pTargets, pSelect, SQL_CLAUSE_WHERE), (SLogicNode*)pFilter);
 
   return (SLogicNode*)pFilter;
 }
 
+static SNodeList* createColumnByRewriteExps(SPlanContext* pCxt, SNodeList* pExprs) {
+  SNodeList* pList = nodesMakeList();
+  CHECK_ALLOC(pList, NULL);
+  SNode* pNode;
+  FOREACH(pNode, pExprs) {
+    if (QUERY_NODE_VALUE == nodeType(pNode)) {
+      continue;
+    } else if (QUERY_NODE_COLUMN == nodeType(pNode)) {
+      SNode* pCol = nodesCloneNode(pNode);
+      if (NULL == pCol) {
+        goto error;
+      }
+      if (TSDB_CODE_SUCCESS != nodesListAppend(pList, pCol)) {
+        goto error;
+      }
+    } else {
+      SExprNode* pExpr = (SExprNode*)pNode;
+      SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+      if (NULL == pCol) {
+        goto error;
+      }
+      pCol->node.resType = pExpr->resType;
+      strcpy(pCol->colName, pExpr->aliasName);
+    }
+  }
+  return pList;
+error:
+  nodesDestroyList(pList);
+  return NULL;
+}
+
 static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) {
   SNodeList* pAggFuncs = NULL;
   CHECK_CODE(nodesCollectFuncs(pSelect, fmIsAggFunc, &pAggFuncs), NULL);
@@ -242,25 +267,54 @@ static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect)
   CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg);
   pAgg->pAggFuncs = nodesCloneList(pAggFuncs);
   CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg);
+
+  // rewrite the expression in subsequent clauses
+  CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg);
+  CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg);
+
   pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving);
   CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg);
 
-  // set the output and rewrite the expression in subsequent clauses with the output
-  SNodeList* pCols = NULL;
-  CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_HAVING, 0, false, &pCols), (SLogicNode*)pAgg);
-  pAgg->node.pTargets = nodesCloneList(pCols);
+  // set the output
+  pAgg->node.pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys);
   CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg);
-  CHECK_CODE(rewriteExpr(pAgg->node.id, pAgg->node.pTargets, pSelect, SQL_CLAUSE_HAVING), (SLogicNode*)pAgg);
+  SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs);
+  CHECK_ALLOC(pTargets, (SLogicNode*)pAgg);
+  nodesListAppendList(pAgg->node.pTargets, pTargets);
   
   return (SLogicNode*)pAgg;
 }
 
+static SNodeList* createColumnByProjections(SPlanContext* pCxt, SNodeList* pExprs) {
+  SNodeList* pList = nodesMakeList();
+  CHECK_ALLOC(pList, NULL);
+  SNode* pNode;
+  FOREACH(pNode, pExprs) {
+    SExprNode* pExpr = (SExprNode*)pNode;
+    SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+    if (NULL == pCol) {
+      goto error;
+    }
+    pCol->node.resType = pExpr->resType;
+    strcpy(pCol->colName, pExpr->aliasName);
+    if (TSDB_CODE_SUCCESS != nodesListAppend(pList, (SNode*)pCol)) {
+      goto error;
+    }
+  }
+  return pList;
+error:
+  nodesDestroyList(pList);
+  return NULL;
+}
+
 static SLogicNode* createProjectLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) {
   SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT);
   CHECK_ALLOC(pProject, NULL);
   pProject->node.id = pCxt->planNodeId++;
 
-  pProject->node.pTargets = nodesCloneList(pSelect->pProjectionList);
+  pProject->pProjections = nodesCloneList(pSelect->pProjectionList);
+
+  pProject->node.pTargets = createColumnByProjections(pCxt,pSelect->pProjectionList);
   CHECK_ALLOC(pProject->node.pTargets, (SLogicNode*)pProject);
 
   return (SLogicNode*)pProject;
@@ -269,7 +323,7 @@ static SLogicNode* createProjectLogicNode(SPlanContext* pCxt, SSelectStmt* pSele
 static SLogicNode* createSelectLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) {
   SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable);
   if (TSDB_CODE_SUCCESS == pCxt->errCode) {
-    pRoot = pushLogicNode(pCxt, pRoot, createWhereFilterLogicNode(pCxt, pSelect));
+    pRoot = pushLogicNode(pCxt, pRoot, createWhereFilterLogicNode(pCxt, pRoot, pSelect));
   }
   if (TSDB_CODE_SUCCESS == pCxt->errCode) {
     pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect));

source/libs/planner/test/CMakeLists.txt

@@ -13,7 +13,7 @@ ADD_EXECUTABLE(plannerTest
 
 TARGET_LINK_LIBRARIES(
         plannerTest
-        PUBLIC os util common planner parser catalog transport gtest function qcom
+        PUBLIC os util common nodes planner parser catalog transport gtest function qcom
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/libs/planner/test/newPlannerTest.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <algorithm>
+
+#include <gtest/gtest.h>
+
+#include "plannerImpl.h"
+#include "newParser.h"
+
+using namespace std;
+using namespace testing;
+
+class NewPlannerTest : public Test {
+protected:
+  void setDatabase(const string& acctId, const string& db) {
+    acctId_ = acctId;
+    db_ = db;
+  }
+
+  void bind(const char* sql) {
+    reset();
+    cxt_.acctId = atoi(acctId_.c_str());
+    cxt_.db = db_.c_str();
+    sqlBuf_ = string(sql);
+    transform(sqlBuf_.begin(), sqlBuf_.end(), sqlBuf_.begin(), ::tolower);
+    cxt_.sqlLen = strlen(sql);
+    cxt_.pSql = sqlBuf_.c_str();
+  }
+
+  bool run() {
+    int32_t code = parser(&cxt_, &query_);
+    // cout << "parser return " << code << endl;
+    if (code != TSDB_CODE_SUCCESS) {
+      cout << "sql:[" << cxt_.pSql << "] parser code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl;
+      return false;
+    }
+    SLogicNode* pLogicPlan = nullptr;
+    code = createLogicPlan(query_.pRoot, &pLogicPlan);
+    if (code != TSDB_CODE_SUCCESS) {
+      cout << "sql:[" << cxt_.pSql << "] plan code:" << tstrerror(code) << endl;
+      return false;
+    }
+    char* pStr = NULL;
+    int32_t len = 0;
+    code = nodesNodeToString((const SNode*)pLogicPlan, &pStr, &len);
+    if (code != TSDB_CODE_SUCCESS) {
+      cout << "sql:[" << cxt_.pSql << "] toString code:" << tstrerror(code) << endl;
+      return false;
+    }
+    cout << "logic plan : " << endl;
+    cout << pStr << endl;
+    return true;
+  }
+
+private:
+  static const int max_err_len = 1024;
+
+  void reset() {
+    memset(&cxt_, 0, sizeof(cxt_));
+    memset(errMagBuf_, 0, max_err_len);
+    cxt_.pMsg = errMagBuf_;
+    cxt_.msgLen = max_err_len;
+  }
+
+  string acctId_;
+  string db_;
+  char errMagBuf_[max_err_len];
+  string sqlBuf_;
+  SParseContext cxt_;
+  SQuery query_;
+};
+
+TEST_F(NewPlannerTest, simple) {
+  setDatabase("root", "test");
+
+  bind("SELECT * FROM t1");
+  ASSERT_TRUE(run());
+}

source/libs/tdb/inc/tdb.h

@@ -22,41 +22,44 @@
 extern "C" {
 #endif
 
-// #define TDB_EXTERN
-// #define TDB_PUBLIC
-// #define TDB_STATIC static
+typedef struct STDb       TDB;
+typedef struct STDbEnv    TENV;
+typedef struct STDbCurosr TDBC;
 
-// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t;
+typedef int32_t pgsz_t;
+typedef int32_t cachesz_t;
 
-// // Forward declarations
-// typedef struct TDB TDB;
-// // typedef struct TDB_MPOOL  TDB_MPOOL;
-// // typedef struct TDB_MPFILE TDB_MPFILE;
-// // typedef struct TDB_CURSOR TDB_CURSOR;
+typedef int (*TdbKeyCmprFn)(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
 
-// typedef struct {
-//   void*    bdata;
-//   uint32_t size;
-// } TDB_KEY, TDB_VALUE;
+// TEVN
+int tdbEnvCreate(TENV **ppEnv, const char *rootDir);
+int tdbEnvOpen(TENV *ppEnv);
+int tdbEnvClose(TENV *pEnv);
 
-// // TDB Operations
-// int tdbCreateDB(TDB** dbpp, tdb_db_t type);
-// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags);
-// int tdbCloseDB(TDB* dbp, uint32_t flags);
-// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags);
-// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags);
+int       tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize);
+pgsz_t    tdbEnvGetPageSize(TENV *pEnv);
+cachesz_t tdbEnvGetCacheSize(TENV *pEnv);
 
-// // TDB_MPOOL
-// int tdbOpenMPool(TDB_MPOOL** mp);
-// int tdbCloseMPool(TDB_MPOOL* mp);
+int tdbEnvBeginTxn(TENV *pEnv);
+int tdbEnvCommit(TENV *pEnv);
 
-// // TDB_MPFILE
-// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp);
-// int tdbCloseMPFile(TDB_MPFILE** mpf);
+// TDB
+int tdbCreate(TDB **ppDb);
+int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv);
+int tdbClose(TDB *pDb);
+int tdbDrop(TDB *pDb);
 
-// // TDB_CURSOR
-// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp);
-// int tdbCloseCurosr(TDB_CURSOR* tdbcp);
+int tdbSetKeyLen(TDB *pDb, int klen);
+int tdbSetValLen(TDB *pDb, int vlen);
+int tdbSetDup(TDB *pDb, int dup);
+int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn);
+int tdbGetKeyLen(TDB *pDb);
+int tdbGetValLen(TDB *pDb);
+int tdbGetDup(TDB *pDb);
+
+int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData);
+
+// TDBC
 
 #ifdef __cplusplus
 }

source/libs/tdb/src/db/tdb.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+struct STDb {
+  char         dbname[TDB_MAX_DBNAME_LEN];
+  SBTree *     pBt;      // current access method (may extend)
+  SPgFile *    pPgFile;  // backend page file this DB is using
+  TENV *       pEnv;     // TENV containing the DB
+  int          klen;     // key length if know
+  int          vlen;     // value length if know
+  bool         dup;      // dup mode
+  TdbKeyCmprFn cFn;      // compare function
+};
+
+struct STDbCurosr {
+  SBtCursor *pBtCur;
+};
+
+static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2);
+
+int tdbCreate(TDB **ppDb) {
+  TDB *pDb;
+
+  // create the handle
+  pDb = (TDB *)calloc(1, sizeof(*pDb));
+  if (pDb == NULL) {
+    return -1;
+  }
+
+  pDb->klen = TDB_VARIANT_LEN;
+  pDb->vlen = TDB_VARIANT_LEN;
+  pDb->dup = false;
+  pDb->cFn = tdbDefaultKeyCmprFn;
+
+  *ppDb = pDb;
+  return 0;
+}
+
+static int tdbDestroy(TDB *pDb) {
+  if (pDb) {
+    free(pDb);
+  }
+  return 0;
+}
+
+int tdbOpen(TDB *pDb, const char *fname, const char *dbname, TENV *pEnv) {
+  int       ret;
+  uint8_t   fileid[TDB_FILE_ID_LEN];
+  SPgFile * pPgFile;
+  SPgCache *pPgCache;
+  SBTree *  pBt;
+  bool      fileExist;
+  size_t    dbNameLen;
+  pgno_t    dbRootPgno;
+  char      dbfname[128];  // TODO: make this as a macro or malloc on the heap
+
+  ASSERT(pDb != NULL);
+  ASSERT(fname != NULL);
+  // TODO: Here we simply put an assert here. In the future, make `pEnv`
+  // can be set as NULL.
+  ASSERT(pEnv != NULL);
+
+  // check the DB name
+  dbNameLen = 0;
+  if (dbname) {
+    dbNameLen = strlen(dbname);
+    if (dbNameLen >= TDB_MAX_DBNAME_LEN) {
+      return -1;
+    }
+
+    memcpy(pDb->dbname, dbname, dbNameLen);
+  }
+
+  pDb->dbname[dbNameLen] = '\0';
+
+  // get page file from the env, if not opened yet, open it
+  pPgFile = NULL;
+  snprintf(dbfname, 128, "%s/%s", tdbEnvGetRootDir(pEnv), fname);
+  fileExist = (tdbCheckFileAccess(fname, TDB_F_OK) == 0);
+  if (fileExist) {
+    tdbGnrtFileID(dbfname, fileid, false);
+    pPgFile = tdbEnvGetPageFile(pEnv, fileid);
+  }
+
+  if (pPgFile == NULL) {
+    ret = pgFileOpen(&pPgFile, dbfname, pEnv);
+    if (ret != 0) {
+      // TODO: handle error
+      return -1;
+    }
+  }
+
+  // TODO: get the root page number from the master DB of the page file
+  // tdbGet(&dbRootPgno);
+  if (dbRootPgno == 0) {
+    // DB not exist, create one
+    ret = pgFileAllocatePage(pPgFile, &dbRootPgno);
+    if (ret != 0) {
+      // TODO: handle error
+    }
+    // tdbInsert(pPgFile->pMasterDB, dbname, strlen(dbname), &dbRootPgno, sizeof(dbRootPgno));
+  }
+
+  ASSERT(dbRootPgno > 1);
+
+  // pDb->pBt->root = dbRootPgno;
+
+  // register
+  pDb->pPgFile = pPgFile;
+  tdbEnvRgstDB(pEnv, pDb);
+  pDb->pEnv = pEnv;
+
+  return 0;
+}
+
+int tdbClose(TDB *pDb) {
+  if (pDb == NULL) return 0;
+  return tdbDestroy(pDb);
+}
+
+int tdbDrop(TDB *pDb) {
+  // TODO
+  return 0;
+}
+
+int tdbSetKeyLen(TDB *pDb, int klen) {
+  // TODO: check `klen`
+  pDb->klen = klen;
+  return 0;
+}
+
+int tdbSetValLen(TDB *pDb, int vlen) {
+  // TODO: check `vlen`
+  pDb->vlen = vlen;
+  return 0;
+}
+
+int tdbSetDup(TDB *pDb, int dup) {
+  if (dup) {
+    pDb->dup = true;
+  } else {
+    pDb->dup = false;
+  }
+  return 0;
+}
+
+int tdbSetCmprFunc(TDB *pDb, TdbKeyCmprFn fn) {
+  if (fn == NULL) {
+    return -1;
+  } else {
+    pDb->cFn = fn;
+  }
+  return 0;
+}
+
+int tdbGetKeyLen(TDB *pDb) { return pDb->klen; }
+
+int tdbGetValLen(TDB *pDb) { return pDb->vlen; }
+
+int tdbGetDup(TDB *pDb) {
+  if (pDb->dup) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+int tdbInsert(TDB *pDb, const void *pKey, int nKey, const void *pData, int nData) {
+  // TODO
+  return 0;
+}
+
+static int tdbDefaultKeyCmprFn(int keyLen1, const void *pKey1, int keyLen2, const void *pKey2) {
+  int mlen;
+  int cret;
+
+  ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL);
+
+  mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2;
+  cret = memcmp(pKey1, pKey2, mlen);
+  if (cret == 0) {
+    if (keyLen1 < keyLen2) {
+      cret = -1;
+    } else if (keyLen1 > keyLen2) {
+      cret = 1;
+    } else {
+      cret = 0;
+    }
+  }
+  return cret;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdbBtree.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+struct SBtCursor {
+  SBTree *pBtree;
+  pgno_t  pgno;
+  SPage * pPage;  // current page traversing
+};
+
+typedef struct {
+  pgno_t pgno;
+  pgsz_t offset;
+} SBtIdx;
+
+// Btree page header definition
+typedef struct __attribute__((__packed__)) {
+  uint8_t  flag;        // page flag
+  int32_t  vlen;        // value length of current page, TDB_VARIANT_LEN for variant length
+  uint16_t nPayloads;   // number of total payloads
+  pgoff_t  freeOff;     // free payload offset
+  pgsz_t   fragSize;    // total fragment size
+  pgoff_t  offPayload;  // payload offset
+  pgno_t   rChildPgno;  // right most child page number
+} SBtPgHdr;
+
+typedef int (*BtreeCmprFn)(const void *, const void *);
+
+#define BTREE_PAGE_HDR(pPage) NULL             /* TODO */
+#define BTREE_PAGE_PAYLOAD_AT(pPage, idx) NULL /*TODO*/
+#define BTREE_PAGE_IS_LEAF(pPage) 0            /* TODO */
+
+static int btreeCreate(SBTree **ppBt);
+static int btreeDestroy(SBTree *pBt);
+static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno);
+
+int btreeOpen(SBTree **ppBt, SPgFile *pPgFile) {
+  SBTree *pBt;
+  int     ret;
+
+  ret = btreeCreate(&pBt);
+  if (ret != 0) {
+    return -1;
+  }
+
+  *ppBt = pBt;
+  return 0;
+}
+
+int btreeClose(SBTree *pBt) {
+  // TODO
+  return 0;
+}
+
+static int btreeCreate(SBTree **ppBt) {
+  SBTree *pBt;
+
+  pBt = (SBTree *)calloc(1, sizeof(*pBt));
+  if (pBt == NULL) {
+    return -1;
+  }
+
+  // TODO
+  return 0;
+}
+
+static int btreeDestroy(SBTree *pBt) {
+  if (pBt) {
+    free(pBt);
+  }
+  return 0;
+}
+
+int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt) {
+  // TODO
+  return 0;
+}
+
+int btreeCursorClose(SBtCursor *pBtCur) {
+  // TODO
+  return 0;
+}
+
+int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey) {
+  SPage *     pPage;
+  SBtPgHdr *  pBtPgHdr;
+  SPgFile *   pPgFile;
+  pgno_t      childPgno;
+  pgno_t      rootPgno;
+  int         nPayloads;
+  void *      pPayload;
+  BtreeCmprFn cmpFn;
+
+  // 1. Move the cursor to the root page
+  if (rootPgno == TDB_IVLD_PGNO) {
+    // No any data in this btree, just return not found (TODO)
+    return 0;
+  } else {
+    // Load the page from the file by the SPgFile handle
+    pPage = pgFileFetch(pPgFile, rootPgno);
+
+    pBtCur->pPage = pPage;
+  }
+
+  // 2. Loop to search over the whole tree
+  for (;;) {
+    int lidx, ridx, midx, cret;
+
+    pPage = pBtCur->pPage;
+    pBtPgHdr = BTREE_PAGE_HDR(pPage);
+    nPayloads = pBtPgHdr->nPayloads;
+
+    // Binary search the page
+    lidx = 0;
+    ridx = nPayloads - 1;
+    midx = (lidx + ridx) >> 1;
+    for (;;) {
+      // get the payload ptr at midx
+      pPayload = BTREE_PAGE_PAYLOAD_AT(pPage, midx);
+
+      // the payload and the key
+      cret = cmpFn(pKey, pPayload);
+
+      if (cret < 0) {
+        /* TODO */
+      } else if (cret > 0) {
+        /* TODO */
+      } else {
+        /* TODO */
+      }
+
+      if (lidx > ridx) break;
+      midx = (lidx + ridx) >> 1;
+    }
+    if (BTREE_PAGE_IS_LEAF(pPage)) {
+      /* TODO */
+      break;
+    } else {
+      /* TODO */
+      btreeCursorMoveToChild(pBtCur, childPgno);
+    }
+  }
+
+  return 0;
+}
+
+static int btreeCursorMoveToChild(SBtCursor *pBtCur, pgno_t pgno) {
+  SPgFile *pPgFile;
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdbEnv.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+struct STDbEnv {
+  char *      rootDir;    // root directory of the environment
+  char *      jname;      // journal file name
+  int         jfd;        // journal file fd
+  pgsz_t      pgSize;     // page size
+  cachesz_t   cacheSize;  // total cache size
+  STDbList    dbList;     // TDB List
+  SPgFileList pgfList;    // SPgFile List
+  SPgCache *  pPgCache;   // page cache
+  struct {
+#define TDB_ENV_PGF_HASH_BUCKETS 17
+    SPgFileList buckets[TDB_ENV_PGF_HASH_BUCKETS];
+  } pgfht;  // page file hash table;
+};
+
+#define TDB_ENV_PGF_HASH(fileid)        \
+  ({                                    \
+    uint8_t *tmp = (uint8_t *)(fileid); \
+    tmp[0] + tmp[1] + tmp[2];           \
+  })
+
+static int tdbEnvDestroy(TENV *pEnv);
+
+int tdbEnvCreate(TENV **ppEnv, const char *rootDir) {
+  TENV * pEnv;
+  size_t slen;
+  size_t jlen;
+
+  ASSERT(rootDir != NULL);
+
+  *ppEnv = NULL;
+  slen = strlen(rootDir);
+  jlen = slen + strlen(TDB_JOURNAL_NAME) + 1;
+  pEnv = (TENV *)calloc(1, sizeof(*pEnv) + slen + 1 + jlen + 1);
+  if (pEnv == NULL) {
+    return -1;
+  }
+
+  pEnv->rootDir = (char *)(&pEnv[1]);
+  pEnv->jname = pEnv->rootDir + slen + 1;
+  pEnv->jfd = -1;
+  pEnv->pgSize = TDB_DEFAULT_PGSIZE;
+  pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE;
+
+  memcpy(pEnv->rootDir, rootDir, slen);
+  pEnv->rootDir[slen] = '\0';
+  sprintf(pEnv->jname, "%s/%s", rootDir, TDB_JOURNAL_NAME);
+
+  TD_DLIST_INIT(&(pEnv->dbList));
+  TD_DLIST_INIT(&(pEnv->pgfList));
+
+  /* TODO */
+
+  *ppEnv = pEnv;
+  return 0;
+}
+
+int tdbEnvOpen(TENV *pEnv) {
+  SPgCache *pPgCache;
+  int       ret;
+
+  ASSERT(pEnv != NULL);
+
+  /* TODO: here we do not need to create the root directory, more
+   * work should be done here
+   */
+  mkdir(pEnv->rootDir, 0755);
+
+  ret = pgCacheOpen(&pPgCache, pEnv);
+  if (ret != 0) {
+    goto _err;
+  }
+
+  pEnv->pPgCache = pPgCache;
+  return 0;
+
+_err:
+  return -1;
+}
+
+int tdbEnvClose(TENV *pEnv) {
+  if (pEnv == NULL) return 0;
+  pgCacheClose(pEnv->pPgCache);
+  tdbEnvDestroy(pEnv);
+  return 0;
+}
+
+int tdbEnvSetCache(TENV *pEnv, pgsz_t pgSize, cachesz_t cacheSize) {
+  if (!TDB_IS_PGSIZE_VLD(pgSize) || cacheSize / pgSize < 10) {
+    return -1;
+  }
+
+  /* TODO */
+
+  pEnv->pgSize = pgSize;
+  pEnv->cacheSize = cacheSize;
+
+  return 0;
+}
+
+pgsz_t tdbEnvGetPageSize(TENV *pEnv) { return pEnv->pgSize; }
+
+cachesz_t tdbEnvGetCacheSize(TENV *pEnv) { return pEnv->cacheSize; }
+
+SPgFile *tdbEnvGetPageFile(TENV *pEnv, const uint8_t fileid[]) {
+  SPgFileList *pBucket;
+  SPgFile *    pPgFile;
+
+  pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(fileid) % TDB_ENV_PGF_HASH_BUCKETS);  // TODO
+  for (pPgFile = TD_DLIST_HEAD(pBucket); pPgFile != NULL; pPgFile = TD_DLIST_NODE_NEXT_WITH_FIELD(pPgFile, envHash)) {
+    if (memcmp(fileid, pPgFile->fileid, TDB_FILE_ID_LEN) == 0) break;
+  };
+
+  return pPgFile;
+}
+
+SPgCache *tdbEnvGetPgCache(TENV *pEnv) { return pEnv->pPgCache; }
+
+static int tdbEnvDestroy(TENV *pEnv) {
+  // TODO
+  return 0;
+}
+
+int tdbEnvBeginTxn(TENV *pEnv) {
+  pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755);
+  if (pEnv->jfd < 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int tdbEnvCommit(TENV *pEnv) {
+  /* TODO */
+  close(pEnv->jfd);
+  pEnv->jfd = -1;
+  return 0;
+}
+
+const char *tdbEnvGetRootDir(TENV *pEnv) { return pEnv->rootDir; }
+
+int tdbEnvRgstPageFile(TENV *pEnv, SPgFile *pPgFile) {
+  SPgFileList *pBucket;
+
+  TD_DLIST_APPEND_WITH_FIELD(&(pEnv->pgfList), pPgFile, envPgfList);
+
+  pBucket = pEnv->pgfht.buckets + (TDB_ENV_PGF_HASH(pPgFile->fileid) % TDB_ENV_PGF_HASH_BUCKETS);  // TODO
+  TD_DLIST_APPEND_WITH_FIELD(pBucket, pPgFile, envHash);
+
+  return 0;
+}
+
+int tdbEnvRgstDB(TENV *pEnv, TDB *pDb) {
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdb_mpool.c → source/libs/tdb/src/db/tdbPgCache.c

@@ -12,10 +12,225 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#include "tdbInt.h"
+
+typedef TD_DLIST(SPage) SPgList;
+struct SPgCache {
+  TENV *  pEnv;  // TENV containing this page cache
+  pgsz_t  pgsize;
+  int32_t npage;
+  SPage **pages;
+  SPgList freeList;
+  SPgList lru;
+  struct {
+    int32_t  nbucket;
+    SPgList *buckets;
+  } pght;  // page hash table
+};
+
+static void pgCachePinPage(SPage *pPage);
+static void pgCacheUnpinPage(SPage *pPage);
+
+int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv) {
+  SPgCache *pPgCache;
+  SPage *   pPage;
+  void *    pData;
+  pgsz_t    pgSize;
+  cachesz_t cacheSize;
+  int32_t   npage;
+  int32_t   nbucket;
+  size_t    msize;
+
+  *ppPgCache = NULL;
+  pgSize = tdbEnvGetPageSize(pEnv);
+  cacheSize = tdbEnvGetCacheSize(pEnv);
+  npage = cacheSize / pgSize;
+  nbucket = npage;
+  msize = sizeof(*pPgCache) + sizeof(SPage *) * npage + sizeof(SPgList) * nbucket;
+
+  // Allocate the handle
+  pPgCache = (SPgCache *)calloc(1, msize);
+  if (pPgCache == NULL) {
+    return -1;
+  }
+
+  // Init the handle
+  pPgCache->pEnv = pEnv;
+  pPgCache->pgsize = pgSize;
+  pPgCache->npage = npage;
+  pPgCache->pages = (SPage **)(&pPgCache[1]);
+  pPgCache->pght.nbucket = nbucket;
+  pPgCache->pght.buckets = (SPgList *)(&(pPgCache->pages[npage]));
+
+  TD_DLIST_INIT(&(pPgCache->freeList));
+
+  for (int32_t i = 0; i < npage; i++) {
+    pData = malloc(pgSize + sizeof(SPage));
+    if (pData == NULL) {
+      return -1;
+      // TODO: handle error
+    }
+
+    pPage = POINTER_SHIFT(pData, pgSize);
+
+    pPage->pgid = TDB_IVLD_PGID;
+    pPage->frameid = i;
+    pPage->pData = pData;
+
+    // add current page to the page cache
+    pPgCache->pages[i] = pPage;
+    TD_DLIST_APPEND_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode);
+  }
+
+#if 0
+  for (int32_t i = 0; i < nbucket; i++) {
+    TD_DLIST_INIT(pPgCache->pght.buckets + i);
+  }
+#endif
+
+  *ppPgCache = pPgCache;
+  return 0;
+}
+
+int pgCacheClose(SPgCache *pPgCache) {
+  SPage *pPage;
+  if (pPgCache) {
+    for (int32_t i = 0; i < pPgCache->npage; i++) {
+      pPage = pPgCache->pages[i];
+      tfree(pPage->pData);
+    }
 
-#include "tdb_mpool.h"
+    free(pPgCache);
+  }
+
+  return 0;
+}
+
+#define PG_CACHE_HASH(fileid, pgno)       \
+  ({                                      \
+    uint64_t *tmp = (uint64_t *)(fileid); \
+    (tmp[0] + tmp[1] + tmp[2] + (pgno));  \
+  })
+
+SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid) {
+  SPage *  pPage;
+  SPgFile *pPgFile;
+  SPgList *pBucket;
+
+  // 1. Search the page hash table SPgCache.pght
+  pBucket = pPgCache->pght.buckets + (PG_CACHE_HASH(pgid.fileid, pgid.pgno) % pPgCache->pght.nbucket);
+  pPage = TD_DLIST_HEAD(pBucket);
+  while (pPage && tdbCmprPgId(&(pPage->pgid), &pgid)) {
+    pPage = TD_DLIST_NODE_NEXT_WITH_FIELD(pPage, pghtNode);
+  }
+
+  if (pPage) {
+    // Page is found, pin the page and return the page
+    pgCachePinPage(pPage);
+    return pPage;
+  }
+
+  // 2. Check the free list
+  pPage = TD_DLIST_HEAD(&(pPgCache->freeList));
+  if (pPage) {
+    TD_DLIST_POP_WITH_FIELD(&(pPgCache->freeList), pPage, freeNode);
+    pgCachePinPage(pPage);
+    return pPage;
+  }
+
+  // 3. Try to recycle a page from the LRU list
+  pPage = TD_DLIST_HEAD(&(pPgCache->lru));
+  if (pPage) {
+    TD_DLIST_POP_WITH_FIELD(&(pPgCache->lru), pPage, lruNode);
+    // TODO: remove from the hash table
+    pgCachePinPage(pPage);
+    return pPage;
+  }
+
+  // 4. If a memory allocator is set, try to allocate from the allocator (TODO)
+
+  return NULL;
+}
+
+int pgCacheRelease(SPage *pPage) {
+  // TODO
+  return 0;
+}
+
+static void pgCachePinPage(SPage *pPage) {
+  // TODO
+}
+
+static void pgCacheUnpinPage(SPage *pPage) {
+  // TODO
+}
+
+#if 0
+// Exposed handle
+typedef struct TDB_MPOOL  TDB_MPOOL;
+typedef struct TDB_MPFILE TDB_MPFILE;
+
+typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t;
+typedef struct pg_t {
+  SRWLatch             rwLatch;
+  frame_id_t           frameid;
+  pgid_t               pgid;
+  uint8_t              dirty;
+  uint8_t              rbit;
+  int32_t              pinRef;
+  pg_free_dlist_node_t free;
+  pg_hash_dlist_node_t hash;
+  void *               p;
+} pg_t;
+
+typedef TD_DLIST(pg_t) pg_list_t;
+typedef struct {
+  SRWLatch latch;
+  TD_DLIST(TDB_MPFILE);
+} mpf_bucket_t;
+struct TDB_MPOOL {
+  int64_t    cachesize;
+  pgsz_t     pgsize;
+  int32_t    npages;
+  pg_t *     pages;
+  pg_list_t  freeList;
+  frame_id_t clockHand;
+  struct {
+    int32_t    nbucket;
+    pg_list_t *hashtab;
+  } pgtab;  // page table, hash<pgid_t, pg_t>
+  struct {
+#define MPF_HASH_BUCKETS 16
+    mpf_bucket_t buckets[MPF_HASH_BUCKETS];
+  } mpfht;  // MPF hash table. MPFs using this MP will be put in this hash table
+};
+
+#define MP_PAGE_AT(mp, idx) (mp)->pages[idx]
+
+typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t;
+struct TDB_MPFILE {
+  char *              fname;                    // file name
+  int                 fd;                       // fd
+  uint8_t             fileid[TDB_FILE_ID_LEN];  // file ID
+  TDB_MPOOL *         mp;                       // underlying memory pool
+  td_mpf_dlist_node_t node;
+};
+
+/*=================================================== Exposed apis ==================================================*/
+// TDB_MPOOL
+int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize);
+int tdbMPoolClose(TDB_MPOOL *mp);
+int tdbMPoolSync(TDB_MPOOL *mp);
+
+// TDB_MPFILE
+int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp);
+int tdbMPoolFileClose(TDB_MPFILE *mpf);
+int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr);
+int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr);
+int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
+int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
+int tdbMPoolFileSync(TDB_MPFILE *mpf);
 
-static int         tdbGnrtFileID(const char *fname, uint8_t *fileid);
 static void        tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
 static void        tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
 static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid);
@@ -23,7 +238,7 @@ static int         tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p);
 static int         tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p);
 static void        tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp);
 
-int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) {
+int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsz_t pgsize) {
   TDB_MPOOL *mp = NULL;
   size_t     tsize;
   pg_t *     pagep;
@@ -120,7 +335,7 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) {
     goto _err;
   }
 
-  if (tdbGnrtFileID(fname, mpf->fileid) < 0) {
+  if (tdbGnrtFileID(fname, mpf->fileid, false) < 0) {
     goto _err;
   }
 
@@ -230,22 +445,6 @@ int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
   return 0;
 }
 
-static int tdbGnrtFileID(const char *fname, uint8_t *fileid) {
-  struct stat statbuf;
-
-  if (stat(fname, &statbuf) < 0) {
-    return -1;
-  }
-
-  memset(fileid, 0, TDB_FILE_ID_LEN);
-
-  ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino;
-  ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev;
-  ((uint64_t *)fileid)[2] = rand();
-
-  return 0;
-}
-
 #define MPF_GET_BUCKETID(fileid)                   \
   ({                                               \
     uint64_t *tmp = (uint64_t *)fileid;            \
@@ -317,7 +516,7 @@ static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
 }
 
 static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) {
-  pgsize_t   pgsize;
+  pgsz_t     pgsize;
   TDB_MPOOL *mp;
   off_t      offset;
   size_t     rsize;
@@ -334,7 +533,7 @@ static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) {
 }
 
 static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) {
-  pgsize_t   pgsize;
+  pgsz_t     pgsize;
   TDB_MPOOL *mp;
   off_t      offset;
 
@@ -377,3 +576,5 @@ static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) {
 
   *pagepp = pagep;
 }
+
+#endif
\ No newline at end of file

source/libs/tdb/src/db/tdbPgFile.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+typedef struct SPage1 {
+  char     magic[64];
+  pgno_t   mdbRootPgno;  // master DB root page number
+  pgno_t   freePgno;     // free list page number
+  uint32_t nFree;        // number of free pages
+} SPage1;
+
+typedef struct SFreePage {
+  /* TODO */
+} SFreePage;
+
+TDB_STATIC_ASSERT(sizeof(SPage1) <= TDB_MIN_PGSIZE, "TDB Page1 definition too large");
+
+static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData);
+
+int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
+  SPgFile * pPgFile;
+  SPgCache *pPgCache;
+  size_t    fnameLen;
+  pgno_t    fsize;
+
+  *ppPgFile = NULL;
+
+  // create the handle
+  fnameLen = strlen(fname);
+  pPgFile = (SPgFile *)calloc(1, sizeof(*pPgFile) + fnameLen + 1);
+  if (pPgFile == NULL) {
+    return -1;
+  }
+
+  ASSERT(pEnv != NULL);
+
+  // init the handle
+  pPgFile->fname = (char *)(&(pPgFile[1]));
+  memcpy(pPgFile->fname, fname, fnameLen);
+  pPgFile->fname[fnameLen] = '\0';
+  pPgFile->fd = -1;
+
+  pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755);
+  if (pPgFile->fd < 0) {
+    // TODO: handle error
+    return -1;
+  }
+
+  tdbGnrtFileID(fname, pPgFile->fileid, false);
+  tdbGetFileSize(fname, tdbEnvGetPageSize(pEnv), &fsize);
+
+  pPgFile->fsize = fsize;
+  pPgFile->lsize = fsize;
+
+  if (pPgFile->fsize == 0) {
+    // A created file
+    pgno_t pgno;
+    pgid_t pgid;
+
+    pgFileAllocatePage(pPgFile, &pgno);
+
+    ASSERT(pgno == 1);
+
+    memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN);
+    pgid.pgno = pgno;
+
+    pgCacheFetch(pPgCache, pgid);
+    // Need to allocate the first page as a description page
+  } else {
+    // An existing file
+  }
+
+  /* TODO: other open operations */
+
+  // add the page file to the environment
+  tdbEnvRgstPageFile(pEnv, pPgFile);
+  pPgFile->pEnv = pEnv;
+
+  *ppPgFile = pPgFile;
+  return 0;
+}
+
+int pgFileClose(SPgFile *pPgFile) {
+  if (pPgFile) {
+    if (pPgFile->fd >= 0) {
+      close(pPgFile->fd);
+    }
+
+    tfree(pPgFile->fname);
+    free(pPgFile);
+  }
+
+  return 0;
+}
+
+SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno) {
+  SPgCache *pPgCache;
+  SPage *   pPage;
+  pgid_t    pgid;
+
+  // 1. Fetch from the page cache
+  // pgCacheFetch(pPgCache, pgid);
+
+  // 2. If only get a page frame, no content, maybe
+  // need to load from the file
+  if (1 /*page not initialized*/) {
+    if (pgno < pPgFile->fsize) {
+      // load the page content from the disk
+      // ?? How about the freed pages ??
+    } else {
+      // zero the page, make the page as a empty
+      // page with zero records.
+    }
+  }
+
+#if 0
+  pPgCache = pPgFile->pPgCache;
+  pPage = NULL;
+  memcpy(pgid.fileid, pPgFile->fileid, TDB_FILE_ID_LEN);
+  pgid.pgno = pgno;
+
+  if (pgno > pPgFile->pgFileSize) {
+    // TODO
+  } else {
+    pPage = pgCacheFetch(pPgCache, pgid);
+    if (1 /*Page is cached, no need to load from file*/) {
+      return pPage;
+    } else {
+      // TODO: handle error
+      if (pgFileRead(pPgFile, pgno, (void *)pPage) < 0) {
+        // todoerr
+      }
+      return pPage;
+    }
+  }
+#endif
+
+  return pPage;
+}
+
+int pgFileRelease(SPage *pPage) {
+  pgCacheRelease(pPage);
+  return 0;
+}
+
+int pgFileWrite(SPage *pPage) {
+  // TODO
+  return 0;
+}
+
+int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno) {
+  pgno_t    pgno;
+  SPage1 *  pPage1;
+  SPgCache *pPgCache;
+  pgid_t    pgid;
+  SPage *   pPage;
+
+  if (pPgFile->lsize == 0) {
+    pgno = ++(pPgFile->lsize);
+  } else {
+    if (0) {
+      // TODO: allocate from the free list
+      pPage = pgCacheFetch(pPgCache, pgid);
+
+      if (pPage1->nFree > 0) {
+        // TODO
+      } else {
+        pgno = ++(pPgFile->lsize);
+      }
+    } else {
+      pgno = ++(pPgFile->lsize);
+    }
+  }
+
+  *pPgno = pgno;
+  return 0;
+}
+
+static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) {
+  pgsz_t   pgSize;
+  ssize_t  rsize;
+  uint8_t *pTData;
+  size_t   szToRead;
+
+#if 0
+
+  // pgSize = ; (TODO)
+  pTData = pData;
+  szToRead = pgSize;
+  for (; szToRead > 0;) {
+    rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize);
+    if (rsize < 0) {
+      if (errno == EINTR) {
+        continue;
+      } else {
+        return -1;
+      }
+    } else if (rsize == 0) {
+      return -1;
+    }
+
+    szToRead -= rsize;
+    pTData += rsize;
+  }
+#endif
+
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/db/tdbUtil.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tdbInt.h"
+
+int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
+  struct stat statbuf;
+
+  if (stat(fname, &statbuf) < 0) {
+    return -1;
+  }
+
+  memset(fileid, 0, TDB_FILE_ID_LEN);
+
+  ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino;
+  ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev;
+  if (unique) {
+    ((uint64_t *)fileid)[2] = rand();
+  }
+
+  return 0;
+}
+
+int tdbCheckFileAccess(const char *pathname, int mode) {
+  int flags = 0;
+
+  if (mode & TDB_F_OK) {
+    flags |= F_OK;
+  }
+
+  if (mode & TDB_R_OK) {
+    flags |= R_OK;
+  }
+
+  if (mode & TDB_W_OK) {
+    flags |= W_OK;
+  }
+
+  return access(pathname, flags);
+}
+
+int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize) {
+  struct stat st;
+  int         ret;
+
+  ret = stat(fname, &st);
+  if (ret != 0) {
+    return -1;
+  }
+
+  ASSERT(st.st_size % pgSize == 0);
+
+  *pSize = st.st_size / pgSize;
+  return 0;
+}
\ No newline at end of file

source/libs/tdb/src/inc/tdbBtree.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_BTREE_H_
+#define _TD_BTREE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SBTree    SBTree;
+typedef struct SBtCursor SBtCursor;
+
+// SBTree
+int btreeOpen(SBTree **ppBt, SPgFile *pPgFile);
+int btreeClose(SBTree *pBt);
+
+// SBtCursor
+int btreeCursorOpen(SBtCursor *pBtCur, SBTree *pBt);
+int btreeCursorClose(SBtCursor *pBtCur);
+int btreeCursorMoveTo(SBtCursor *pBtCur, int kLen, const void *pKey);
+int btreeCursorNext(SBtCursor *pBtCur);
+
+struct SBTree {
+  pgno_t   root;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_BTREE_H_*/
\ No newline at end of file

source/libs/tdb/src/db/tdb_db.c → source/libs/tdb/src/inc/tdbEnv.h

@@ -13,14 +13,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tdb_db.h"
+#ifndef _TDB_ENV_H_
+#define _TDB_ENV_H_
 
-int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) {
-  // TODO
-  return 0;
-}
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+const char* tdbEnvGetRootDir(TENV* pEnv);
+SPgFile*    tdbEnvGetPageFile(TENV* pEnv, const uint8_t fileid[]);
+SPgCache*   tdbEnvGetPgCache(TENV* pEnv);
+int         tdbEnvRgstPageFile(TENV* pEnv, SPgFile* pPgFile);
+int         tdbEnvRgstDB(TENV* pEnv, TDB* pDb);
 
-int tdbClose(TDB *dbp, uint32_t flags) {
-  // TODO
-  return 0;
+#ifdef __cplusplus
 }
+#endif
+
+#endif /*_TDB_ENV_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TDB_INTERNAL_H_
+#define _TD_TDB_INTERNAL_H_
+
+#include "tlist.h"
+#include "tlockfree.h"
+
+#include "tdb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SPgFile SPgFile;
+
+// pgno_t
+typedef int32_t pgno_t;
+#define TDB_IVLD_PGNO ((pgno_t)0)
+
+// fileid
+#define TDB_FILE_ID_LEN 24
+
+// pgid_t
+typedef struct {
+  uint8_t fileid[TDB_FILE_ID_LEN];
+  pgno_t  pgno;
+} pgid_t;
+
+#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO};
+
+static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) {
+  pgid_t *pgid1 = (pgid_t *)p1;
+  pgid_t *pgid2 = (pgid_t *)p2;
+  int     rcode;
+
+  rcode = memcmp(pgid1->fileid, pgid2->fileid, TDB_FILE_ID_LEN);
+  if (rcode) {
+    return rcode;
+  } else {
+    if (pgid1->pgno > pgid2->pgno) {
+      return 1;
+    } else if (pgid1->pgno < pgid2->pgno) {
+      return -1;
+    } else {
+      return 0;
+    }
+  }
+}
+
+// framd_id_t
+typedef int32_t frame_id_t;
+
+// pgsz_t
+#define TDB_MIN_PGSIZE 512
+#define TDB_MAX_PGSIZE 65536
+#define TDB_DEFAULT_PGSIZE 4096
+#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
+
+// pgoff_t
+typedef pgsz_t pgoff_t;
+
+// cache
+#define TDB_DEFAULT_CACHE_SIZE (256 * 4096)  // 1M
+
+// dbname
+#define TDB_MAX_DBNAME_LEN 24
+
+// tdb_log
+#define tdbError(var)
+
+typedef TD_DLIST(STDb) STDbList;
+typedef TD_DLIST(SPgFile) SPgFileList;
+typedef TD_DLIST_NODE(SPgFile) SPgFileListNode;
+
+#define TERR_A(val, op, flag)  \
+  do {                         \
+    if (((val) = (op)) != 0) { \
+      goto flag;               \
+    }                          \
+  } while (0)
+
+#define TERR_B(val, op, flag)     \
+  do {                            \
+    if (((val) = (op)) == NULL) { \
+      goto flag;                  \
+    }                             \
+  } while (0)
+
+#define TDB_VARIANT_LEN (int)-1
+
+// page payload format
+// <keyLen> + <valLen> + [key] + [value]
+#define TDB_DECODE_PAYLOAD(pPayload, keyLen, pKey, valLen, pVal) \
+  do {                                                           \
+    if ((keyLen) == TDB_VARIANT_LEN) {                           \
+      /* TODO: decode the keyLen */                              \
+    }                                                            \
+    if ((valLen) == TDB_VARIANT_LEN) {                           \
+      /* TODO: decode the valLen */                              \
+    }                                                            \
+    /* TODO */                                                   \
+  } while (0)
+
+#define TDB_JOURNAL_NAME "tdb.journal"
+
+#include "tdbUtil.h"
+
+#include "tdbBtree.h"
+
+#include "tdbPgCache.h"
+
+#include "tdbPgFile.h"
+
+#include "tdbEnv.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TDB_INTERNAL_H_*/

source/libs/tdb/src/inc/tdbPgCache.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_PAGE_CACHE_H_
+#define _TD_PAGE_CACHE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SPgCache SPgCache;
+typedef struct SPage    SPage;
+
+// SPgCache
+int pgCacheOpen(SPgCache **ppPgCache, TENV *pEnv);
+int pgCacheClose(SPgCache *pPgCache);
+
+SPage *pgCacheFetch(SPgCache *pPgCache, pgid_t pgid);
+int    pgCacheRelease(SPage *pPage);
+
+// SPage
+typedef TD_DLIST_NODE(SPage) SPgListNode;
+struct SPage {
+  pgid_t      pgid;      // page id
+  frame_id_t  frameid;   // frame id
+  uint8_t *   pData;     // real data
+  SPgListNode freeNode;  // for SPgCache.freeList
+  SPgListNode pghtNode;  // for pght
+  SPgListNode lruNode;   // for LRU
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_PAGE_CACHE_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdbPgFile.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_PAGE_FILE_H_
+#define _TD_PAGE_FILE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct __attribute__((__packed__)) {
+  char    hdrInfo[16];  // info string
+  pgsz_t  szPage;       // page size of current file
+  int32_t cno;          // commit number counter
+  pgno_t  freePgno;     // freelist page number
+  uint8_t resv[100];    // reserved space
+} SPgFileHdr;
+
+#define TDB_PG_FILE_HDR_SIZE 128
+
+TDB_STATIC_ASSERT(sizeof(SPgFileHdr) == TDB_PG_FILE_HDR_SIZE, "Page file header size if not 128");
+
+struct SPgFile {
+  TENV *          pEnv;                     // env containing this page file
+  char *          fname;                    // backend file name
+  uint8_t         fileid[TDB_FILE_ID_LEN];  // file id
+  pgno_t          lsize;                    // page file logical size (for count)
+  pgno_t          fsize;                    // real file size on disk (for rollback)
+  int             fd;
+  SPgFileListNode envHash;
+  SPgFileListNode envPgfList;
+};
+
+int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv);
+int pgFileClose(SPgFile *pPgFile);
+
+SPage *pgFileFetch(SPgFile *pPgFile, pgno_t pgno);
+int    pgFileRelease(SPage *pPage);
+
+int pgFileWrite(SPage *pPage);
+int pgFileAllocatePage(SPgFile *pPgFile, pgno_t *pPgno);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_PAGE_FILE_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdb_inc.h → source/libs/tdb/src/inc/tdbUtil.h

@@ -13,46 +13,32 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TDB_INC_H_
-#define _TD_TDB_INC_H_
-
-#include "os.h"
-#include "tlist.h"
-#include "tlockfree.h"
+#ifndef _TDB_UTIL_H_
+#define _TDB_UTIL_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-// pgno_t
-typedef int32_t pgno_t;
-#define TDB_IVLD_PGNO ((pgno_t)-1)
-
-// fileid
-#define TDB_FILE_ID_LEN 24
+#if __STDC_VERSION__ >= 201112L
+#define TDB_STATIC_ASSERT(op, info) static_assert(op, info)
+#else
+#define TDB_STATIC_ASSERT(op, info)
+#endif
 
-// pgid_t
-typedef struct {
-  uint8_t fileid[TDB_FILE_ID_LEN];
-  pgno_t  pgno;
-} pgid_t;
-#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO};
+#define TDB_ROUND8(x) (((x) + 7) & ~7)
 
-// framd_id_t
-typedef int32_t frame_id_t;
+int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique);
 
-// pgsize_t
-typedef int32_t pgsize_t;
-#define TDB_MIN_PGSIZE 512
-#define TDB_MAX_PGSIZE 16384
-#define TDB_DEFAULT_PGSIZE 4096
-#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
+#define TDB_F_OK 0x1
+#define TDB_R_OK 0x2
+#define TDB_W_OK 0x4
+int tdbCheckFileAccess(const char *pathname, int mode);
 
-// tdb_log
-#define tdbError(var)
+int tdbGetFileSize(const char *fname, pgsz_t pgSize, pgno_t *pSize);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TDB_INC_H_*/
+#endif /*_TDB_UTIL_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdb_mpool.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TDB_MPOOL_H_
-#define _TD_TDB_MPOOL_H_
-
-#include "tdb_inc.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// Exposed handle
-typedef struct TDB_MPOOL  TDB_MPOOL;
-typedef struct TDB_MPFILE TDB_MPFILE;
-
-typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t;
-typedef struct pg_t {
-  SRWLatch             rwLatch;
-  frame_id_t           frameid;
-  pgid_t               pgid;
-  uint8_t              dirty;
-  uint8_t              rbit;
-  int32_t              pinRef;
-  pg_free_dlist_node_t free;
-  pg_hash_dlist_node_t hash;
-  void *               p;
-} pg_t;
-
-typedef TD_DLIST(pg_t) pg_list_t;
-typedef struct {
-  SRWLatch latch;
-  TD_DLIST(TDB_MPFILE);
-} mpf_bucket_t;
-struct TDB_MPOOL {
-  int64_t    cachesize;
-  pgsize_t   pgsize;
-  int32_t    npages;
-  pg_t *     pages;
-  pg_list_t  freeList;
-  frame_id_t clockHand;
-  struct {
-    int32_t    nbucket;
-    pg_list_t *hashtab;
-  } pgtab;  // page table, hash<pgid_t, pg_t>
-  struct {
-#define MPF_HASH_BUCKETS 16
-    mpf_bucket_t buckets[MPF_HASH_BUCKETS];
-  } mpfht;  // MPF hash table. MPFs using this MP will be put in this hash table
-};
-
-#define MP_PAGE_AT(mp, idx) (mp)->pages[idx]
-
-typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t;
-struct TDB_MPFILE {
-  char *              fname;                    // file name
-  int                 fd;                       // fd
-  uint8_t             fileid[TDB_FILE_ID_LEN];  // file ID
-  TDB_MPOOL *         mp;                       // underlying memory pool
-  td_mpf_dlist_node_t node;
-};
-
-/*=================================================== Exposed apis ==================================================*/
-// TDB_MPOOL
-int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize);
-int tdbMPoolClose(TDB_MPOOL *mp);
-int tdbMPoolSync(TDB_MPOOL *mp);
-
-// TDB_MPFILE
-int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp);
-int tdbMPoolFileClose(TDB_MPFILE *mpf);
-int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr);
-int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr);
-int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
-int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
-int tdbMPoolFileSync(TDB_MPFILE *mpf);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_TDB_MPOOL_H_*/
\ No newline at end of file

source/libs/tdb/test/CMakeLists.txt

-# tdbMPoolTest
-add_executable(tdbMPoolTest "tdbMPoolTest.cpp")
-target_link_libraries(tdbMPoolTest tdb gtest gtest_main)
-
 # tdbTest
 add_executable(tdbTest "tdbTest.cpp")
 target_link_libraries(tdbTest tdb gtest gtest_main)
\ No newline at end of file

source/libs/tdb/test/tdbMPoolTest.cpp

-#include "gtest/gtest.h"
-
-#include <iostream>
-
-#include "tdb_mpool.h"
-
-TEST(tdb_mpool_test, test1) {
-  TDB_MPOOL * mp;
-  TDB_MPFILE *mpf;
-  pgno_t      pgno;
-  void *      pgdata;
-
-  // open mp
-  tdbMPoolOpen(&mp, 16384, 4096);
-
-  // open mpf
-  tdbMPoolFileOpen(&mpf, "test.db", mp);
-
-#define TEST1_TOTAL_PAGES 100
-  for (int i = 0; i < TEST1_TOTAL_PAGES; i++) {
-    tdbMPoolFileNewPage(mpf, &pgno, pgdata);
-
-    *(pgno_t *)pgdata = i;
-  }
-
-  // close mpf
-  tdbMPoolFileClose(mpf);
-
-  // close mp
-  tdbMPoolClose(mp);
-}

source/libs/tdb/test/tdbTest.cpp

@@ -2,13 +2,67 @@
 
 #include "tdb.h"
 
-TEST(tdb_api_test, tdb_create_open_close_db_test) {
-  // int  ret;
-  // TDB *dbp;
+TEST(tdb_test, simple_test) {
+  TENV *    pEnv;
+  TDB *     pDb1, *pDb2, *pDb3;
+  pgsz_t    pgSize = 1024;
+  cachesz_t cacheSize = 10240;
 
-  // tdbCreateDB(&dbp, TDB_BTREE_T);
+  // ENV
+  GTEST_ASSERT_EQ(tdbEnvCreate(&pEnv, "./testtdb"), 0);
 
-  // tdbOpenDB(dbp, 0);
+  GTEST_ASSERT_EQ(tdbEnvSetCache(pEnv, pgSize, cacheSize), 0);
 
-  // tdbCloseDB(dbp, 0);
+  GTEST_ASSERT_EQ(tdbEnvGetCacheSize(pEnv), cacheSize);
+
+  GTEST_ASSERT_EQ(tdbEnvGetPageSize(pEnv), pgSize);
+
+  GTEST_ASSERT_EQ(tdbEnvOpen(pEnv), 0);
+
+#if 1
+  // DB
+  GTEST_ASSERT_EQ(tdbCreate(&pDb1), 0);
+
+  // GTEST_ASSERT_EQ(tdbSetKeyLen(pDb1, 8), 0);
+
+  // GTEST_ASSERT_EQ(tdbGetKeyLen(pDb1), 8);
+
+  // GTEST_ASSERT_EQ(tdbSetValLen(pDb1, 3), 0);
+
+  // GTEST_ASSERT_EQ(tdbGetValLen(pDb1), 3);
+
+  // GTEST_ASSERT_EQ(tdbSetDup(pDb1, 1), 0);
+
+  // GTEST_ASSERT_EQ(tdbGetDup(pDb1), 1);
+
+  // GTEST_ASSERT_EQ(tdbSetCmprFunc(pDb1, NULL), 0);
+
+  tdbEnvBeginTxn(pEnv);
+
+  GTEST_ASSERT_EQ(tdbOpen(pDb1, "db.db", "db1", pEnv), 0);
+
+  // char *key = "key1";
+  // char *val = "value1";
+  // tdbInsert(pDb1, (void *)key, strlen(key), (void *)val, strlen(val));
+
+  tdbEnvCommit(pEnv);
+
+#if 0
+  // Insert
+
+  // Query
+
+  // Delete
+
+  // Query
+#endif
+
+  // GTEST_ASSERT_EQ(tdbOpen(&pDb2, "db.db", "db2", pEnv), 0);
+  // GTEST_ASSERT_EQ(tdbOpen(&pDb3, "index.db", NULL, pEnv), 0);
+  // tdbClose(pDb3);
+  // tdbClose(pDb2);
+  tdbClose(pDb1);
+#endif
+
+  tdbEnvClose(pEnv);
 }
\ No newline at end of file

source/util/CMakeLists.txt

@@ -10,7 +10,7 @@ target_link_libraries(
     util
     PRIVATE os
     PUBLIC lz4_static
-    PUBLIC api
+    PUBLIC api cjson
 )
 
 if(${BUILD_TEST})

source/util/src/tjson.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tjson.h"
+
+#include "taoserror.h"
+#include "cJSON.h"
+
+SJson* tjsonCreateObject() {
+  return cJSON_CreateObject();
+}
+
+void tjsonDelete(SJson* pJson) {
+  cJSON_Delete((cJSON*)pJson);
+}
+
+int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number) {
+  char tmp[40] = {0};
+  snprintf(tmp, tListLen(tmp), "%"PRId64, number);
+  return tjsonAddStringToObject(pJson, pName, tmp);
+}
+
+int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) {
+  return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
+}
+
+SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) {
+  return cJSON_AddArrayToObject((cJSON*)pJson, pName);
+}
+
+int32_t tjsonAddItemToObject(SJson *pJson, const char* pName, SJson* pItem) {
+  return (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED);
+}
+
+int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) {
+  return (cJSON_AddItemToArray((cJSON*)pJson, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED);
+}
+
+int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) {
+  if (NULL == pObj) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SJson* pJobj = tjsonCreateObject();
+  if (NULL == pJobj || TSDB_CODE_SUCCESS != func(pObj, pJobj)) {
+    printf("%s:%d code = %d\n", __FUNCTION__, __LINE__, TSDB_CODE_FAILED);
+    tjsonDelete(pJobj);
+    return TSDB_CODE_FAILED;
+  }
+  return tjsonAddItemToObject(pJson, pName, pJobj);
+}
+
+int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) {
+  SJson* pJobj = tjsonCreateObject();
+  if (NULL == pJobj || TSDB_CODE_SUCCESS != func(pObj, pJobj)) {
+    tjsonDelete(pJobj);
+    return TSDB_CODE_FAILED;
+  }
+  return tjsonAddItemToArray(pJson, pJobj);
+}
+
+char* tjsonToString(const SJson* pJson) {
+  return cJSON_Print((cJSON*)pJson);
+}