Merge branch 'feature/vnode' into 3.0

.devcontainer/Dockerfile

+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile
+
+# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
+ARG VARIANT="bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
+
+# [Optional] Uncomment this section to install additional packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>

.devcontainer/devcontainer.json

+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp
+{
+	"name": "C++",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
+		// Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon
+		"args": { "VARIANT": "ubuntu-21.04" }
+	},
+	"runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-vscode.cpptools",
+		"ms-vscode.cmake-tools",
+		"austin.code-gnu-global",
+		"visualstudioexptteam.vscodeintel"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "gcc -v",
+
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode"
+}

.gitignore

@@ -101,4 +101,4 @@ TAGS
 
 contrib/*
 !contrib/CMakeLists.txt
-!contrib/test
+!contrib/test
\ No newline at end of file

CMakeLists.txt

@@ -10,7 +10,7 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
 set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
 include(${CMAKE_SUPPORT_DIR}/cmake.options)
 
-SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma")
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
 
 # contrib
 add_subdirectory(contrib)
@@ -20,9 +20,13 @@ add_library(api INTERFACE)
 target_include_directories(api INTERFACE "include/client")
 
 # src
+if(${BUILD_TEST})
+    include(CTest)
+    enable_testing()
+endif(${BUILD_TEST})
 add_subdirectory(source)
 
 # docs 
 add_subdirectory(docs)
 
-# tests (TODO)
+# tests (TODO)
\ No newline at end of file

include/dnode/vnode/vnode.h

@@ -36,6 +36,8 @@ typedef struct SVnodeCfg {
   struct {
     /** write buffer size */
     uint64_t wsize;
+    uint64_t ssize;
+    uint64_t lsize;
     /** use heap allocator or arena allocator */
     bool isHeapAllocator;
   };

include/util/tdlist.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_TDLIST_H_
+#define _TD_UTIL_TDLIST_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Single linked list
+#define TD_SLIST_NODE(TYPE) \
+  struct {                  \
+    struct TYPE *sl_next_;  \
+  }
+
+#define TD_SLIST(TYPE)      \
+  struct {                  \
+    struct TYPE *sl_head_;  \
+    int          sl_neles_; \
+  }
+
+#define TD_SLIST_HEAD(sl) ((sl)->sl_head_)
+#define TD_SLIST_NELES(sl) ((sl)->sl_neles_)
+#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_)
+
+#define tSListInit(sl)     \
+  do {                     \
+    (sl)->sl_head_ = NULL; \
+    (sl)->sl_neles_ = 0;   \
+  } while (0)
+
+#define tSListPush(sl, sln)                      \
+  do {                                           \
+    TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \
+    TD_SLIST_HEAD(sl) = (sln);                   \
+    TD_SLIST_NELES(sl) += 1;                     \
+  } while (0)
+
+#define tSListPop(sl)                                          \
+  do {                                                         \
+    TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \
+    TD_SLIST_NELES(sl) -= 1;                                   \
+  } while (0)
+
+// Double linked list
+#define TD_DLIST_NODE(TYPE) \
+  struct {                  \
+    TYPE *dl_prev_;         \
+    TYPE *dl_next_;         \
+  }
+
+#define TD_DLIST(TYPE)      \
+  struct {                  \
+    struct TYPE *dl_head_;  \
+    struct TYPE *dl_tail_;  \
+    int          dl_neles_; \
+  }
+
+#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_)
+#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_)
+#define TD_DLIST_HEAD(dl) ((dl)->dl_head_)
+#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_)
+#define TD_DLIST_NELES(dl) ((dl)->dl_neles_)
+
+#define tDListInit(dl)                            \
+  do {                                            \
+    TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \
+    TD_DLIST_NELES(dl) = 0;                       \
+  } while (0)
+
+#define tDListAppend(dl, dln)                                   \
+  do {                                                          \
+    if (TD_DLIST_HEAD(dl) == NULL) {                            \
+      TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \
+      TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln);            \
+    } else {                                                    \
+      TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl);              \
+      TD_DLIST_NODE_NEXT(dln) = NULL;                           \
+      TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln);            \
+      TD_DLIST_TAIL(dl) = (dln);                                \
+    }                                                           \
+    TD_DLIST_NELES(dl) += 1;                                    \
+  } while (0)
+
+#define tDListPrepend(dl, dln)                                  \
+  do {                                                          \
+    if (TD_DLIST_HEAD(dl) == NULL) {                            \
+      TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \
+      TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln);            \
+    } else {                                                    \
+      TD_DLIST_NODE_PREV(dln) = NULL;                           \
+      TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl);              \
+      TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln);            \
+      TD_DLIST_HEAD(dl) = (dln);                                \
+    }                                                           \
+    TD_DLIST_NELES(dl) += 1;                                    \
+  } while (0)
+
+#define tDListPop(dl, dln)                                                   \
+  do {                                                                       \
+    if (TD_DLIST_HEAD(dl) == (dln)) {                                        \
+      TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln);                           \
+    }                                                                        \
+    if (TD_DLIST_TAIL(dl) == (dln)) {                                        \
+      TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln);                           \
+    }                                                                        \
+    if (TD_DLIST_NODE_PREV(dln) != NULL) {                                   \
+      TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \
+    }                                                                        \
+    if (TD_DLIST_NODE_NEXT(dln) != NULL) {                                   \
+      TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \
+    }                                                                        \
+    TD_DLIST_NELES(dl) -= 1;                                                 \
+    TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL;                \
+  } while (0)
+
+#if 0
+// List iterator
+#define TD_LIST_FITER 0
+#define TD_LIST_BITER 1
+#define TD_LIST_ITER(S)     \
+  struct {                  \
+    int it_dir_;            \
+    S * it_next_;           \
+    S * it_ptr_;            \
+    TD_DLIST(S) * it_list_; \
+  }
+
+#define tlistIterInit(it, l, dir)   \
+  (it)->it_dir_ = (dir);            \
+  (it)->it_list_ = l;               \
+  if ((dir) == TD_LIST_FITER) {     \
+    (it)->it_next_ = (l)->dl_head_; \
+  } else {                          \
+    (it)->it_next_ = (l)->dl_tail_; \
+  }
+
+#define tlistIterNext(it)                       \
+  ({                                            \
+    (it)->it_ptr_ = (it)->it_next_;             \
+    if ((it)->it_next_ != NULL) {               \
+      if ((it)->it_dir_ == TD_LIST_FITER) {     \
+        (it)->it_next_ = (it)->it_next_->next_; \
+      } else {                                  \
+        (it)->it_next_ = (it)->it_next_->prev_; \
+      }                                         \
+    }                                           \
+    (it)->it_ptr_;                              \
+  })
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_TDLIST_H_*/
\ No newline at end of file

include/util/tmacro.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_MACRO_H_
+#define _TD_UTIL_MACRO_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Module init/clear MACRO definitions
+#define TD_MOD_UNINITIALIZED 0
+#define TD_MOD_INITIALIZED 1
+
+#define TD_MOD_UNCLEARD 0
+#define TD_MOD_CLEARD 1
+
+#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED
+#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD
+
+#define TD_CHECK_AND_SET_MODE_INIT(MOD) \
+  atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED)
+
+#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_MACRO_H_*/
\ No newline at end of file

source/dnode/vnode/impl/CMakeLists.txt

@@ -18,6 +18,6 @@ target_link_libraries(
 )
 
 # test
-#if(${BUILD_TEST})
-#    add_subdirectory(test)
-#endif(${BUILD_TEST})
\ No newline at end of file
+if(${BUILD_TEST})
+   add_subdirectory(test)
+endif(${BUILD_TEST})
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeBufferPool.h

@@ -28,6 +28,7 @@ typedef struct SVBufPool SVBufPool;
 int   vnodeOpenBufPool(SVnode *pVnode);
 void  vnodeCloseBufPool(SVnode *pVnode);
 void *vnodeMalloc(SVnode *pVnode, uint64_t size);
+bool  vnodeBufPoolIsFull(SVnode *pVnode);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/impl/inc/vnodeCommit.h

@@ -22,8 +22,8 @@
 extern "C" {
 #endif
 
-bool vnodeShouldCommit(SVnode *pVnode);
-int  vnodeAsyncCommit(SVnode *pVnode);
+#define vnodeShouldCommit vnodeBufPoolIsFull
+int vnodeAsyncCommit(SVnode *pVnode);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/impl/inc/vnodeDef.h

@@ -18,15 +18,17 @@
 
 #include "mallocator.h"
 #include "sync.h"
+#include "tcoding.h"
+#include "tdlist.h"
 #include "tlockfree.h"
 #include "wal.h"
-#include "tcoding.h"
 
 #include "vnode.h"
 #include "vnodeBufferPool.h"
 #include "vnodeCfg.h"
 #include "vnodeCommit.h"
 #include "vnodeFS.h"
+#include "vnodeMemAllocator.h"
 #include "vnodeRequest.h"
 #include "vnodeStateMgr.h"
 #include "vnodeSync.h"

source/dnode/vnode/impl/inc/vnodeMemAllocator.h

@@ -16,15 +16,37 @@
 #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
 #define _TD_VNODE_MEM_ALLOCATOR_H_
 
-#include "mallocator.h"
-#include "vnode.h"
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode);
-void           vnodeDestroyMemAllocator(SMemAllocator *pma);
+typedef struct SVArenaNode    SVArenaNode;
+typedef struct SVMemAllocator SVMemAllocator;
+
+struct SVArenaNode {
+  TD_SLIST_NODE(SVArenaNode);
+  uint64_t size;  // current node size
+  void *   ptr;
+  char     data[];
+};
+
+struct SVMemAllocator {
+  TD_DLIST_NODE(SVMemAllocator);
+  uint64_t     capacity;
+  uint64_t     ssize;
+  uint64_t     lsize;
+  SVArenaNode *pNode;
+  TD_SLIST(SVArenaNode) nlist;
+};
+
+SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize);
+void            vmaDestroy(SVMemAllocator *pVMA);
+void            vmaReset(SVMemAllocator *pVMA);
+void *          vmaMalloc(SVMemAllocator *pVMA, uint64_t size);
+void            vmaFree(SVMemAllocator *pVMA, void *ptr);
+bool            vmaIsFull(SVMemAllocator *pVMA);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/impl/src/vnodeArenaMAImpl.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnodeDef.h"
+
+static SVArenaNode *vArenaNodeNew(uint64_t capacity);
+static void         vArenaNodeFree(SVArenaNode *pNode);
+
+SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) {
+  SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA));
+  if (pVMA == NULL) {
+    return NULL;
+  }
+
+  pVMA->capacity = capacity;
+  pVMA->ssize = ssize;
+  pVMA->lsize = lsize;
+  tSListInit(&(pVMA->nlist));
+
+  pVMA->pNode = vArenaNodeNew(capacity);
+  if (pVMA->pNode == NULL) {
+    free(pVMA);
+    return NULL;
+  }
+
+  tSListPush(&(pVMA->nlist), pVMA->pNode);
+
+  return pVMA;
+}
+
+void vmaDestroy(SVMemAllocator *pVMA) {
+  if (pVMA) {
+    while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) {
+      SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
+      tSListPop(&(pVMA->nlist));
+      vArenaNodeFree(pNode);
+    }
+
+    free(pVMA);
+  }
+}
+
+void vmaReset(SVMemAllocator *pVMA) {
+  while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) {
+    SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
+    tSListPop(&(pVMA->nlist));
+    vArenaNodeFree(pNode);
+  }
+
+  SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
+  pNode->ptr = pNode->data;
+}
+
+void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) {
+  SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
+  void *       ptr;
+
+  if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) {
+    uint64_t capacity = MAX(pVMA->ssize, size);
+    pNode = vArenaNodeNew(capacity);
+    if (pNode == NULL) {
+      // TODO: handle error
+      return NULL;
+    }
+
+    tSListPush(&(pVMA->nlist), pNode);
+  }
+
+  ptr = pNode->ptr;
+  pNode->ptr = POINTER_SHIFT(ptr, size);
+
+  return ptr;
+}
+
+void vmaFree(SVMemAllocator *pVMA, void *ptr) {
+  // TODO
+}
+
+bool vmaIsFull(SVMemAllocator *pVMA) {
+  SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
+
+  return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) ||
+         (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize);
+}
+
+/* ------------------------ STATIC METHODS ------------------------ */
+static SVArenaNode *vArenaNodeNew(uint64_t capacity) {
+  SVArenaNode *pNode = NULL;
+
+  pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity);
+  if (pNode == NULL) {
+    return NULL;
+  }
+
+  pNode->size = capacity;
+  pNode->ptr = pNode->data;
+
+  return pNode;
+}
+
+static void vArenaNodeFree(SVArenaNode *pNode) {
+  if (pNode) {
+    free(pNode);
+  }
+}
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeBufferPool.c

@@ -19,110 +19,59 @@
 #define VNODE_BUF_POOL_SHARDS 3
 
 struct SVBufPool {
-  // buffer pool impl
-  SList      free;
-  SList      incycle;
-  SListNode *inuse;
+  TD_DLIST(SVMemAllocator) free;
+  TD_DLIST(SVMemAllocator) incycle;
+  SVMemAllocator *inuse;
   // MAF for submodules
-  SMemAllocatorFactory maf;
+  // SMemAllocatorFactory maf;
 };
 
-typedef enum {
-  // Heap allocator
-  E_V_HEAP_ALLOCATOR = 0,
-  // Arena allocator
-  E_V_ARENA_ALLOCATOR
-} EVMemAllocatorT;
-
-typedef struct {
-  /* TODO */
-} SVHeapAllocator;
-
-typedef struct SVArenaNode {
-  struct SVArenaNode *prev;
-  uint64_t            size;
-  void *              ptr;
-  char                data[];
-} SVArenaNode;
-
-typedef struct {
-  uint64_t     ssize;  // step size
-  uint64_t     lsize;  // limit size
-  SVArenaNode *inuse;
-  SVArenaNode  node;
-} SVArenaAllocator;
-
-typedef struct {
-  SVnode *   pVnode;
-  SListNode *pNode;
-} SVMAWrapper;
-
-typedef struct {
-  T_REF_DECLARE()
-  uint64_t        capacity;
-  EVMemAllocatorT type;
-  union {
-    SVHeapAllocator  vha;
-    SVArenaAllocator vaa;
-  };
-} SVMemAllocator;
-
-static SListNode *    vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type);
-static void           vBufPoolFreeNode(SListNode *pNode);
-static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf);
-static void           vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma);
-static void *         vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size);
-
 int vnodeOpenBufPool(SVnode *pVnode) {
-  uint64_t        capacity;
-  EVMemAllocatorT type = E_V_ARENA_ALLOCATOR;
+  uint64_t capacity;
+  // EVMemAllocatorT type = E_V_ARENA_ALLOCATOR;
 
   if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) {
     /* TODO */
     return -1;
   }
 
-  tdListInit(&(pVnode->pBufPool->free), 0);
-  tdListInit(&(pVnode->pBufPool->incycle), 0);
+  tDListInit(&(pVnode->pBufPool->free));
+  tDListInit(&(pVnode->pBufPool->incycle));
 
+  pVnode->pBufPool->inuse = NULL;
+
+  // TODO
   capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS;
-  if (pVnode->config.isHeapAllocator) {
-    type = E_V_HEAP_ALLOCATOR;
-  }
 
   for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) {
-    SListNode *pNode = vBufPoolNewNode(capacity, type);
-    if (pNode == NULL) {
-      vnodeCloseBufPool(pVnode);
+    SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize);
+    if (pVMA == NULL) {
+      // TODO: handle error
       return -1;
     }
 
-    tdListAppendNode(&(pVnode->pBufPool->free), pNode);
+    tDListAppend(&(pVnode->pBufPool->free), pVMA);
   }
 
-  pVnode->pBufPool->maf.impl = pVnode;
-  pVnode->pBufPool->maf.create = vBufPoolCreateMA;
-  pVnode->pBufPool->maf.destroy = vBufPoolDestroyMA;
-
   return 0;
 }
 
 void vnodeCloseBufPool(SVnode *pVnode) {
-  SListNode *pNode;
   if (pVnode->pBufPool) {
-    // Clear free list
-    while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) {
-      vBufPoolFreeNode(pNode);
-    }
+    vmaDestroy(pVnode->pBufPool->inuse);
 
-    // Clear incycle list
-    while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) {
-      vBufPoolFreeNode(pNode);
+    while (true) {
+      SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle));
+      if (pVMA == NULL) break;
+      tDListPop(&(pVnode->pBufPool->incycle), pVMA);
+      vmaDestroy(pVMA);
     }
 
-    // Free inuse node
-    if (pVnode->pBufPool->inuse) {
-      vBufPoolFreeNode(pVnode->pBufPool->inuse);
+    while (true) {
+      SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free));
+      if (pVMA == NULL) break;
+      tDListPop(&(pVnode->pBufPool->free), pVMA);
+      vmaDestroy(pVMA);
     }
 
     free(pVnode->pBufPool);
@@ -131,48 +80,73 @@ void vnodeCloseBufPool(SVnode *pVnode) {
 }
 
 void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
-  void *ptr;
-
-  if (pVnode->pBufPool->inuse == NULL) {
-    SListNode *pNode;
-    while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) == NULL) {
-      // todo
-      // tsem_wait();
-      ASSERT(0);
+  SVBufPool *pBufPool = pVnode->pBufPool;
+
+  if (pBufPool->inuse == NULL) {
+    while (true) {
+      // TODO: add sem_wait and sem_post
+      pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free));
+      if (pBufPool->inuse) {
+        tDListPop(&(pBufPool->free), pBufPool->inuse);
+        break;
+      }
     }
-
-    pVnode->pBufPool->inuse = pNode;
   }
 
-  SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data);
-  return vBufPoolMalloc(pvma, size);
+  return vmaMalloc(pBufPool->inuse, size);
 }
 
-/* ------------------------ STATIC METHODS ------------------------ */
-static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */
-  pvaa->ssize = ssize;
-  pvaa->lsize = lsize;
-  pvaa->inuse = &pvaa->node;
-
-  pvaa->node.prev = NULL;
-  pvaa->node.size = capacity;
-  pvaa->node.ptr = pvaa->node.data;
+bool vnodeBufPoolIsFull(SVnode *pVnode) {
+  if (pVnode->pBufPool->inuse == NULL) return false;
+  return vmaIsFull(pVnode->pBufPool->inuse);
 }
 
-static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { /* TODO */
-  while (pvaa->inuse != &(pvaa->node)) {
-    SVArenaNode *pANode = pvaa->inuse;
-    pvaa->inuse = pANode->prev;
-    free(pANode);
-  }
-}
+#if 0
 
+typedef enum {
+  // Heap allocator
+  E_V_HEAP_ALLOCATOR = 0,
+  // Arena allocator
+  E_V_ARENA_ALLOCATOR
+} EVMemAllocatorT;
+
+typedef struct {
+  /* TODO */
+} SVHeapAllocator;
+
+typedef struct SVArenaNode {
+  struct SVArenaNode *prev;
+  uint64_t            size;
+  void *              ptr;
+  char                data[];
+} SVArenaNode;
+
+typedef struct {
+  uint64_t     ssize;  // step size
+  uint64_t     lsize;  // limit size
+  SVArenaNode *inuse;
+  SVArenaNode  node;
+} SVArenaAllocator;
+
+typedef struct {
+  SVnode *   pVnode;
+  SListNode *pNode;
+} SVMAWrapper;
+
+
+static SListNode *    vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type);
+static void           vBufPoolFreeNode(SListNode *pNode);
+static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf);
+static void           vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma);
+static void *         vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size);
+
+/* ------------------------ STATIC METHODS ------------------------ */
 static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) {
   SListNode *     pNode;
   SVMemAllocator *pvma;
   uint64_t        msize;
-  uint64_t        ssize = 0;  // TODO
-  uint64_t        lsize = 0;  // TODO
+  uint64_t        ssize = 4096;  // TODO
+  uint64_t        lsize = 1024;  // TODO
 
   msize = sizeof(SListNode) + sizeof(SVMemAllocator);
   if (type == E_V_ARENA_ALLOCATOR) {
@@ -317,4 +291,5 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) {
     tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode));
     // tsem_post(); todo: sem_post
   }
-}
\ No newline at end of file
+}
+#endif
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeCfg.c

@@ -15,7 +15,7 @@
 
 #include "vnodeDef.h"
 
-const SVnodeCfg defaultVnodeOptions = {0}; /* TODO */
+const SVnodeCfg defaultVnodeOptions = {.wsize = 16*1024*1024, .walCfg = {.walLevel = TAOS_WAL_WRITE}}; /* TODO */
 
 void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */
   vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions);

source/dnode/vnode/impl/src/vnodeCommit.c

@@ -18,8 +18,6 @@
 static int vnodeStartCommit(SVnode *pVnode);
 static int vnodeEndCommit(SVnode *pVnode);
 
-bool vnodeShouldCommit(SVnode *pVnode) { return false; }
-
 int vnodeAsyncCommit(SVnode *pVnode) {
 #if 0
   if (vnodeStartCommit(pVnode) < 0) {

source/dnode/vnode/impl/src/vnodeMain.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "tmacro.h"
 #include "vnodeDef.h"
 
 static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg);
@@ -20,8 +21,14 @@ static void    vnodeFree(SVnode *pVnode);
 static int     vnodeOpenImpl(SVnode *pVnode);
 static void    vnodeCloseImpl(SVnode *pVnode);
 
+TD_DEF_MOD_INIT_FLAG(vnode);
+TD_DEF_MOD_CLEAR_FLAG(vnode);
+
 int vnodeInit() {
-  // TODO
+  if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) {
+    return 0;
+  }
+
   if (walInit() < 0) {
     return -1;
   }
@@ -30,6 +37,10 @@ int vnodeInit() {
 }
 
 void vnodeClear() {
+  if (TD_CHECK_AND_SET_MOD_CLEAR(vnode) == TD_MOD_CLEARD) {
+    return;
+  }
+
   walCleanUp();
 }
 

source/dnode/vnode/impl/src/vnodeMemAllocator.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnodeDef.h"
-
-SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) {
-  SMemAllocator *pma = NULL;
-  /* TODO */
-  return pma;
-}
-
-void vnodeDestroyMemAllocator(SMemAllocator *pma) {
-  // TODO
-}
-
-#if 0
-#define VNODE_HEAP_ALLOCATOR 0
-#define VNODE_ARENA_ALLOCATOR 1
-
-typedef struct {
-  uint64_t tsize;
-  uint64_t used;
-} SVHeapAllocator;
-
-typedef struct SVArenaNode {
-  struct SVArenaNode *prev;
-  void *              nptr;
-  char                data[];
-} SVArenaNode;
-
-typedef struct {
-  SVArenaNode *inuse;
-  SVArenaNode  node;
-} SVArenaAllocator;
-
-typedef struct {
-  int8_t   type;
-  uint64_t tsize;
-  T_REF_DECLARE()
-  union {
-    SVHeapAllocator  vha;
-    SVArenaAllocator vaa;
-  };
-} SVMemAllocator;
-
-SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) {
-  SMemAllocator * pma;
-  uint64_t        msize;
-  SVMemAllocator *pva;
-
-  msize = sizeof(*pma) + sizeof(SVMemAllocator);
-  if (type == VNODE_ARENA_ALLOCATOR) {
-    msize += tsize;
-  }
-
-  pma = (SMemAllocator *)calloc(1, msize);
-  if (pma == NULL) {
-    return NULL;
-  }
-
-  pma->impl = POINTER_SHIFT(pma, sizeof(*pma));
-  pva = (SVMemAllocator *)(pma->impl);
-  pva->type = type;
-  pva->tsize = tsize;
-
-  if (type == VNODE_HEAP_ALLOCATOR) {
-    pma->malloc = NULL;
-    pma->calloc = NULL;
-    pma->realloc = NULL;
-    pma->free = NULL;
-    pma->usage = NULL;
-  } else if (type == VNODE_ARENA_ALLOCATOR) {
-    pma->malloc = NULL;
-    pma->calloc = NULL;
-    pma->realloc = NULL;
-    pma->free = NULL;
-    pma->usage = NULL;
-  } else {
-    ASSERT(0);
-  }
-
-  return pma;
-}
-
-void vnodeDestroyMemAllocator(SMemAllocator *pma) {
-  // TODO
-}
-
-void vnodeRefMemAllocator(SMemAllocator *pma) {
-  // TODO
-}
-
-void vnodeUnrefMemAllocator(SMemAllocator *pma) {
-  // TODO
-}
-
-/* ------------------------ Heap Allocator IMPL ------------------------ */
-
-/* ------------------------ Arena Allocator IMPL ------------------------ */
-
-#endif
\ No newline at end of file

source/dnode/vnode/impl/test/CMakeLists.txt

@@ -4,4 +4,9 @@ target_sources(vnodeApiTests
     PRIVATE
     "vnodeApiTests.cpp"
 )
-target_link_libraries(vnodeApiTests vnode gtest gtest_main)
\ No newline at end of file
+target_link_libraries(vnodeApiTests vnode gtest gtest_main)
+
+add_test(
+    NAME vnode_api_tests 
+    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests
+    )
\ No newline at end of file

source/dnode/vnode/impl/test/vnodeApiTests.cpp

+/**
+ * @file vnodeApiTests.cpp
+ * @author hzcheng (hzcheng@taosdata.com)
+ * @brief VNODE module API tests
+ * @version 0.1
+ * @date 2021-12-13
+ *
+ * @copyright Copyright (c) 2021
+ *
+ */
+
 #include <gtest/gtest.h>
 #include <iostream>