more

include/dnode/vnode/vnode.h

@@ -36,6 +36,8 @@ typedef struct SVnodeCfg {
   struct {
     /** write buffer size */
     uint64_t wsize;
+    uint64_t ssize;
+    uint64_t lsize;
     /** use heap allocator or arena allocator */
     bool isHeapAllocator;
   };

include/util/tdlist.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_TDLIST_H_
+#define _TD_UTIL_TDLIST_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TD_LIST_NODE(S) \
+  struct {              \
+    S *prev_;           \
+    S *next_;           \
+  }
+
+#define TD_LIST(S) \
+  struct {         \
+    S * head_;     \
+    S * tail_;     \
+    int neles_;    \
+  }
+
+#define tlistInit(l)              \
+  (l)->head_ = (l)->tail_ = NULL; \
+  (l)->neles_ = 0;
+
+#define tlistHead(l) (l)->head_
+#define tlistTail(l) (l)->tail_
+#define tlistNEles(l) (l)->neles_
+
+#define tlistAppend(l, n)           \
+  if ((l)->head_ == NULL) {         \
+    (n)->prev_ = (n)->next_ = NULL; \
+    (l)->head_ = (l)->tail_ = (n);  \
+  } else {                          \
+    (n)->prev_ = (l)->tail_;        \
+    (n)->next_ = NULL;              \
+    (l)->tail_->next_ = (n);        \
+    (l)->tail_ = (n);               \
+  }                                 \
+  (l)->neles_ += 1;
+
+#define tlistPrepend(l, n)          \
+  if ((l)->head_ == NULL) {         \
+    (n)->prev_ = (n)->next_ = NULL; \
+    (l)->head_ = (l)->tail_ = (n);  \
+  } else {                          \
+    (n)->prev_ = NULL;              \
+    (n)->next_ = (l)->head_;        \
+    (l)->head_->prev_ = (n);        \
+    (l)->head_ = (n);               \
+  }                                 \
+  (l)->neles_ += 1;
+
+#define tlistPop(l, n)                  \
+  ({                                    \
+    if ((n)) {                          \
+      if ((l)->head_ == (n)) {          \
+        (l)->head_ = (n)->next_;        \
+      }                                 \
+      if ((l)->tail_ == (n)) {          \
+        (l)->tail_ = (n)->prev_;        \
+      }                                 \
+      if ((n)->prev_ != NULL) {         \
+        (n)->prev_->next_ = (n)->next_; \
+      }                                 \
+      if ((n)->next_ != NULL) {         \
+        (n)->next_->prev_ = (n)->prev_; \
+      }                                 \
+      (l)->neles_ -= 1;                 \
+      (n)->prev_ = (n)->next_ = NULL;   \
+    }                                   \
+    (n);                                \
+  })
+
+#define tlistPopHead(l) tlistPop(l, (l)->head_)
+
+#define tlistPopTail(l) tlistPop(l, (l)->tail_)
+
+#define tlistIterInit(it, l, dir)
+
+// List iterator
+#define TD_LIST_FITER 0
+#define TD_LIST_BITER 1
+#define TD_LIST_ITER(S)    \
+  struct {                 \
+    int it_dir_;           \
+    S * it_next_;          \
+    S * it_ptr_;           \
+    TD_LIST(S) * it_list_; \
+  }
+
+#define tlistIterInit(it, l, dir) \
+  (it)->it_dir_ = (dir);          \
+  (it)->it_list_ = l;             \
+  if ((dir) == TD_LIST_FITER) {   \
+    (it)->it_next_ = (l)->head_;  \
+  } else {                        \
+    (it)->it_next_ = (l)->tail_;  \
+  }
+
+#define tlistIterNext(it)                       \
+  ({                                            \
+    (it)->it_ptr_ = (it)->it_next_;             \
+    if ((it)->it_next_ != NULL) {               \
+      if ((it)->it_dir_ == TD_LIST_FITER) {     \
+        (it)->it_next_ = (it)->it_next_->next_; \
+      } else {                                  \
+        (it)->it_next_ = (it)->it_next_->prev_; \
+      }                                         \
+    }                                           \
+    (it)->it_ptr_;                              \
+  })
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_TDLIST_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeDef.h

@@ -18,15 +18,17 @@
 
 #include "mallocator.h"
 #include "sync.h"
+#include "tcoding.h"
+#include "tdlist.h"
 #include "tlockfree.h"
 #include "wal.h"
-#include "tcoding.h"
 
 #include "vnode.h"
 #include "vnodeBufferPool.h"
 #include "vnodeCfg.h"
 #include "vnodeCommit.h"
 #include "vnodeFS.h"
+#include "vnodeMemAllocator.h"
 #include "vnodeRequest.h"
 #include "vnodeStateMgr.h"
 #include "vnodeSync.h"

source/dnode/vnode/impl/inc/vnodeMemAllocator.h

@@ -16,15 +16,35 @@
 #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
 #define _TD_VNODE_MEM_ALLOCATOR_H_
 
-#include "mallocator.h"
-#include "vnode.h"
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode);
-void           vnodeDestroyMemAllocator(SMemAllocator *pma);
+typedef struct SVArenaNode    SVArenaNode;
+typedef struct SVMemAllocator SVMemAllocator;
+
+struct SVArenaNode {
+  TD_LIST_NODE(SVArenaNode);
+  uint64_t nsize;  // current node size
+  void *   ptr;
+  char     data[];
+};
+
+struct SVMemAllocator {
+  TD_LIST_NODE(SVMemAllocator);
+  uint64_t capacity;
+  uint64_t ssize;
+  uint64_t lsize;
+  TD_LIST(SVArenaNode) nlist;
+};
+
+SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize);
+void            vmaDestroy(SVMemAllocator *pVMA);
+void            vmaReset(SVMemAllocator *pVMA);
+void *          vmaMalloc(SVMemAllocator *pVMA, uint64_t size);
+void            vmaFree(SVMemAllocator *pVMA, void *ptr);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/impl/src/vnodeArenaMAImpl.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "vnodeDef.h"
+
+static SVArenaNode *vArenaNodeNew(uint64_t capacity);
+static void         vArenaNodeFree(SVArenaNode *pNode);
+
+SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) {
+  SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA));
+  if (pVMA == NULL) {
+    // TODO: handle error
+    return NULL;
+  }
+
+  pVMA->capacity = capacity;
+  pVMA->ssize = ssize;
+  pVMA->lsize = lsize;
+  tlistInit(&(pVMA->nlist));
+
+  SVArenaNode *pNode = vArenaNodeNew(capacity);
+  if (pNode == NULL) {
+    // TODO
+    return NULL;
+  }
+
+  tlistAppend(&(pVMA->nlist), pNode);
+
+  return pVMA;
+}
+
+void vmaDestroy(SVMemAllocator *pVMA) {
+  // TODO
+}
+
+void vmaReset(SVMemAllocator *pVMA) {
+  // TODO
+}
+
+void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) {
+  // TODO
+  return NULL;
+}
+
+void vmaFree(SVMemAllocator *pVMA, void *ptr) {
+  // TODO
+}
+
+/* ------------------------ STATIC METHODS ------------------------ */
+static SVArenaNode *vArenaNodeNew(uint64_t capacity) {
+  SVArenaNode *pNode = NULL;
+  // TODO
+  return pNode;
+}
+
+static void vArenaNodeFree(SVArenaNode *pNode) {
+  // TODO
+}
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeBufferPool.c

@@ -19,14 +19,76 @@
 #define VNODE_BUF_POOL_SHARDS 3
 
 struct SVBufPool {
-  // buffer pool impl
-  SList      free;
-  SList      incycle;
-  SListNode *inuse;
+  TD_LIST(SVMemAllocator) free;
+  TD_LIST(SVMemAllocator) incycle;
+  SVMemAllocator *inuse;
   // MAF for submodules
-  SMemAllocatorFactory maf;
+  // SMemAllocatorFactory maf;
 };
 
+int vnodeOpenBufPool(SVnode *pVnode) {
+  uint64_t capacity;
+  // EVMemAllocatorT type = E_V_ARENA_ALLOCATOR;
+
+  if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) {
+    /* TODO */
+    return -1;
+  }
+
+  tlistInit(&(pVnode->pBufPool->free));
+  tlistInit(&(pVnode->pBufPool->incycle));
+
+  pVnode->pBufPool->inuse = NULL;
+
+  // TODO
+  capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS;
+
+  for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) {
+    SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize);
+    if (pVMA == NULL) {
+      // TODO: handle error
+      return -1;
+    }
+
+    tlistAppend(&(pVnode->pBufPool->free), pVMA);
+  }
+
+  return 0;
+}
+
+void vnodeCloseBufPool(SVnode *pVnode) {
+  if (pVnode->pBufPool) {
+    vmaDestroy(pVnode->pBufPool->inuse);
+
+    while (true) {
+      SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->incycle));
+      if (pVMA == NULL) break;
+      vmaDestroy(pVMA);
+    }
+
+    while (true) {
+      SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->free));
+      if (pVMA == NULL) break;
+      vmaDestroy(pVMA);
+    }
+
+    free(pVnode->pBufPool);
+    pVnode->pBufPool = NULL;
+  }
+}
+
+void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
+  // TODO
+  return NULL;
+}
+
+bool vnodeBufPoolIsFull(SVnode *pVnode) {
+  // TODO
+  return false;
+}
+
+#if 0
+
 typedef enum {
   // Heap allocator
   E_V_HEAP_ALLOCATOR = 0,
@@ -57,15 +119,6 @@ typedef struct {
   SListNode *pNode;
 } SVMAWrapper;
 
-typedef struct {
-  T_REF_DECLARE()
-  uint64_t        capacity;
-  EVMemAllocatorT type;
-  union {
-    SVHeapAllocator  vha;
-    SVArenaAllocator vaa;
-  };
-} SVMemAllocator;
 
 static SListNode *    vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type);
 static void           vBufPoolFreeNode(SListNode *pNode);
@@ -73,63 +126,6 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf);
 static void           vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma);
 static void *         vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size);
 
-int vnodeOpenBufPool(SVnode *pVnode) {
-  uint64_t        capacity;
-  EVMemAllocatorT type = E_V_ARENA_ALLOCATOR;
-
-  if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) {
-    /* TODO */
-    return -1;
-  }
-
-  tdListInit(&(pVnode->pBufPool->free), 0);
-  tdListInit(&(pVnode->pBufPool->incycle), 0);
-
-  capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS;
-  if (pVnode->config.isHeapAllocator) {
-    type = E_V_HEAP_ALLOCATOR;
-  }
-
-  for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) {
-    SListNode *pNode = vBufPoolNewNode(capacity, type);
-    if (pNode == NULL) {
-      vnodeCloseBufPool(pVnode);
-      return -1;
-    }
-
-    tdListAppendNode(&(pVnode->pBufPool->free), pNode);
-  }
-
-  pVnode->pBufPool->maf.impl = pVnode;
-  pVnode->pBufPool->maf.create = vBufPoolCreateMA;
-  pVnode->pBufPool->maf.destroy = vBufPoolDestroyMA;
-
-  return 0;
-}
-
-void vnodeCloseBufPool(SVnode *pVnode) {
-  SListNode *pNode;
-  if (pVnode->pBufPool) {
-    // Clear free list
-    while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) {
-      vBufPoolFreeNode(pNode);
-    }
-
-    // Clear incycle list
-    while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) {
-      vBufPoolFreeNode(pNode);
-    }
-
-    // Free inuse node
-    if (pVnode->pBufPool->inuse) {
-      vBufPoolFreeNode(pVnode->pBufPool->inuse);
-    }
-
-    free(pVnode->pBufPool);
-    pVnode->pBufPool = NULL;
-  }
-}
-
 void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
   void *ptr;
 
@@ -335,4 +331,5 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) {
     tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode));
     // tsem_post(); todo: sem_post
   }
-}
\ No newline at end of file
+}
+#endif
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeMemAllocator.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnodeDef.h"
-
-SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) {
-  SMemAllocator *pma = NULL;
-  /* TODO */
-  return pma;
-}
-
-void vnodeDestroyMemAllocator(SMemAllocator *pma) {
-  // TODO
-}
-
-#if 0
-#define VNODE_HEAP_ALLOCATOR 0
-#define VNODE_ARENA_ALLOCATOR 1
-
-typedef struct {
-  uint64_t tsize;
-  uint64_t used;
-} SVHeapAllocator;
-
-typedef struct SVArenaNode {
-  struct SVArenaNode *prev;
-  void *              nptr;
-  char                data[];
-} SVArenaNode;
-
-typedef struct {
-  SVArenaNode *inuse;
-  SVArenaNode  node;
-} SVArenaAllocator;
-
-typedef struct {
-  int8_t   type;
-  uint64_t tsize;
-  T_REF_DECLARE()
-  union {
-    SVHeapAllocator  vha;
-    SVArenaAllocator vaa;
-  };
-} SVMemAllocator;
-
-SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) {
-  SMemAllocator * pma;
-  uint64_t        msize;
-  SVMemAllocator *pva;
-
-  msize = sizeof(*pma) + sizeof(SVMemAllocator);
-  if (type == VNODE_ARENA_ALLOCATOR) {
-    msize += tsize;
-  }
-
-  pma = (SMemAllocator *)calloc(1, msize);
-  if (pma == NULL) {
-    return NULL;
-  }
-
-  pma->impl = POINTER_SHIFT(pma, sizeof(*pma));
-  pva = (SVMemAllocator *)(pma->impl);
-  pva->type = type;
-  pva->tsize = tsize;
-
-  if (type == VNODE_HEAP_ALLOCATOR) {
-    pma->malloc = NULL;
-    pma->calloc = NULL;
-    pma->realloc = NULL;
-    pma->free = NULL;
-    pma->usage = NULL;
-  } else if (type == VNODE_ARENA_ALLOCATOR) {
-    pma->malloc = NULL;
-    pma->calloc = NULL;
-    pma->realloc = NULL;
-    pma->free = NULL;
-    pma->usage = NULL;
-  } else {
-    ASSERT(0);
-  }
-
-  return pma;
-}
-
-void vnodeDestroyMemAllocator(SMemAllocator *pma) {
-  // TODO
-}
-
-void vnodeRefMemAllocator(SMemAllocator *pma) {
-  // TODO
-}
-
-void vnodeUnrefMemAllocator(SMemAllocator *pma) {
-  // TODO
-}
-
-/* ------------------------ Heap Allocator IMPL ------------------------ */
-
-/* ------------------------ Arena Allocator IMPL ------------------------ */
-
-#endif
\ No newline at end of file