Merge remote-tracking branch 'origin/3.0' into feature/dnode3

include/dnode/vnode/tsdb/tsdb.h

@@ -23,13 +23,12 @@ extern "C" {
 // TYPES EXPOSED
 typedef struct STsdb    STsdb;
 typedef struct STsdbCfg STsdbCfg;
-typedef struct STsdbMemAllocator STsdbMemAllocator;
 
 // STsdb
-STsdb *tsdbOpen(const char *path, const STsdbCfg *);
+STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg);
 void   tsdbClose(STsdb *);
 void   tsdbRemove(const char *path);
-int    tsdbInsertData(STsdb *pTsdb, void *pData, int len);
+int    tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg);
 
 // STsdbCfg
 int  tsdbOptionsInit(STsdbCfg *);
@@ -41,7 +40,6 @@ struct STsdbCfg {
   uint32_t keep0;
   uint32_t keep1;
   uint32_t keep2;
-  /* TODO */
 };
 
 #ifdef __cplusplus

include/util/tdef.h

@@ -25,7 +25,9 @@ extern "C" {
 #define TSDB__packed
 
 #define TSKEY int64_t
-#define TSKEY_INITIAL_VAL    INT64_MIN
+#define TSKEY_MIN INT64_MIN
+#define TSKEY_MAX (INT64_MAX - 1)
+#define TSKEY_INITIAL_VAL TSKEY_MIN
 
 // Bytes for each type.
 extern const int32_t TYPE_BYTES[15];

source/dnode/vnode/impl/CMakeLists.txt

@@ -19,5 +19,5 @@ target_link_libraries(
 
 # test
 if(${BUILD_TEST})
-#    add_subdirectory(test)
+   add_subdirectory(test)
 endif(${BUILD_TEST})
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeWrite.c

@@ -72,7 +72,9 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
           }
           break;
         case TSDB_MSG_TYPE_SUBMIT:
-          /* code */
+          if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) {
+            // TODO: handle error
+          }
           break;
         default:
           break;

source/dnode/vnode/impl/test/vnodeApiTests.cpp

@@ -92,6 +92,8 @@ TEST(vnodeApiTest, test_create_table_encode_and_decode_function) {
 #endif
 
 TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) {
+  vnodeDestroy("vnode1");
+
   GTEST_ASSERT_GE(vnodeInit(2), 0);
 
   // Create and open a vnode

source/dnode/vnode/tsdb/inc/tsdbDef.h

@@ -17,6 +17,10 @@
 #define _TD_TSDB_DEF_H_
 
 #include "mallocator.h"
+#include "taosmsg.h"
+#include "tdlist.h"
+#include "thash.h"
+#include "tskiplist.h"
 
 #include "tsdb.h"
 #include "tsdbMemTable.h"
@@ -29,6 +33,8 @@ extern "C" {
 struct STsdb {
   char *                path;
   STsdbCfg              options;
+  STsdbMemTable *       mem;
+  STsdbMemTable *       imem;
   SMemAllocatorFactory *pmaf;
 };
 

source/dnode/vnode/tsdb/inc/tsdbMemTable.h

@@ -22,10 +22,11 @@
 extern "C" {
 #endif
 
-typedef struct SMemTable {
-  /* TODO */
-  SMemAllocator *pma;
-} SMemTable;
+typedef struct STsdbMemTable STsdbMemTable;
+
+STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF);
+void           tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable);
+int            tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/tsdb/src/tsdbCommit.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbDef.h"
+
+int tsdbCommit(STsdb *pTsdb) {
+  // TODO
+  return 0;
+}
\ No newline at end of file

source/dnode/vnode/tsdb/src/tsdbMemTable.c

@@ -12,3 +12,83 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+
+#include "tsdbDef.h"
+
+#if 1
+typedef struct STbData {
+  TD_SLIST_NODE(STbData);
+  SSubmitMsg *pMsg;
+} STbData;
+#else
+typedef struct STbData {
+  TD_SLIST_NODE(STbData);
+  uint64_t   uid;  // TODO: change here as tb_uid_t
+  TSKEY      keyMin;
+  TSKEY      keyMax;
+  uint64_t   nRows;
+  SSkipList *pData;  // Here need a container, may not use the SL
+  T_REF_DECLARE()
+} STbData;
+#endif
+
+struct STsdbMemTable {
+  T_REF_DECLARE()
+  SRWLatch       latch;
+  TSKEY          keyMin;
+  TSKEY          keyMax;
+  uint64_t       nRow;
+  SMemAllocator *pMA;
+  // Container
+  TD_SLIST(STbData) list;
+};
+
+STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) {
+  STsdbMemTable *pMemTable;
+  SMemAllocator *pMA;
+
+  pMA = (*pMAF->create)(pMAF);
+  ASSERT(pMA != NULL);
+
+  pMemTable = (STsdbMemTable *)((*pMA->malloc)(pMA, sizeof(*pMemTable)));
+  if (pMemTable == NULL) {
+    (*pMAF->destroy)(pMAF, pMA);
+    return NULL;
+  }
+
+  T_REF_INIT_VAL(pMemTable, 1);
+  taosInitRWLatch(&(pMemTable->latch));
+  pMemTable->keyMin = TSKEY_MAX;
+  pMemTable->keyMax = TSKEY_MIN;
+  pMemTable->nRow = 0;
+  pMemTable->pMA = pMA;
+  tSListInit(&(pMemTable->list));
+
+  // TODO
+  return pMemTable;
+}
+
+void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) {
+  SMemAllocator *pMA = pMemTable->pMA;
+
+  if (pMA->free) {
+    // TODO
+    ASSERT(0);
+  }
+
+  (*pMAF->destroy)(pMAF, pMA);
+}
+
+int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) {
+  SMemAllocator *pMA = pMemTable->pMA;
+  STbData *      pTbData = (STbData *)((*pMA->malloc)(pMA, sizeof(*pTbData)));
+  if (pTbData == NULL) {
+    // TODO
+  }
+
+  tSListPush(&(pMemTable->list), pTbData);
+
+  return 0;
+}
+
+/* ------------------------ STATIC METHODS ------------------------ */
\ No newline at end of file

source/dnode/vnode/tsdb/src/tsdbWrite.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbDef.h"
+
+int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg) {
+  // Check if mem is there. If not, create one.
+  pTsdb->mem = tsdbNewMemTable(pTsdb->pmaf);
+  if (pTsdb->mem == NULL) {
+    return -1;
+  }
+  return tsdbInsertDataToMemTable(pTsdb->mem, pMsg);
+}
\ No newline at end of file

src/tsdb/inc/tsdbMemTable.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_TSDB_MEMTABLE_H_
-#define _TD_TSDB_MEMTABLE_H_
-
-typedef struct {
-  int   rowsInserted;
-  int   rowsUpdated;
-  int   rowsDeleteSucceed;
-  int   rowsDeleteFailed;
-  int   nOperations;
-  TSKEY keyFirst;
-  TSKEY keyLast;
-} SMergeInfo;
-
-typedef struct {
-  STable *           pTable;
-  SSkipListIterator *pIter;
-} SCommitIter;
-
-struct STableData {
-  uint64_t   uid;
-  TSKEY      keyFirst;
-  TSKEY      keyLast;
-  int64_t    numOfRows;
-  SSkipList* pData;
-  T_REF_DECLARE()
-};
-
-enum { TSDB_UPDATE_META, TSDB_DROP_META };
-
-#ifdef WINDOWS
-#pragma pack(push ,1) 
-typedef struct {
-#else
-typedef struct __attribute__((packed)){
-#endif
-  char     act;
-  uint64_t uid;
-} SActObj;
-#ifdef WINDOWS
-#pragma pack(pop) 
-#endif
-
-typedef struct {
-  int  len;
-  char cont[];
-} SActCont;
-
-int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
-int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
-int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot, SArray* pATable);
-void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot);
-void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
-int   tsdbAsyncCommit(STsdbRepo* pRepo);
-int   tsdbSyncCommitConfig(STsdbRepo* pRepo);
-int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
-                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
-void* tsdbCommitData(STsdbRepo* pRepo);
-
-static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator* pIter) {
-  if (pIter == NULL) return NULL;
-
-  SSkipListNode* node = tSkipListIterGet(pIter);
-  if (node == NULL) return NULL;
-
-  return (SMemRow)SL_GET_NODE_DATA(node);
-}
-
-static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
-  SMemRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
-
-  return memRowKey(row);
-}
-
-static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
-  SMemRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TKEY_NULL;
-
-  return memRowTKey(row);
-}
-
-#endif /* _TD_TSDB_MEMTABLE_H_ */
\ No newline at end of file

src/tsdb/src/tsdbBuffer.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "tsdbint.h"
-#include "tsdbHealth.h"
-
-#define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0)
-
-// ---------------- INTERNAL FUNCTIONS ----------------
-STsdbBufPool *tsdbNewBufPool() {
-  STsdbBufPool *pBufPool = (STsdbBufPool *)calloc(1, sizeof(*pBufPool));
-  if (pBufPool == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  int code = pthread_cond_init(&(pBufPool->poolNotEmpty), NULL);
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    goto _err;
-  }
-
-  pBufPool->bufBlockList = tdListNew(sizeof(STsdbBufBlock *));
-  if (pBufPool->bufBlockList == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  return pBufPool;
-
-_err:
-  tsdbFreeBufPool(pBufPool);
-  return NULL;
-}
-
-void tsdbFreeBufPool(STsdbBufPool *pBufPool) {
-  if (pBufPool) {
-    if (pBufPool->bufBlockList) {
-      ASSERT(listNEles(pBufPool->bufBlockList) == 0);
-      tdListFree(pBufPool->bufBlockList);
-    }
-
-    pthread_cond_destroy(&pBufPool->poolNotEmpty);
-
-    free(pBufPool);
-  }
-}
-
-int tsdbOpenBufPool(STsdbRepo *pRepo) {
-  STsdbCfg *    pCfg = &(pRepo->config);
-  STsdbBufPool *pPool = pRepo->pPool;
-
-  ASSERT(pPool != NULL);
-  pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; // MB
-  pPool->tBufBlocks = pCfg->totalBlocks;
-  pPool->nBufBlocks = 0;
-  pPool->nElasticBlocks = 0;
-  pPool->index = 0;
-  pPool->nRecycleBlocks = 0;
-
-  for (int i = 0; i < pCfg->totalBlocks; i++) {
-    STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
-    if (pBufBlock == NULL) goto _err;
-
-    if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
-      tsdbFreeBufBlock(pBufBlock);
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      goto _err;
-    }
-
-    pPool->nBufBlocks++;
-  }
-
-  tsdbDebug("vgId:%d buffer pool is opened! bufBlockSize:%d tBufBlocks:%d nBufBlocks:%d", REPO_ID(pRepo),
-            pPool->bufBlockSize, pPool->tBufBlocks, pPool->nBufBlocks);
-
-  return 0;
-
-_err:
-  tsdbCloseBufPool(pRepo);
-  return -1;
-}
-
-void tsdbCloseBufPool(STsdbRepo *pRepo) {
-  if (pRepo == NULL) return;
-
-  STsdbBufPool * pBufPool = pRepo->pPool;
-  STsdbBufBlock *pBufBlock = NULL;
-
-  if (pBufPool) {
-    SListNode *pNode = NULL;
-    while ((pNode = tdListPopHead(pBufPool->bufBlockList)) != NULL) {
-      tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
-      tsdbFreeBufBlock(pBufBlock);
-      free(pNode);
-    }
-  }
-
-  tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo));
-}
-
-SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) {
-  ASSERT(pRepo != NULL && pRepo->pPool != NULL);
-  ASSERT(IS_REPO_LOCKED(pRepo));
-
-  STsdbBufPool *pBufPool = pRepo->pPool;
-
-  while (POOL_IS_EMPTY(pBufPool)) {
-    if(tsDeadLockKillQuery) {
-      // supply new Block 
-      if(tsdbInsertNewBlock(pRepo) > 0) {
-        tsdbWarn("vgId:%d add new elastic block . elasticBlocks=%d cur free Blocks=%d", REPO_ID(pRepo), pBufPool->nElasticBlocks, pBufPool->bufBlockList->numOfEles);
-        break;
-      } else {
-        // no newBlock, kill query free
-        if(!tsdbUrgeQueryFree(pRepo))
-          tsdbWarn("vgId:%d Urge query free thread start failed.", REPO_ID(pRepo));
-      }
-    }
-
-    pRepo->repoLocked = false;
-    pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex));
-    pRepo->repoLocked = true;
-  }
-
-  SListNode *    pNode = tdListPopHead(pBufPool->bufBlockList);
-  ASSERT(pNode != NULL);
-  STsdbBufBlock *pBufBlock = NULL;
-  tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock));
-
-  pBufBlock->blockId = pBufPool->index++;
-  pBufBlock->offset = 0;
-  pBufBlock->remain = pBufPool->bufBlockSize;
-
-  tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId);
-  return pNode;
-}
-
-// ---------------- LOCAL FUNCTIONS ----------------
-STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) {
-  STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize);
-  if (pBufBlock == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  pBufBlock->blockId = 0;
-  pBufBlock->offset = 0;
-  pBufBlock->remain = bufBlockSize;
-
-  return pBufBlock;
-}
-
- void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); }
-
-int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) {
-  if (oldTotalBlocks == pRepo->config.totalBlocks) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  int err = TSDB_CODE_SUCCESS;
-
-  if (tsdbLockRepo(pRepo) < 0) return terrno;
-  STsdbBufPool* pPool = pRepo->pPool;
-
-  if (pRepo->config.totalBlocks > oldTotalBlocks) {
-    for (int i = 0; i < pRepo->config.totalBlocks - oldTotalBlocks; i++) {
-      STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize);
-      if (pBufBlock == NULL) goto err;
-
-      if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) {
-        tsdbFreeBufBlock(pBufBlock);
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        err = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        goto err;
-      }
-
-      pPool->nBufBlocks++;      
-    }
-    pthread_cond_signal(&pPool->poolNotEmpty);
-  } else {
-    pPool->nRecycleBlocks = oldTotalBlocks - pRepo->config.totalBlocks;
-  } 
-
-err:
-  tsdbUnlockRepo(pRepo);
-  return err;
-}
-
-void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic) {
-  STsdbBufBlock *pBufBlock = NULL;
-  tdListNodeGetData(pPool->bufBlockList, pNode, (void *)(&pBufBlock));
-  tsdbFreeBufBlock(pBufBlock);
-  free(pNode);
-  if(bELastic)
-  {
-    pPool->nElasticBlocks--;
-    tsdbWarn("pPool=%p elastic block reduce one . nElasticBlocks=%d cur free Blocks=%d", pPool, pPool->nElasticBlocks, pPool->bufBlockList->numOfEles);
-  }
-  else
-    pPool->nBufBlocks--;
-}
\ No newline at end of file