Merge pull request #1490 from taosdata/feature/2.0tsdb

Feature/2.0tsdb

src/vnode/tsdb/inc/tsdbMain.h

@@ -78,7 +78,7 @@ typedef struct STable {
 
 void *  tsdbEncodeTable(STable *pTable, int *contLen);
 STable *tsdbDecodeTable(void *cont, int contLen);
-void *  tsdbFreeEncode(void *cont);
+void    tsdbFreeEncode(void *cont);
 
 // ---------- TSDB META HANDLE DEFINITION
 typedef struct {
@@ -97,7 +97,7 @@ typedef struct {
   int        maxCols;
 } STsdbMeta;
 
-STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables);
+STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables);
 int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
 STSchema * tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable);
 
@@ -216,7 +216,7 @@ typedef struct {
 
 STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles);
 void        tsdbCloseFileH(STsdbFileH *pFileH);
-int         tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose);
+int         tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose);
 int         tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables);
 int         tsdbOpenFile(SFile *pFile, int oflag);
 int         tsdbCloseFile(SFile *pFile); SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid);

src/vnode/tsdb/src/tsdbCache.c

@@ -72,7 +72,7 @@ void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) {
   if (bytes > pCache->cacheBlockSize) return NULL;
 
   if (pCache->curBlock == NULL || pCache->curBlock->remain < bytes) {
-    if (pCache->curBlock !=NULL && (pCache->mem->list) >= pCache->totalCacheBlocks/2) {
+    if (pCache->curBlock !=NULL && listNEles(pCache->mem->list) >= pCache->totalCacheBlocks/2) {
       tsdbTriggerCommit(pCache->pRepo);
     }
 
@@ -130,7 +130,7 @@ static int tsdbAllocBlockFromPool(STsdbCache *pCache) {
 
   if (pCache->mem == NULL) { // Create a new one
     pCache->mem = (SCacheMem *)malloc(sizeof(SCacheMem));
-    if (pCache->mem == NULL) return NULL;
+    if (pCache->mem == NULL) return -1;
     pCache->mem->keyFirst = INT64_MAX;
     pCache->mem->keyLast = 0;
     pCache->mem->numOfPoints = 0;

src/vnode/tsdb/src/tsdbFile.c

@@ -33,7 +33,7 @@ const char *tsdbFileSuffix[] = {
 
 static int compFGroupKey(const void *key, const void *fgroup);
 static int compFGroup(const void *arg1, const void *arg2);
-static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname);
+static int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname);
 static int tsdbWriteFileHead(SFile *pFile);
 static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables);
 static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid);
@@ -53,8 +53,6 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) {
   }
 
   struct dirent *dp = NULL;
-  int fid = 0;
-  SFileGroup fGroup = {0};
   while ((dp = readdir(dir)) != NULL) {
     if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 1) == 0) continue;
     int fid = 0;
@@ -70,7 +68,7 @@ STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles) {
 
 void tsdbCloseFileH(STsdbFileH *pFileH) { free(pFileH); }
 
-static int tsdbInitFile(char *dataDir, int fid, char *suffix, SFile *pFile) {
+static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile) {
   tsdbGetFileName(dataDir, fid, suffix, pFile->fname);
   if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1;
   pFile->fd = -1;
@@ -82,7 +80,6 @@ static int tsdbInitFile(char *dataDir, int fid, char *suffix, SFile *pFile) {
 static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) {
   if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0;
 
-  char fname[128] = "\0";
   SFileGroup fGroup = {0};
   fGroup.fileId = fid;
 
@@ -163,8 +160,8 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {
   SFileGroup *ret = pIter->pFileGroup;
   if (ret == NULL) return NULL;
 
-  if (pIter->direction = TSDB_FGROUP_ITER_FORWARD) {
-    if (pIter->pFileGroup + 1 == pIter->base + pIter->numOfFGroups) {
+  if (pIter->direction == TSDB_FGROUP_ITER_FORWARD) {
+    if ((pIter->pFileGroup + 1) == (pIter->base + pIter->numOfFGroups)) {
       pIter->pFileGroup = NULL;
     } else {
       pIter->pFileGroup += 1;
@@ -223,7 +220,7 @@ int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInf
   if (pCompData == NULL) return -1;
 
   // Load data from the block
-  if (tsdbLoadDataBlock(pOutFile, pStartBlock, numOfBlocks, pCols, pCompData));
+  // if (tsdbLoadDataBlock(pOutFile, pStartBlock, numOfBlocks, pCols, pCompData));
 
   // Write data block to the file
   {
@@ -315,7 +312,7 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
   return 0;
 }
 
-static int tsdbGetFileName(char *dataDir, int fileId, char *suffix, char *fname) {
+static int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) {
   if (dataDir == NULL || fname == NULL) return -1;
 
   sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix);
@@ -348,7 +345,7 @@ SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) {
   return pGroup;
 }
 
-int tsdbCreateFile(char *dataDir, int fileId, char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose) {
+int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader, int toClose) {
   memset((void *)pFile, 0, sizeof(SFile));
   pFile->fd = -1;
 

src/vnode/tsdb/src/tsdbMain.c

@@ -806,7 +806,7 @@ static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) {
 }
 
 static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) {
-  SSkipListIterator **iters = (SSkipListIterator *)calloc(maxTables, sizeof(SSkipListIterator *));
+  SSkipListIterator **iters = (SSkipListIterator **)calloc(maxTables, sizeof(SSkipListIterator *));
   if (iters == NULL) return NULL;
 
   for (int tid = 0; tid < maxTables; tid++) {
@@ -893,9 +893,9 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
   SFileGroup *pGroup = NULL;
   SCompIdx *  pIndices = NULL;
   SCompInfo * pCompInfo = NULL;
-  size_t      compInfoSize = 0;
-  SCompBlock  compBlock;
-  SCompBlock *pBlock = &compBlock;
+  // size_t      compInfoSize = 0;
+  // SCompBlock  compBlock;
+  // SCompBlock *pBlock = &compBlock;
 
   TSKEY minKey = 0, maxKey = 0;
   tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
@@ -963,10 +963,11 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
           TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pTBlock, nBlocks);
 
           SCompBlock tBlock;
-          int64_t toffset, tlen;
+          int64_t toffset;
+          int32_t tlen;
           tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_LAST], pTBlock, nBlocks, pCols, &tBlock);
 
-          tsdbWriteBlockToFileImpl(&lFile, pCols, pCols->numOfPoints, &toffset, tlen, pTable->tableId.uid);
+          tsdbWriteBlockToFileImpl(&lFile, pCols, pCols->numOfPoints, &toffset, &tlen, pTable->tableId.uid);
           pTBlock = TSDB_COMPBLOCK_AT(pCompInfo, pIdx->numOfSuperBlocks);
           pTBlock->offset = toffset;
           pTBlock->len = tlen;
@@ -1164,7 +1165,6 @@ static int compareKeyBlock(const void *arg1, const void *arg2) {
 
 int tsdbWriteBlockToFile(STsdbRepo *pRepo, SFileGroup *pGroup, SCompIdx *pIdx, SCompInfo *pCompInfo, SDataCols *pCols, SCompBlock *pCompBlock, SFile *lFile, int64_t uid) {
   STsdbCfg * pCfg = &(pRepo->config);
-  SCompData *pCompData = NULL;
   SFile *    pFile = NULL;
   int        numOfPointsToWrite = 0;
   int64_t    offset = 0;

src/vnode/tsdb/src/tsdbMeta.c

@@ -89,7 +89,7 @@ STable *tsdbDecodeTable(void *cont, int contLen) {
   return pTable;
 }
 
-void *tsdbFreeEncode(void *cont) {
+void tsdbFreeEncode(void *cont) {
   if (cont != NULL) free(cont);
 }
 
@@ -124,7 +124,7 @@ void tsdbOrgMeta(void *pHandle) {
  * Initialize the meta handle
  * ASSUMPTIONS: VALID PARAMETER
  */
-STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables) {
+STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables) {
   STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta));
   if (pMeta == NULL) return NULL;
 

src/vnode/tsdb/src/tsdbMetaFile.c

@@ -174,7 +174,7 @@ int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t co
 
 void tsdbCloseMetaFile(SMetaFile *mfh) {
   if (mfh == NULL) return;
-  close(mfh);
+  close(mfh->fd);
 
   taosHashCleanup(mfh->map);
 }
@@ -210,7 +210,7 @@ static int tsdbCreateMetaFile(char *fname) {
 
   if (tsdbWriteMetaHeader(fd) < 0) {
     close(fd);
-    return NULL;
+    return -1;
   }
 
   return fd;
@@ -242,7 +242,7 @@ static int tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh) {
   mfh->fd = fd;
 
   void *buf = NULL;
-  int buf_size = 0;
+  // int buf_size = 0;
 
   SRecordInfo info;
   while (1) {