TD-100

src/tsdb/inc/tsdbMain.h

@@ -351,13 +351,14 @@ typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
 
 typedef struct {
   tsdb_rw_helper_t type;  // helper type
-  int              maxTables;
-  int              maxRowSize;
-  int              maxRows;
-  int              maxCols;
-  int              minRowsPerFileBlock;
-  int              maxRowsPerFileBlock;
-  int8_t           compress;
+
+  int    maxTables;
+  int    maxRowSize;
+  int    maxRows;
+  int    maxCols;
+  int    minRowsPerFileBlock;
+  int    maxRowsPerFileBlock;
+  int8_t compress;
 } SHelperCfg;
 
 typedef struct {
@@ -388,17 +389,14 @@ typedef struct {
   // For file set usage
   SHelperFile files;
   SCompIdx *  pCompIdx;
-  // size_t      compIdxSize;
 
   // For table set usage
   SHelperTable tableInfo;
   SCompInfo *  pCompInfo;
-  // size_t       compInfoSize;
   bool         hasOldLastBlock;
 
   // For block set usage
   SCompData *pCompData;
-  // size_t     compDataSize;
   SDataCols *pDataCols[2];
 
 } SRWHelper;

src/tsdb/src/tsdbFile.c

@@ -25,6 +25,7 @@
 
 #include "tutil.h"
 #include "tsdbMain.h"
+#include "tchecksum.h"
 
 const char *tsdbFileSuffix[] = {
     ".head",  // TSDB_FILE_TYPE_HEAD
@@ -310,7 +311,7 @@ static int tsdbWriteFileHead(SFile *pFile) {
 }
 
 static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
-  int   size = sizeof(SCompIdx) * maxTables;
+  int   size = sizeof(SCompIdx) * maxTables + sizeof(TSCKSUM);
   void *buf = calloc(1, size);
   if (buf == NULL) return -1;
 
@@ -319,6 +320,8 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
     return -1;
   }
 
+  taosCalcChecksumAppend(0, (uint8_t *)buf, size);
+
   if (write(pFile->fd, buf, size) < 0) {
     free(buf);
     return -1;

src/tsdb/src/tsdbMain.c

@@ -56,7 +56,8 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock);
 static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg);
 static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname);
 static void *  tsdbCommitData(void *arg);
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols);
+static int     tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper,
+                                SDataCols *pDataCols);
 static TSKEY   tsdbNextIterKey(SSkipListIterator *pIter);
 static int     tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey);
 // static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len,
@@ -847,6 +848,7 @@ static void *tsdbCommitData(void *arg) {
       .maxRows = pCfg->maxRowsPerFileBlock,
       .maxCols = pMeta->maxCols,
       .minRowsPerFileBlock = pCfg->minRowsPerFileBlock,
+      .maxRowsPerFileBlock = pCfg->maxRowsPerFileBlock,
       .compress = 2  // TODO make it a configuration
   };
   if (tsdbInitHelper(&whelper, &hcfg) < 0) goto _exit;
@@ -911,6 +913,8 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
   // Loop to commit data in each table
   for (int tid = 0; tid < pCfg->maxTables; tid++) {
     STable *           pTable = pMeta->tables[tid];
+    if (pTable == NULL) continue;
+
     SSkipListIterator *pIter = iters[tid];
 
     // Set the helper and the buffer dataCols object to help to write this table
@@ -929,6 +933,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
       ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey);
 
       int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols);
+      ASSERT(rowsWritten != 0);
       if (rowsWritten < 0) goto _err;
       ASSERT(rowsWritten <= pDataCols->numOfPoints);
 

src/tsdb/tests/tsdbTests.cpp

@@ -48,98 +48,99 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) {
   ASSERT_EQ(memcmp(pTable->schema, tTable->schema, sizeof(STSchema) + sizeof(STColumn) * nCols), 0);
 }
 
-TEST(TsdbTest, DISABLED_createRepo) {
-// TEST(TsdbTest, createRepo) {
-  // STsdbCfg config;
-
-  // // 1. Create a tsdb repository
-  // tsdbSetDefaultCfg(&config);
-  // tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL);
-  // ASSERT_NE(pRepo, nullptr);
-
-  // // 2. Create a normal table
-  // STableCfg tCfg;
-  // ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
-  // ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
-
-  // int       nCols = 5;
-  // STSchema *schema = tdNewSchema(nCols);
-
-  // for (int i = 0; i < nCols; i++) {
-  //   if (i == 0) {
-  //     tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
-  //   } else {
-  //     tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
-  //   }
-  // }
-
-  // tsdbTableSetSchema(&tCfg, schema, true);
-
-  // tsdbCreateTable(pRepo, &tCfg);
-
-  // // // 3. Loop to write some simple data
-  // int nRows = 1;
-  // int rowsPerSubmit = 1;
-  // int64_t start_time = 1584081000000;
-
-  // SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit);
-
-  // double stime = getCurTime();
-
-  // for (int k = 0; k < nRows/rowsPerSubmit; k++) {
-  //   memset((void *)pMsg, 0, sizeof(SSubmitMsg));
-  //   SSubmitBlk *pBlock = pMsg->blocks;
-  //   pBlock->uid = 987607499877672L;
-  //   pBlock->tid = 0;
-  //   pBlock->sversion = 0;
-  //   pBlock->len = 0;
-  //   for (int i = 0; i < rowsPerSubmit; i++) {
-  //     // start_time += 1000;
-  //     start_time += 1000;
-  //     SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
-  //     tdInitDataRow(row, schema);
-
-  //     for (int j = 0; j < schemaNCols(schema); j++) {
-  //       if (j == 0) {  // Just for timestamp
-  //         tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j));
-  //       } else {  // For int
-  //         int val = 10;
-  //         tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j));
-  //       }
-  //     }
-  //     pBlock->len += dataRowLen(row);
-  //   }
-  //   pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
-  //   pMsg->numOfBlocks = 1;
-
-  //   pBlock->len = htonl(pBlock->len);
-  //   pBlock->numOfRows = htonl(pBlock->numOfRows);
-  //   pBlock->uid = htobe64(pBlock->uid);
-  //   pBlock->tid = htonl(pBlock->tid);
-
-  //   pBlock->sversion = htonl(pBlock->sversion);
-  //   pBlock->padding = htonl(pBlock->padding);
-
-  //   pMsg->length = htonl(pMsg->length);
-  //   pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
-  //   pMsg->compressed = htonl(pMsg->numOfBlocks);
-
-  //   tsdbInsertData(pRepo, pMsg);
-  // }
-
-  // double etime = getCurTime();
-
-  // void *ptr = malloc(150000);
-  // free(ptr);
-
-  // printf("Spent %f seconds to write %d records\n", etime - stime, nRows);
-
-  // tsdbCloseRepo(pRepo);
+// TEST(TsdbTest, DISABLED_createRepo) {
+TEST(TsdbTest, createRepo) {
+  STsdbCfg config;
 
+  // 1. Create a tsdb repository
+  tsdbSetDefaultCfg(&config);
+  ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0);
+
+  tsdb_repo_t *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
+  ASSERT_NE(pRepo, nullptr);
+
+  // 2. Create a normal table
+  STableCfg tCfg;
+  ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
+  ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
+
+  int       nCols = 5;
+  STSchema *schema = tdNewSchema(nCols);
+
+  for (int i = 0; i < nCols; i++) {
+    if (i == 0) {
+      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
+    } else {
+      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
+    }
+  }
+
+  tsdbTableSetSchema(&tCfg, schema, true);
+
+  tsdbCreateTable(pRepo, &tCfg);
+
+  // // 3. Loop to write some simple data
+  int nRows = 10000000;
+  int rowsPerSubmit = 10;
+  int64_t start_time = 1584081000000;
+
+  SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit);
+
+  double stime = getCurTime();
+
+  for (int k = 0; k < nRows/rowsPerSubmit; k++) {
+    memset((void *)pMsg, 0, sizeof(SSubmitMsg));
+    SSubmitBlk *pBlock = pMsg->blocks;
+    pBlock->uid = 987607499877672L;
+    pBlock->tid = 0;
+    pBlock->sversion = 0;
+    pBlock->len = 0;
+    for (int i = 0; i < rowsPerSubmit; i++) {
+      // start_time += 1000;
+      start_time += 1000;
+      SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
+      tdInitDataRow(row, schema);
+
+      for (int j = 0; j < schemaNCols(schema); j++) {
+        if (j == 0) {  // Just for timestamp
+          tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j));
+        } else {  // For int
+          int val = 10;
+          tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j));
+        }
+      }
+      pBlock->len += dataRowLen(row);
+    }
+    pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
+    pMsg->numOfBlocks = 1;
+
+    pBlock->len = htonl(pBlock->len);
+    pBlock->numOfRows = htonl(pBlock->numOfRows);
+    pBlock->uid = htobe64(pBlock->uid);
+    pBlock->tid = htonl(pBlock->tid);
+
+    pBlock->sversion = htonl(pBlock->sversion);
+    pBlock->padding = htonl(pBlock->padding);
+
+    pMsg->length = htonl(pMsg->length);
+    pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
+    pMsg->compressed = htonl(pMsg->numOfBlocks);
+
+    tsdbInsertData(pRepo, pMsg);
+  }
+
+  double etime = getCurTime();
+
+  void *ptr = malloc(150000);
+  free(ptr);
+
+  printf("Spent %f seconds to write %d records\n", etime - stime, nRows);
+
+  tsdbCloseRepo(pRepo);
 }
 
-// TEST(TsdbTest, DISABLED_openRepo) {
-TEST(TsdbTest, openRepo) {
+TEST(TsdbTest, DISABLED_openRepo) {
+// TEST(TsdbTest, openRepo) {
   // tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL);
   // ASSERT_NE(repo, nullptr);