more work

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -172,7 +172,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) {
   SDelFile  *pDelFileW = NULL;  // TODO
 
   if (pDelFileR) {
-    code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR);
+    code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb, NULL);
     if (code) {
       goto _err;
     }
@@ -183,7 +183,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) {
     }
   }
 
-  code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, pDelFileW);
+  code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, pDelFileW, pTsdb);
   if (code) {
     goto _err;
   }
@@ -299,11 +299,11 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
   }
 
 _exit:
-  tsdbDebug("vgId:%d commit del data, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel);
+  tsdbDebug("vgId:%d commit del data done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel);
   return code;
 
 _err:
-  tsdbError("vgId:%d failed to commit del data since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d commit del data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -15,6 +15,9 @@
 
 #include "tsdb.h"
 
+#define TSDB_FHDR_SIZE 512
+#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F)
+
 // SDFileSetWritter ====================================================
 struct SDFileSetWritter {
   STsdb   *pTsdb;
@@ -75,13 +78,39 @@ int32_t tsdbLoadSBlockStatis(SDFileSetReader *pReader, SBlock *pBlock, SBlockSta
 
 // SDelFWriter ====================================================
 struct SDelFWriter {
+  STsdb    *pTsdb;
   SDelFile *pFile;
   TdFilePtr pWriteH;
 };
 
-int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile) {
+int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) {
   int32_t      code = 0;
-  // TODO
+  char        *fname = NULL;  // TODO
+  SDelFWriter *pDelFWriter;
+
+  pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter));
+  if (pDelFWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  pDelFWriter->pTsdb = pTsdb;
+  pDelFWriter->pFile = pFile;
+  pDelFWriter->pWriteH = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE);
+  if (pDelFWriter->pWriteH == NULL) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  if (taosLSeekFile(pDelFWriter->pWriteH, TSDB_FHDR_SIZE, SEEK_SET) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
@@ -91,15 +120,44 @@ int32_t tsdbDelFWriterClose(SDelFWriter *pWriter) {
   return code;
 }
 
-int32_t tsdbWriteDelData(SDelFWriter *pWriter, SDelData *pDelData, uint8_t **ppBuf) {
+int32_t tsdbWriteDelData(SDelFWriter *pWriter, SDelData *pDelData, uint8_t **ppBuf, SDelIdxItem *pItem) {
   int32_t code = 0;
+  int64_t size;
+
   // TODO
   return code;
 }
 
 int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SDelIdx *pDelIdx, uint8_t **ppBuf) {
   int32_t code = 0;
-  // TODO
+  int64_t size;
+
+  size = tPutDelIdx(NULL, pDelIdx) + sizeof(TSCKSUM);
+
+  // alloc
+  code = tsdbRealloc(ppBuf, size);
+  if (code) {
+    goto _err;
+  }
+
+  // encode
+  tPutDelIdx(*ppBuf, pDelIdx);
+
+  // checksum
+  taosCalcChecksumAppend(0, *ppBuf, size);
+
+  // write
+  if (taosWriteFile(pWriter->pWriteH, *ppBuf, size) < size) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  pWriter->pFile->offset = pWriter->pFile->size;
+  pWriter->pFile->size += size;
+
+  return code;
+
+_err:
   return code;
 }
 
@@ -110,15 +168,65 @@ struct SDelFReader {
   TdFilePtr pReadH;
 };
 
-int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile) {
+int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf) {
   int32_t      code = 0;
-  // TODO
+  char        *fname = NULL;  // todo
+  SDelFReader *pDelFReader;
+
+  // alloc
+  pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
+  if (pDelFReader == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  // open impl
+  pDelFReader->pTsdb = pTsdb;
+  pDelFReader->pFile = pFile;
+  pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ);
+  if (pDelFReader == NULL) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    taosMemoryFree(pDelFReader);
+    goto _err;
+  }
+
+  // load and check hdr if buffer is given
+  if (ppBuf) {
+    code = tsdbRealloc(ppBuf, TSDB_FHDR_SIZE);
+    if (code) {
+      goto _err;
+    }
+
+    if (taosReadFile(pDelFReader->pReadH, *ppBuf, TSDB_FHDR_SIZE) < TSDB_FHDR_SIZE) {
+      code = TSDB_CODE_FILE_CORRUPTED;
+      goto _err;
+    }
+
+    if (!taosCheckChecksumWhole(*ppBuf, TSDB_FHDR_SIZE)) {
+      code = TSDB_CODE_FILE_CORRUPTED;
+      goto _err;
+    }
+
+    // TODO: check the content
+  }
+
+_exit:
+  *ppReader = pDelFReader;
+  return code;
+
+_err:
+  *ppReader = NULL;
   return code;
 }
 
 int32_t tsdbDelFReaderClose(SDelFReader *pReader) {
   int32_t code = 0;
-  // TODO
+
+  if (pReader) {
+    taosCloseFile(&pReader->pReadH);
+    taosMemoryFree(pReader);
+  }
+
   return code;
 }
 
@@ -134,20 +242,20 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SDelIdx *pDelIdx, uint8_t **ppBuf)
   int64_t size = pReader->pFile->size - offset;
 
   // seek
-  if (taosLSeekFile(pReader->pReadH, pReader->pFile->offset, SEEK_SET) < 0) {
+  if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
 
-  // read
-  if (taosReadFile(pReader->pReadH, *ppBuf, size) < size) {
-    code = TAOS_SYSTEM_ERROR(errno);
+  // alloc
+  code = tsdbRealloc(ppBuf, size);
+  if (code) {
     goto _err;
   }
 
-  // realloc buf
-  code = tsdbRealloc(ppBuf, size);
-  if (code) {
+  // read
+  if (taosReadFile(pReader->pReadH, *ppBuf, size) < size) {
+    code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
 
@@ -158,6 +266,10 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SDelIdx *pDelIdx, uint8_t **ppBuf)
   }
 
   // decode
+  int32_t n = tGetDelIdx(*ppBuf, pDelIdx);
+  ASSERT(n == size - sizeof(TSCKSUM));
+  ASSERT(pDelIdx->delimiter == TSDB_FILE_DLMT);
+  ASSERT(pDelIdx->nOffset > 0 && pDelIdx->nData > 0);
 
   return code;
 

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -88,22 +88,46 @@ int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
   return 0;
 }
 
-int32_t tPutTMap(uint8_t *p, STMap *pMap) {
+int32_t tPutDelIdx(uint8_t *p, SDelIdx *pDelIdx) {
   int32_t n = 0;
 
-  n += tPutU8(p ? p + n : p, pMap->flags);
-  n += tPutBinary(p ? p + n : p, pMap->pOffset, pMap->nOffset);
-  n += tPutBinary(p ? p + n : p, pMap->pData, pMap->nData);
+  n += tPutU32(p ? p + n : p, pDelIdx->delimiter);
+  n += tPutU8(p ? p + n : p, pDelIdx->flags);
+  n += tPutBinary(p ? p + n : p, pDelIdx->pOffset, pDelIdx->nOffset);
+  n += tPutBinary(p ? p + n : p, pDelIdx->pData, pDelIdx->nData);
 
   return n;
 }
 
-int32_t tGetTMap(uint8_t *p, STMap *pMap) {
+int32_t tGetDelIdx(uint8_t *p, SDelIdx *pDelIdx) {
   int32_t n = 0;
 
-  n += tGetU8(p, &pMap->flags);
-  n += tGetBinary(p, &pMap->pOffset, &pMap->nOffset);
-  n += tGetBinary(p, &pMap->pData, &pMap->nData);
+  n += tGetU32(p, &pDelIdx->delimiter);
+  n += tGetU8(p, &pDelIdx->flags);
+  n += tGetBinary(p, &pDelIdx->pOffset, &pDelIdx->nOffset);
+  n += tGetBinary(p, &pDelIdx->pData, &pDelIdx->nData);
+
+  return n;
+}
+
+int32_t tPutDelData(uint8_t *p, SDelData *pDelData) {
+  int32_t n = 0;
+
+  n += tPutU32(p ? p + n : p, pDelData->delimiter);
+  n += tPutU8(p ? p + n : p, pDelData->flags);
+  n += tPutBinary(p ? p + n : p, pDelData->pOffset, pDelData->nOffset);
+  n += tPutBinary(p ? p + n : p, pDelData->pData, pDelData->nData);
+
+  return n;
+}
+
+int32_t tGetDelData(uint8_t *p, SDelData *pDelData) {
+  int32_t n = 0;
+
+  n += tGetU32(p, &pDelData->delimiter);
+  n += tGetU8(p, &pDelData->flags);
+  n += tGetBinary(p, &pDelData->pOffset, &pDelData->nOffset);
+  n += tGetBinary(p, &pDelData->pData, &pDelData->nData);
 
   return n;
 }
\ No newline at end of file