remove idx file back

src/tsdb/inc/tsdbMain.h

@@ -132,12 +132,18 @@ typedef struct {
 // ------------------ tsdbFile.c
 extern const char* tsdbFileSuffix[];
 typedef enum {
+#ifdef TSDB_IDX
   TSDB_FILE_TYPE_IDX = 0,
   TSDB_FILE_TYPE_HEAD,
+#else
+  TSDB_FILE_TYPE_HEAD = 0,
+#endif
   TSDB_FILE_TYPE_DATA,
   TSDB_FILE_TYPE_LAST,
   TSDB_FILE_TYPE_MAX,
+#ifdef TSDB_IDX
   TSDB_FILE_TYPE_NIDX,
+#endif
   TSDB_FILE_TYPE_NHEAD,
   TSDB_FILE_TYPE_NLAST
 } TSDB_FILE_TYPE;
@@ -147,6 +153,7 @@ typedef struct {
   uint32_t len;
   uint32_t totalBlocks;
   uint32_t totalSubBlocks;
+  uint32_t offset;
   uint64_t size;      // total size of the file
   uint64_t tombSize;  // unused file size
 } STsdbFileInfo;
@@ -450,11 +457,13 @@ void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
 #define helperState(h) (h)->state
 #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0)
 #define helperFileId(h) ((h)->files.fGroup.fileId)
+#ifdef TSDB_IDX
 #define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX]))
+#define helperNewIdxF(h) (&((h)->files.nIdxF))
+#endif
 #define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD]))
 #define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA]))
 #define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST]))
-#define helperNewIdxF(h) (&((h)->files.nIdxF))
 #define helperNewHeadF(h) (&((h)->files.nHeadF))
 #define helperNewLastF(h) (&((h)->files.nLastF))
 

src/tsdb/src/tsdbFile.c

@@ -30,7 +30,11 @@
 #include "ttime.h"
 #include "tfile.h"
 
+#ifdef TSDB_IDX
 const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"};
+#else
+const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"};
+#endif
 
 static int   tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type);
 static void  tsdbDestroyFile(SFile *pFile);
@@ -108,7 +112,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
 
     memset((void *)(&fileGroup), 0, sizeof(SFileGroup));
     fileGroup.fileId = fid;
-    for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) {
+    for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
       if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) {
         tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type);
         goto _err;
@@ -126,7 +130,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
   return 0;
 
 _err:
-  for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]);
+  for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]);
 
   tfree(tDataDir);
   if (dir != NULL) closedir(dir);
@@ -139,7 +143,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) {
 
   for (int i = 0; i < pFileH->nFGroups; i++) {
     SFileGroup *pFGroup = pFileH->pFGroup + i;
-    for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) {
+    for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
       tsdbDestroyFile(&pFGroup->files[type]);
     }
   }
@@ -156,7 +160,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int
   SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
   if (pGroup == NULL) {  // if not exists, create one
     pFGroup->fileId = fid;
-    for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) {
+    for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
       if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0)
         goto _err;
     }
@@ -169,7 +173,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int
   return pGroup;
 
 _err:
-  for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]);
+  for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]);
   return NULL;
 }
 
@@ -325,10 +329,11 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) {
   int tlen = 0;
   tlen += taosEncodeFixedU32(buf, pInfo->magic);
   tlen += taosEncodeFixedU32(buf, pInfo->len);
-  tlen += taosEncodeFixedU64(buf, pInfo->size);
-  tlen += taosEncodeFixedU64(buf, pInfo->tombSize);
   tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks);
   tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks);
+  tlen += taosEncodeFixedU32(buf, pInfo->offset);
+  tlen += taosEncodeFixedU64(buf, pInfo->size);
+  tlen += taosEncodeFixedU64(buf, pInfo->tombSize);
 
   return tlen;
 }
@@ -336,10 +341,11 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) {
 void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) {
   buf = taosDecodeFixedU32(buf, &(pInfo->magic));
   buf = taosDecodeFixedU32(buf, &(pInfo->len));
-  buf = taosDecodeFixedU64(buf, &(pInfo->size));
-  buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
   buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks));
   buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks));
+  buf = taosDecodeFixedU32(buf, &(pInfo->offset));
+  buf = taosDecodeFixedU64(buf, &(pInfo->size));
+  buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
 
   return buf;
 }
@@ -358,7 +364,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) {
   pFileH->nFGroups--;
   ASSERT(pFileH->nFGroups >= 0);
 
-  for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) {
+  for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
     if (remove(fileGroup.files[type].fname) < 0) {
       tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname);
     }

src/tsdb/src/tsdbMemTable.c

@@ -628,7 +628,9 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
   tsdbCloseHelperFile(pHelper, 0);
 
   pthread_rwlock_wrlock(&(pFileH->fhlock));
+#ifdef TSDB_IDX
   pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper));
+#endif
   pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper));
   pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper));
   pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper));

src/tsdb/src/tsdbRWHelper.c

@@ -109,21 +109,27 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) {
   // Set the files
   pHelper->files.fGroup = *pGroup;
   if (helperType(pHelper) == TSDB_WRITE_HELPER) {
+#ifdef TSDB_IDX
     tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname);
+#endif
     tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname);
     tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname);
   }
 
   // Open the files
+#ifdef TSDB_IDX
   if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) goto _err;
+#endif
   if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) goto _err;
   if (helperType(pHelper) == TSDB_WRITE_HELPER) {
     if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) goto _err;
     if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) goto _err;
 
+#ifdef TSDB_IDX
     // Create and open .i file
     if (tsdbOpenFile(helperNewIdxF(pHelper), O_WRONLY | O_CREAT) < 0) return -1;
     if (tsdbUpdateFileHeader(helperNewIdxF(pHelper), 0) < 0) return -1;
+#endif
 
     // Create and open .h
     if (tsdbOpenFile(helperNewHeadF(pHelper), O_WRONLY | O_CREAT) < 0) return -1;
@@ -150,11 +156,13 @@ _err:
 int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
   SFile *pFile = NULL;
 
+#ifdef TSDB_IDX
   pFile = helperIdxF(pHelper);
   if (pFile->fd > 0) {
     close(pFile->fd);
     pFile->fd = -1;
   }
+#endif
 
   pFile = helperHeadF(pHelper);
   if (pFile->fd > 0) {
@@ -182,6 +190,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
   }
 
   if (helperType(pHelper) == TSDB_WRITE_HELPER) {
+#ifdef TSDB_IDX
     pFile = helperNewIdxF(pHelper);
     if (pFile->fd > 0) {
       if (!hasError) tsdbUpdateFileHeader(pFile, 0);
@@ -200,6 +209,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) {
         helperIdxF(pHelper)->info = pFile->info;
       }
     }
+#endif
 
     pFile = helperNewHeadF(pHelper);
     if (pFile->fd > 0) {
@@ -365,12 +375,13 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
 int tsdbWriteCompInfo(SRWHelper *pHelper) {
   SCompIdx *pIdx = &(pHelper->curCompIdx);
   off_t     offset = 0;
+  SFile *   pFile = helperNewHeadF(pHelper);
 
   if (pIdx->len > 0) {
     if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) {
-      offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END);
+      offset = lseek(pFile->fd, 0, SEEK_END);
       if (offset < 0) {
-        tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname,
+        tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname,
                   strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
@@ -381,9 +392,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
       pIdx->tid = pHelper->tableInfo.tid;
       ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
-      if (tsendfile(helperNewHeadF(pHelper)->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) {
+      if (tsendfile(pFile->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) {
         tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
-                  helperHeadF(pHelper)->fname, helperNewHeadF(pHelper)->fname, strerror(errno));
+                  helperHeadF(pHelper)->fname, pFile->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
       }
@@ -394,9 +405,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
       ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) &&
              (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0);
       taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len);
-      offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END);
+      offset = lseek(pFile->fd, 0, SEEK_END);
       if (offset < 0) {
-        tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname,
+        tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname,
                   strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
@@ -406,15 +417,19 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
       pIdx->tid = pHelper->tableInfo.tid;
       ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
-      if (twrite(helperNewHeadF(pHelper)->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
+      if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
         tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
-                  helperNewHeadF(pHelper)->fname, strerror(errno));
+                  pFile->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
       }
     }
 
-    if (tsizeof(pHelper->pWIdx) < helperNewIdxF(pHelper)->info.len + sizeof(SCompIdx) + 12) {
+#ifdef TSDB_IDX
+    pFile = helperNewIdxF(pHelper);
+#endif
+
+    if (tsizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) {
       pHelper->pWIdx = trealloc(pHelper->pWIdx, tsizeof(pHelper->pWIdx) == 0 ? 1024 : tsizeof(pHelper->pWIdx) * 2);
       if (pHelper->pWIdx == NULL) {
         terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -422,8 +437,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
       }
     }
 
-    void *pBuf = POINTER_SHIFT(pHelper->pWIdx, helperNewIdxF(pHelper)->info.len);
-    helperNewIdxF(pHelper)->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx));
+    void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len);
+    pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx));
   }
 
   return 0;
@@ -431,9 +446,13 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
 
 int tsdbWriteCompIdx(SRWHelper *pHelper) {
   ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER);
-  // STsdbCfg *pCfg = &pHelper->pRepo->config;
+  off_t offset = 0;
 
+#ifdef TSDB_IDX
   SFile *pFile = helperNewIdxF(pHelper);
+#else
+  SFile *pFile = helperNewHeadF(pHelper);
+#endif
 
   pFile->info.len += sizeof(TSCKSUM);
   if (tsizeof(pHelper->pWIdx) < pFile->info.len) {
@@ -445,6 +464,15 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) {
   }
   taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len);
 
+  offset = lseek(pFile->fd, 0, SEEK_END);
+  if (offset < 0) {
+    tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  pFile->info.offset = offset;
+
   if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
               pFile->fname, strerror(errno));
@@ -457,8 +485,12 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) {
 
 int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
   ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN);
+#ifdef TSDB_IDX
   SFile *pFile = helperIdxF(pHelper);
-  int    fd = pFile->fd;
+#else
+  SFile *pFile = helperHeadF(pHelper);
+#endif
+  int fd = pFile->fd;
 
   if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) {
     // If not load from file, just load it in object
@@ -468,7 +500,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
         return -1;
       }
 
-      if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {
+      if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) {
         tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
@@ -516,11 +548,6 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
 
         ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM));
       }
-
-      // if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) {
-      //   terrno = TAOS_SYSTEM_ERROR(errno);
-      //   return -1;
-      // }
     }
   }
   helperSetState(pHelper, TSDB_HELPER_IDX_LOAD);
@@ -1031,13 +1058,15 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
   pHelper->idxH.numOfIdx = 0;
   pHelper->idxH.curIdx = 0;
   memset((void *)&pHelper->files, 0, sizeof(pHelper->files));
-  helperIdxF(pHelper)->fd = -1;
   helperHeadF(pHelper)->fd = -1;
   helperDataF(pHelper)->fd = -1;
   helperLastF(pHelper)->fd = -1;
-  helperNewIdxF(pHelper)->fd = -1;
   helperNewHeadF(pHelper)->fd = -1;
   helperNewLastF(pHelper)->fd = -1;
+#ifdef TSDB_IDX
+  helperIdxF(pHelper)->fd = -1;
+  helperNewIdxF(pHelper)->fd = -1;
+#endif
 }
 
 static int tsdbInitHelperFile(SRWHelper *pHelper) {