add a idx file for query speed

src/tsdb/inc/tsdbMain.h

@@ -132,21 +132,23 @@ typedef struct {
 // ------------------ tsdbFile.c
 extern const char* tsdbFileSuffix[];
 typedef enum {
-  TSDB_FILE_TYPE_HEAD = 0,
+  TSDB_FILE_TYPE_IDX = 0,
+  TSDB_FILE_TYPE_HEAD,
   TSDB_FILE_TYPE_DATA,
   TSDB_FILE_TYPE_LAST,
   TSDB_FILE_TYPE_MAX,
+  TSDB_FILE_TYPE_NIDX,
   TSDB_FILE_TYPE_NHEAD,
   TSDB_FILE_TYPE_NLAST
 } TSDB_FILE_TYPE;
 
 typedef struct {
-  uint32_t offset;
+  uint32_t magic;
   uint32_t len;
-  uint64_t size;      // total size of the file
-  uint64_t tombSize;  // unused file size
   uint32_t totalBlocks;
   uint32_t totalSubBlocks;
+  uint64_t size;      // total size of the file
+  uint64_t tombSize;  // unused file size
 } STsdbFileInfo;
 
 typedef struct {
@@ -249,16 +251,12 @@ typedef struct {
 typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
 
 typedef struct {
-  int   fid;
-  TSKEY minKey;
-  TSKEY maxKey;
-  // For read/write purpose
-  SFile headF;
-  SFile dataF;
-  SFile lastF;
-  // For write purpose only
-  SFile nHeadF;
-  SFile nLastF;
+  TSKEY      minKey;
+  TSKEY      maxKey;
+  SFileGroup fGroup;
+  SFile      nIdxF;
+  SFile      nHeadF;
+  SFile      nLastF;
 } SHelperFile;
 
 typedef struct {
@@ -444,6 +442,14 @@ void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
 #define helperRepo(h) (h)->pRepo
 #define helperState(h) (h)->state
 #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0)
+#define helperFileId(h) ((h)->files.fGroup.fileId)
+#define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX]))
+#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD]))
+#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA]))
+#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST]))
+#define helperNewIdxF(h) (&((h)->files.nIdxF))
+#define helperNewHeadF(h) (&((h)->files.nHeadF))
+#define helperNewLastF(h) (&((h)->files.nLastF))
 
 int  tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
 int  tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);

src/tsdb/src/tsdbFile.c

@@ -30,7 +30,7 @@
 #include "ttime.h"
 #include "tfile.h"
 
-const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"};
+const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"};
 
 static int   tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type);
 static void  tsdbDestroyFile(SFile *pFile);
@@ -108,7 +108,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
 
     memset((void *)(&fileGroup), 0, sizeof(SFileGroup));
     fileGroup.fileId = fid;
-    for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+    for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) {
       if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) {
         tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type);
         goto _err;
@@ -126,7 +126,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
   return 0;
 
 _err:
-  for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]);
+  for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]);
 
   tfree(tDataDir);
   if (dir != NULL) closedir(dir);
@@ -139,7 +139,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) {
 
   for (int i = 0; i < pFileH->nFGroups; i++) {
     SFileGroup *pFGroup = pFileH->pFGroup + i;
-    for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+    for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) {
       tsdbDestroyFile(&pFGroup->files[type]);
     }
   }
@@ -156,7 +156,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int
   SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
   if (pGroup == NULL) {  // if not exists, create one
     pFGroup->fileId = fid;
-    for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+    for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) {
       if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0)
         goto _err;
     }
@@ -169,7 +169,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int
   return pGroup;
 
 _err:
-  for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]);
+  for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]);
   return NULL;
 }
 
@@ -323,7 +323,7 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) {
 
 int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) {
   int tlen = 0;
-  tlen += taosEncodeFixedU32(buf, pInfo->offset);
+  tlen += taosEncodeFixedU32(buf, pInfo->magic);
   tlen += taosEncodeFixedU32(buf, pInfo->len);
   tlen += taosEncodeFixedU64(buf, pInfo->size);
   tlen += taosEncodeFixedU64(buf, pInfo->tombSize);
@@ -334,7 +334,7 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) {
 }
 
 void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) {
-  buf = taosDecodeFixedU32(buf, &(pInfo->offset));
+  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
   buf = taosDecodeFixedU32(buf, &(pInfo->len));
   buf = taosDecodeFixedU64(buf, &(pInfo->size));
   buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
@@ -358,7 +358,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) {
   pFileH->nFGroups--;
   ASSERT(pFileH->nFGroups >= 0);
 
-  for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
+  for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) {
     if (remove(fileGroup.files[type].fname) < 0) {
       tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname);
     }

src/tsdb/src/tsdbMain.c

@@ -214,7 +214,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
   char *prefix = dirname(sdup);
 
   if (name[0] == 0) {  // get the file from index or after, but not larger than eindex
-    int fid = (*index) / 3;
+    int fid = (*index) / TSDB_FILE_TYPE_MAX;
 
     if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) {
       if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) {
@@ -228,11 +228,11 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
       SFileGroup *pFGroup =
           taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE);
       if (pFGroup->fileId == fid) {
-        fname = strdup(pFGroup->files[(*index) % 3].fname);
+        fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname);
       } else {
-        if (pFGroup->fileId * 3 + 2 < eindex) {
+        if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < eindex) {
           fname = strdup(pFGroup->files[0].fname);
-          *index = pFGroup->fileId * 3;
+          *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX;
         } else {
           tfree(sdup);
           return 0;
@@ -244,14 +244,14 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
     if (*index == TSDB_META_FILE_INDEX) {  // get meta file
       fname = tsdbGetMetaFileName(pRepo->rootDir);
     } else {
-      int         fid = (*index) / 3;
+      int         fid = (*index) / TSDB_FILE_TYPE_MAX;
       SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
       if (pFGroup == NULL) {  // not found
         tfree(sdup);
         return 0;
       }
 
-      SFile *pFile = &pFGroup->files[(*index) % 3];
+      SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX];
       fname = strdup(pFile->fname);
     }
   }

src/tsdb/src/tsdbMemTable.c

@@ -628,9 +628,10 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe
   tsdbCloseHelperFile(pHelper, 0);
 
   pthread_rwlock_wrlock(&(pFileH->fhlock));
-  pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF;
-  pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF;
-  pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF;
+  pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper));
+  pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper));
+  pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper));
+  pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper));
   pthread_rwlock_unlock(&(pFileH->fhlock));
 
   return 0;