From 7fab74daaf24ec045a8d8b1cc7134b561dd28882 Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Tue, 11 Jul 2023 15:33:19 +0800
Subject: [PATCH] enhance: add tsdb stt block data cache

---
 source/dnode/vnode/src/inc/tsdb.h           |  3 ++
 source/dnode/vnode/src/tsdb/tsdbCache.c     |  3 ++
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 48 +++++++++++++++++++--
 3 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index d7694ebfd5..f51d0d2fd6 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -376,6 +376,8 @@ struct STsdb {
   SLRUCache       *biCache;
   TdThreadMutex    biMutex;
   SRocksCache      rCache;
+  SLRUCache*       sttBlockCache;
+  SLRUCache*       sttBlkCache;
 };
 
 struct TSDBKEY {
@@ -705,6 +707,7 @@ typedef struct SSttBlockLoadInfo {
   SArray    *aSttBlk;
   int32_t    blockIndex[2];  // to denote the loaded block in the corresponding position.
   int32_t    currentLoadBlockIndex;
+  LRUHandle *blockDataHandle;
   int32_t    loadBlocks;
   double     elapsedTime;
   STSchema  *pSchema;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index f4e888919e..1b90bee6be 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -1210,12 +1210,15 @@ int32_t tsdbOpenCache(STsdb *pTsdb) {
   pTsdb->flushState.pTsdb = pTsdb;
   pTsdb->flushState.flush_count = 0;
 
+  pTsdb->sttBlockCache = taosLRUCacheInit(128 * 1024 * 1024, -1, 0.0);
+  pTsdb->sttBlkCache = taosLRUCacheInit(4 * 1024 * 1024, -1, 0.0);
 _err:
   pTsdb->lruCache = pCache;
   return code;
 }
 
 void tsdbCloseCache(STsdb *pTsdb) {
+  taosLRUCacheCleanup(pTsdb->sttBlockCache);
   SLRUCache *pCache = pTsdb->lruCache;
   if (pCache) {
     taosLRUCacheEraseUnrefEntries(pCache);
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index 79f4a17f65..b945e0f80b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -90,11 +90,52 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
   taosMemoryFree(pLoadInfo);
   return NULL;
 }
-
+static void deleteSttBlockDataCache(const void *key, size_t keyLen, void *value, void *ud) {
+  SBlockData* pBlockData = value;
+  tBlockDataDestroy(pBlockData);
+}
 static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
-  int32_t code = 0;
 
+  // (pReader->pSet->fid, iStt, pReader->pSet->pHeadF->commitID) -> aSttBlk global cache
+
+  // (pIter->pReader->pSet->fid, pIter->iStt, pIter->pReader->pSet->pHeadF->commitID, pIter->pSttBlk->bInfo.offset) -> SBlockData
   SSttBlockLoadInfo *pInfo = pIter->pBlockLoadInfo;
+  if (pIter->pSttBlk == NULL || pInfo->pSchema == NULL) {
+    pInfo->blockDataHandle = NULL;
+    return NULL;
+  }
+  struct {
+    int32_t fid;
+    int32_t iStt;
+    int64_t cid;
+    int64_t offset;
+  } key = {.fid = pIter->pReader->pSet->fid, .iStt = pIter->iStt, .cid = pIter->pReader->pSet->pHeadF->commitID, .offset = pIter->pSttBlk->bInfo.offset};
+  int32_t code = 0;
+
+  LRUHandle* h = taosLRUCacheLookup(pIter->pReader->pTsdb->sttBlockCache, &key, sizeof(key));
+  if (!h) {
+    SSttBlockLoadInfo *pInfo = pIter->pBlockLoadInfo;
+
+    SBlockData* pBlockData = taosMemoryMalloc(sizeof(SBlockData));
+
+    TABLEID id = {0};
+    if (pIter->pSttBlk->suid != 0) {
+      id.suid = pIter->pSttBlk->suid;
+    } else {
+      id.uid = pIter->uid;
+    }
+
+    tBlockDataInit(pBlockData, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols);
+    tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData);
+    int charge = 2 * 1024 * 1024; //TODO
+
+    taosLRUCacheInsert(pIter->pReader->pTsdb->sttBlockCache, &key, sizeof(key), pBlockData, charge, deleteSttBlockDataCache, &h, TAOS_LRU_PRIORITY_LOW, NULL);
+  }
+
+  SBlockData* pBlockData = taosLRUCacheValue(pIter->pReader->pTsdb->sttBlockCache, h);
+  pInfo->blockDataHandle = h;
+  return pBlockData;
+
   if (pInfo->blockIndex[0] == pIter->iSttBlk) {
     if (pInfo->currentLoadBlockIndex != 0) {
       tsdbDebug("current load index is set to 0, block index:%d, file index:%d, due to uid:%" PRIu64 ", load data, %s",
@@ -271,7 +312,7 @@ int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t
   if (!pBlockLoadInfo->sttBlockLoaded) {
     int64_t st = taosGetTimestampUs();
     pBlockLoadInfo->sttBlockLoaded = true;
-
+    // (pReader->pSet->fid, iStt, pReader->pSet->pHeadF->commitID, iStt) -> aSttBlk global cache
     code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
     if (code) {
       return code;
@@ -512,6 +553,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
     }
 
     if (skipBlock || pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {
+      taosLRUCacheRelease(pIter->pReader->pTsdb->sttBlockCache, pIter->pBlockLoadInfo->blockDataHandle, false);
       tLDataIterNextBlock(pIter, idStr);
       if (pIter->pSttBlk == NULL) {  // no more data
         goto _exit;
-- 
GitLab