From 338a7638537358d6fddafd766b5f02d87409a39c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 02:41:37 +0000 Subject: [PATCH] more vnode snapshot --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 4 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 63 ++++++++++++---------- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 7 +-- 5 files changed, 44 insertions(+), 34 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4df9f96514..819076536d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -202,7 +202,7 @@ int32_t tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile); int32_t tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet); void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid); SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState); -SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid); +SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag); // tsdbReaderWriter.c ============================================================================================== // SDataFWriter int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8a189d98b3..577ef03a8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -263,7 +263,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->aBlockIdx); tMapDataReset(&pCommitter->oBlockMap); tBlockDataReset(&pCommitter->oBlockData); - pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid); + pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->pReader, pTsdb, pRSet); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 53b6735c30..70fbd90646 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -698,6 +698,6 @@ void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid) { SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState) { return pState->pDelFile; } -SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid) { - return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); +SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag) { + return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, flag); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index b02e1dabd3..7dc7e29ad7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -40,10 +40,12 @@ struct STsdbSnapReader { static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; while (true) { if (pReader->pDataFReader == NULL) { SDFileSet* pSet = NULL; + // taosArraySearch(pTsdb->fs->cState->aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSe) // search the next data file set to read (todo) if (0 /* TODO */) { @@ -106,10 +108,13 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } _exit: + // if (*ppData) { + // tsdbInfo("vgId:%d "); + // } return code; _err: - tsdbError("vgId:%d snap read data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb read data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -209,14 +214,13 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe pReader->sver = sver; pReader->ever = ever; + pReader->fid = INT32_MIN; pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pReader->aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pReader->mBlock = tMapDataInit(); - code = tBlockDataInit(&pReader->blkData); if (code) goto _err; @@ -225,7 +229,6 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pReader->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pReader->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -246,37 +249,42 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { int32_t code = 0; STsdbSnapReader* pReader = *ppReader; - taosArrayDestroy(pReader->aDelData); - taosArrayDestroy(pReader->aDelIdx); - if (pReader->pDelFReader) { - tsdbDelFReaderClose(&pReader->pDelFReader); - } - tBlockDataClear(&pReader->blkData); - tMapDataClear(&pReader->mBlock); - taosArrayDestroy(pReader->aBlockIdx); if (pReader->pDataFReader) { tsdbDataFReaderClose(&pReader->pDataFReader); } + taosArrayDestroy(pReader->aBlockIdx); + tMapDataClear(&pReader->mBlock); + tBlockDataClear(&pReader->blkData); + + if (pReader->pDelFReader) { + tsdbDelFReaderClose(&pReader->pDelFReader); + } + taosArrayDestroy(pReader->aDelIdx); + taosArrayDestroy(pReader->aDelData); + + tsdbInfo("vgId:%d vnode snapshot tsdb reader closed", TD_VID(pReader->pTsdb->pVnode)); + taosMemoryFree(pReader); *ppReader = NULL; - return code; } int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + *ppData = NULL; + // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { - pReader->dataDone = 1; + goto _err; + } else { + if (*ppData) { + goto _exit; } else { - goto _err; + pReader->dataDone = 1; } - } else { - goto _exit; } } @@ -284,23 +292,24 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { if (!pReader->delDone) { code = tsdbSnapReadDel(pReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { - pReader->delDone = 1; + goto _err; + } else { + if (*ppData) { + goto _exit; } else { - goto _err; + pReader->delDone = 1; } - } else { - goto _exit; } } - code = TSDB_CODE_VND_READ_END; - _exit: + if (*ppData) { + } else { + } return code; _err: - tsdbError("vgId:%d snapshot read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -520,7 +529,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (code) goto _err; pWriter->fid = fid; - SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid); + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ); // reader if (pSet) { // open diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 0e14df0380..ed09f0edc3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -125,7 +125,8 @@ _exit: pReader->index++; *nData = sizeof(SSnapDataHdr) + pHdr->size; pHdr->index = pReader->index; - vInfo("vgId:%d vnode snapshot read data, nData:%d index:%" PRId64, TD_VID(pReader->pVnode), *nData, pReader->index); + vInfo("vgId:%d vnode snapshot read data,index:%" PRId64 " type:%d nData:%d ", TD_VID(pReader->pVnode), + pReader->index, pHdr->type, *nData); } else { vInfo("vgId:%d vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); } @@ -236,8 +237,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { // } _exit: - vInfo("vgId:%d vnode snapshot write data, type:%d index:%" PRId64 " nData:%d", TD_VID(pVnode), pHdr->type, - pHdr->index, nData); + vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->type, nData, + pHdr->index); return code; _err: -- GitLab