diff --git a/source/dnode/vnode/src/tsdb/dev/inc/tsdbCommit.h b/source/dnode/vnode/src/tsdb/dev/inc/tsdbCommit.h index be6cbbfe81f8aa23f9efcb0e078a915efe7bbc25..ad60b52824c463445804b68c66af6829f382dc93 100644 --- a/source/dnode/vnode/src/tsdb/dev/inc/tsdbCommit.h +++ b/source/dnode/vnode/src/tsdb/dev/inc/tsdbCommit.h @@ -13,7 +13,9 @@ * along with this program. If not, see . */ +#include "tsdbDataFileRW.h" #include "tsdbFS.h" +#include "tsdbIter.h" #include "tsdbSttFileRW.h" #ifndef _TSDB_COMMIT_H_ diff --git a/source/dnode/vnode/src/tsdb/dev/inc/tsdbFile.h b/source/dnode/vnode/src/tsdb/dev/inc/tsdbFile.h index 885560d8973ca0901a7ddd65d2828ca9ed242acd..3ac04263fd74c6446d5afb7d6cfae4fe91a772ed 100644 --- a/source/dnode/vnode/src/tsdb/dev/inc/tsdbFile.h +++ b/source/dnode/vnode/src/tsdb/dev/inc/tsdbFile.h @@ -71,7 +71,7 @@ struct STFile { struct STFileObj { TdThreadMutex mutex; - STFile f; + STFile f[1]; int32_t state; int32_t ref; char fname[TSDB_FILENAME_LEN]; diff --git a/source/dnode/vnode/src/tsdb/dev/inc/tsdbSttFileRW.h b/source/dnode/vnode/src/tsdb/dev/inc/tsdbSttFileRW.h index ae68cc3efbdb97fca0250b272cb0db28d2a23e4d..604a137aaefa6ec2c89bedbc94fd49def0e6132d 100644 --- a/source/dnode/vnode/src/tsdb/dev/inc/tsdbSttFileRW.h +++ b/source/dnode/vnode/src/tsdb/dev/inc/tsdbSttFileRW.h @@ -60,11 +60,11 @@ struct SSttFileReaderConfig { typedef struct SSttFileWriter SSttFileWriter; typedef struct SSttFileWriterConfig SSttFileWriterConfig; -int32_t tsdbSttFWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer); -int32_t tsdbSttFWriterClose(SSttFileWriter **writer, int8_t abort, STFileOp *op); -int32_t tsdbSttFWriteTSData(SSttFileWriter *writer, SRowInfo *row); -int32_t tsdbSttFWriteTSDataBlock(SSttFileWriter *writer, SBlockData *pBlockData); -int32_t tsdbSttFWriteDLData(SSttFileWriter *writer, TABLEID *tbid, SDelData *pDelData); +int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer); +int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, STFileOp *op); +int32_t tsdbSttFileWriteTSData(SSttFileWriter *writer, SRowInfo *row); +int32_t tsdbSttFileWriteTSDataBlock(SSttFileWriter *writer, SBlockData *pBlockData); +int32_t tsdbSttFileWriteDLData(SSttFileWriter *writer, TABLEID *tbid, SDelData *pDelData); struct SSttFileWriterConfig { STsdb *tsdb; diff --git a/source/dnode/vnode/src/tsdb/dev/tsdbCommit.c b/source/dnode/vnode/src/tsdb/dev/tsdbCommit.c index a211b2504ef0ca26ae75972a166c950e2ba06c03..536882d6978b32c01a79785cd6e9fcde9e877a64 100644 --- a/source/dnode/vnode/src/tsdb/dev/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/dev/tsdbCommit.c @@ -17,63 +17,67 @@ // extern dependencies typedef struct { - STsdb *pTsdb; - - // config + STsdb *tsdb; int32_t minutes; int8_t precision; int32_t minRow; int32_t maxRow; int8_t cmprAlg; int8_t sttTrigger; - - SArray *aTbDataP; // SArray - TFileOpArray opArray; - int64_t eid; // edit id - - // context - TSKEY nextKey; - int32_t fid; - int32_t expLevel; - TSKEY minKey; - TSKEY maxKey; - STFileSet *fset; + int64_t compactVersion; + + struct { + int64_t now; + TSKEY nextKey; + int32_t fid; + int32_t expLevel; + TSKEY minKey; + TSKEY maxKey; + STFileSet *fset; + TABLEID tbid[1]; + } ctx[1]; + + int64_t eid; // edit id + TFileOpArray fopArray[1]; + TTsdbIterArray iterArray[1]; + SIterMerger *iterMerger; // writer - SSttFileWriter *pWriter; -} SCommitter; + SDataFileWriter *dataWriter; + SSttFileWriter *sttWriter; +} SCommitter2; -static int32_t open_writer_with_new_stt(SCommitter *pCommitter) { +static int32_t tsdbCommitOpenNewSttWriter(SCommitter2 *pCommitter) { int32_t code = 0; int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; + STsdb *pTsdb = pCommitter->tsdb; SVnode *pVnode = pTsdb->pVnode; int32_t vid = TD_VID(pVnode); - SSttFileWriterConfig config; - SDiskID did; + SSttFileWriterConfig config[1]; + SDiskID did[1]; - if (tfsAllocDisk(pVnode->pTfs, pCommitter->expLevel, &did) < 0) { + if (tfsAllocDisk(pVnode->pTfs, pCommitter->ctx->expLevel, did) < 0) { code = TSDB_CODE_FS_NO_VALID_DISK; TSDB_CHECK_CODE(code, lino, _exit); } - config.tsdb = pTsdb; - config.maxRow = pCommitter->maxRow; - config.szPage = pVnode->config.tsdbPageSize; - config.cmprAlg = pCommitter->cmprAlg; - config.skmTb = NULL; - config.skmRow = NULL; - config.aBuf = NULL; - config.file.type = TSDB_FTYPE_STT; - config.file.did = did; - config.file.fid = pCommitter->fid; - config.file.cid = pCommitter->eid; - config.file.size = 0; - config.file.stt->level = 0; - config.file.stt->nseg = 0; - - code = tsdbSttFWriterOpen(&config, &pCommitter->pWriter); + config->tsdb = pTsdb; + config->maxRow = pCommitter->maxRow; + config->szPage = pVnode->config.tsdbPageSize; + config->cmprAlg = pCommitter->cmprAlg; + config->skmTb = NULL; + config->skmRow = NULL; + config->aBuf = NULL; + config->file.type = TSDB_FTYPE_STT; + config->file.did = did[0]; + config->file.fid = pCommitter->ctx->fid; + config->file.cid = pCommitter->eid; + config->file.size = 0; + config->file.stt->level = 0; + config->file.stt->nseg = 0; + + code = tsdbSttFileWriterOpen(config, &pCommitter->sttWriter); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -84,10 +88,10 @@ _exit: } return code; } -static int32_t open_writer_with_exist_stt(SCommitter *pCommitter, const STFile *pFile) { +static int32_t tsdbCommitOpenExistSttWriter(SCommitter2 *pCommitter, const STFile *pFile) { int32_t code = 0; int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; + STsdb *pTsdb = pCommitter->tsdb; SVnode *pVnode = pTsdb->pVnode; int32_t vid = TD_VID(pVnode); @@ -103,7 +107,7 @@ static int32_t open_writer_with_exist_stt(SCommitter *pCommitter, const STFile * .file = *pFile // }; - code = tsdbSttFWriterOpen(&config, &pCommitter->pWriter); + code = tsdbSttFileWriterOpen(&config, &pCommitter->sttWriter); TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -114,105 +118,115 @@ _exit: } return code; } -static int32_t open_committer_writer(SCommitter *pCommitter) { - if (!pCommitter->fset) { - return open_writer_with_new_stt(pCommitter); +static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { + if (!committer->ctx->fset) { + return tsdbCommitOpenNewSttWriter(committer); } - const SSttLvl *lvl0 = tsdbTFileSetGetLvl(pCommitter->fset, 0); + const SSttLvl *lvl0 = tsdbTFileSetGetLvl(committer->ctx->fset, 0); if (lvl0 == NULL) { - return open_writer_with_new_stt(pCommitter); + return tsdbCommitOpenNewSttWriter(committer); } ASSERT(TARRAY2_SIZE(&lvl0->farr) > 0); STFileObj *fobj = TARRAY2_LAST(&lvl0->farr); - if (fobj->f.stt->nseg >= pCommitter->sttTrigger) { - return open_writer_with_new_stt(pCommitter); + if (fobj->f->stt->nseg >= committer->sttTrigger) { + return tsdbCommitOpenNewSttWriter(committer); } else { - return open_writer_with_exist_stt(pCommitter, &fobj->f); + return tsdbCommitOpenExistSttWriter(committer, fobj->f); } } -static int32_t tsdbCommitWriteTSData(SCommitter *pCommitter, SRowInfo *pRowInfo) { - int32_t code = 0; - int32_t lino = 0; - int32_t vid = TD_VID(pCommitter->pTsdb->pVnode); - - if (pCommitter->pWriter == NULL) { - code = open_committer_writer(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbSttFWriteTSData(pCommitter->pWriter, pRowInfo); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d failed at line %d since %s", vid, lino, tstrerror(code)); - } - return 0; +static int32_t tsdbCommitTSRow(SCommitter2 *committer, SRowInfo *row) { + return tsdbSttFileWriteTSData(committer->sttWriter, row); } -static int32_t tsdbCommitWriteDelData(SCommitter *pCommitter, int64_t suid, int64_t uid, int64_t version, int64_t sKey, +static int32_t tsdbCommitWriteDelData(SCommitter2 *pCommitter, int64_t suid, int64_t uid, int64_t version, int64_t sKey, int64_t eKey) { int32_t code = 0; // TODO return code; } -static int32_t commit_timeseries_data(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - int64_t nRow = 0; - STsdb *pTsdb = pCommitter->pTsdb; - int32_t vid = TD_VID(pTsdb->pVnode); - SMemTable *pMem = pTsdb->imem; - - if (pMem->nRow == 0) goto _exit; - - TSDBKEY from = {.ts = pCommitter->minKey, .version = VERSION_MIN}; - for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) { - STbDataIter iter; - STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); - SRowInfo rowInfo = {.suid = pTbData->suid, .uid = pTbData->uid}; +static int32_t tsdbCommitTSData(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + int64_t nRow = 0; + int32_t vid = TD_VID(committer->tsdb->pVnode); + SRowInfo *row; + + if (committer->tsdb->imem->nRow == 0) goto _exit; + + // open iter and iter merger + STsdbIter *iter; + STsdbIterConfig config[1] = {{ + .type = TSDB_ITER_TYPE_MEMT, + .memt = committer->tsdb->imem, + .from = {{ + .ts = committer->ctx->minKey, + .version = VERSION_MIN, + }}, + }}; + + code = tsdbIterOpen(config, &iter); + TSDB_CHECK_CODE(code, lino, _exit); - tsdbTbDataIterOpen(pTbData, &from, 0, &iter); + code = TARRAY2_APPEND(committer->iterArray, iter); + TSDB_CHECK_CODE(code, lino, _exit); - for (TSDBROW *pRow; (pRow = tsdbTbDataIterGet(&iter)) != NULL; tsdbTbDataIterNext(&iter)) { - TSDBKEY rowKey = TSDBROW_KEY(pRow); + code = tsdbIterMergerInit(committer->iterArray, &committer->iterMerger); + TSDB_CHECK_CODE(code, lino, _exit); - if (rowKey.ts > pCommitter->maxKey) { - pCommitter->nextKey = TMIN(pCommitter->nextKey, rowKey.ts); - break; + // loop iter + while ((row = tsdbIterMergerGet(committer->iterMerger)) != NULL) { + if (row->uid != committer->ctx->tbid->uid) { + committer->ctx->tbid->suid = row->suid; + committer->ctx->tbid->uid = row->uid; + + // Ignore deleted table + SMetaInfo info[1]; + if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, info, NULL) != 0) { + code = tsdbIterMergerSkipTableData(committer->iterMerger, committer->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + continue; } + } - rowInfo.row = *pRow; - code = tsdbCommitWriteTSData(pCommitter, &rowInfo); + TSKEY ts = TSDBROW_TS(&row->row); + if (ts > committer->ctx->maxKey) { + committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); + + code = tsdbIterMergerSkipTableData(committer->iterMerger, committer->ctx->tbid); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbCommitTSRow(committer, row); TSDB_CHECK_CODE(code, lino, _exit); - nRow++; + code = tsdbIterMergerNext(committer->iterMerger); + TSDB_CHECK_CODE(code, lino, _exit); } } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(vid, lino, code); } else { - tsdbDebug("vgId:%d %s done, fid:%d nRow:%" PRId64, vid, __func__, pCommitter->fid, nRow); + tsdbDebug("vgId:%d %s done, fid:%d nRow:%" PRId64, vid, __func__, committer->ctx->fid, nRow); } return code; } -static int32_t commit_delete_data(SCommitter *pCommitter) { +static int32_t tsdbCommitDelData(SCommitter2 *pCommitter) { int32_t code = 0; int32_t lino; return 0; +#if 0 ASSERTS(0, "TODO: Not implemented yet"); int64_t nDel = 0; - SMemTable *pMem = pCommitter->pTsdb->imem; + SMemTable *pMem = pCommitter->tsdb->imem; if (pMem->nDel == 0) { // no del data goto _exit; @@ -222,9 +236,9 @@ static int32_t commit_delete_data(SCommitter *pCommitter) { STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) { - if (pDelData->eKey < pCommitter->minKey) continue; - if (pDelData->sKey > pCommitter->maxKey) { - pCommitter->nextKey = TMIN(pCommitter->nextKey, pDelData->sKey); + if (pDelData->eKey < pCommitter->ctx->minKey) continue; + if (pDelData->sKey > pCommitter->ctx->maxKey) { + pCommitter->ctx->nextKey = TMIN(pCommitter->ctx->nextKey, pDelData->sKey); continue; } @@ -236,44 +250,56 @@ static int32_t commit_delete_data(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), lino, tstrerror(code)); + tsdbError("vgId:%d failed at line %d since %s", TD_VID(pCommitter->tsdb->pVnode), lino, tstrerror(code)); } else { - tsdbDebug("vgId:%d %s done, fid:%d nDel:%" PRId64, TD_VID(pCommitter->pTsdb->pVnode), __func__, pCommitter->fid, + tsdbDebug("vgId:%d %s done, fid:%d nDel:%" PRId64, TD_VID(pCommitter->tsdb->pVnode), __func__, pCommitter->ctx->fid, pMem->nDel); } return code; +#endif } -static int32_t commit_fset_start(SCommitter *pCommitter) { - STsdb *pTsdb = pCommitter->pTsdb; - int32_t vid = TD_VID(pTsdb->pVnode); +static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { + int32_t code = 0; + int32_t lino = 0; + STsdb *tsdb = committer->tsdb; + int32_t vid = TD_VID(tsdb->pVnode); + + committer->ctx->fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision); + tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey, + &committer->ctx->maxKey); + committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now); + committer->ctx->nextKey = TSKEY_MAX; - pCommitter->fid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); - tsdbFidKeyRange(pCommitter->fid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, - &pCommitter->maxKey); - pCommitter->expLevel = tsdbFidLevel(pCommitter->fid, &pTsdb->keepCfg, taosGetTimestampSec()); - pCommitter->nextKey = TSKEY_MAX; + // TODO: use a thread safe function to get fset + tsdbFSGetFSet(tsdb->pFS, committer->ctx->fid, &committer->ctx->fset); - tsdbFSGetFSet(pTsdb->pFS, pCommitter->fid, &pCommitter->fset); + code = tsdbCommitOpenWriter(committer); + TSDB_CHECK_CODE(code, lino, _exit); - tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", vid, __func__, pCommitter->fid, - pCommitter->minKey, pCommitter->maxKey, pCommitter->expLevel); +_exit: + if (code) { + TSDB_ERROR_LOG(vid, lino, code); + } else { + tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", vid, __func__, + committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel); + } return 0; } -static int32_t commit_fset_end(SCommitter *pCommitter) { +static int32_t tsdbCommitFileSetEnd(SCommitter2 *pCommitter) { int32_t code = 0; int32_t lino = 0; - int32_t vid = TD_VID(pCommitter->pTsdb->pVnode); + int32_t vid = TD_VID(pCommitter->tsdb->pVnode); - if (pCommitter->pWriter == NULL) return 0; + if (pCommitter->sttWriter == NULL) return 0; STFileOp op; - code = tsdbSttFWriterClose(&pCommitter->pWriter, 0, &op); + code = tsdbSttFileWriterClose(&pCommitter->sttWriter, 0, &op); TSDB_CHECK_CODE(code, lino, _exit); if (op.optype != TSDB_FOP_NONE) { - code = TARRAY2_APPEND(&pCommitter->opArray, op); + code = TARRAY2_APPEND(pCommitter->fopArray, op); TSDB_CHECK_CODE(code, lino, _exit); } @@ -281,90 +307,86 @@ _exit: if (code) { tsdbError("vgId:%d failed at line %d since %s", vid, lino, tstrerror(code)); } else { - tsdbDebug("vgId:%d %s done, fid:%d", vid, __func__, pCommitter->fid); + tsdbDebug("vgId:%d %s done, fid:%d", vid, __func__, pCommitter->ctx->fid); } return code; } -static int32_t commit_fset(SCommitter *pCommitter) { +static int32_t tsdbCommitFileSet(SCommitter2 *committer) { int32_t code = 0; int32_t lino = 0; - int32_t vid = TD_VID(pCommitter->pTsdb->pVnode); + int32_t vid = TD_VID(committer->tsdb->pVnode); // fset commit start - code = commit_fset_start(pCommitter); + code = tsdbCommitFileSetBegin(committer); TSDB_CHECK_CODE(code, lino, _exit); // commit fset - code = commit_timeseries_data(pCommitter); + code = tsdbCommitTSData(committer); TSDB_CHECK_CODE(code, lino, _exit); - code = commit_delete_data(pCommitter); + code = tsdbCommitDelData(committer); TSDB_CHECK_CODE(code, lino, _exit); // fset commit end - code = commit_fset_end(pCommitter); + code = tsdbCommitFileSetEnd(committer); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(vid, lino, code); } else { - tsdbDebug("vgId:%d %s done", vid, __func__); + tsdbDebug("vgId:%d %s done, fid:%d", vid, __func__, committer->ctx->fid); } return code; } -static int32_t open_committer(STsdb *pTsdb, SCommitInfo *pInfo, SCommitter *pCommitter) { +static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *committer) { int32_t code = 0; - int32_t lino; + int32_t lino = 0; + int32_t vid = TD_VID(tsdb->pVnode); - // set config - memset(pCommitter, 0, sizeof(SCommitter)); - pCommitter->pTsdb = pTsdb; - pCommitter->minutes = pTsdb->keepCfg.days; - pCommitter->precision = pTsdb->keepCfg.precision; - pCommitter->minRow = pInfo->info.config.tsdbCfg.minRows; - pCommitter->maxRow = pInfo->info.config.tsdbCfg.maxRows; - pCommitter->cmprAlg = pInfo->info.config.tsdbCfg.compression; - pCommitter->sttTrigger = pInfo->info.config.sttTrigger; - - pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); - if (pCommitter->aTbDataP == NULL) { - taosArrayDestroy(pCommitter->aTbDataP); - TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); - } - TARRAY2_INIT(&pCommitter->opArray); - tsdbFSAllocEid(pTsdb->pFS, &pCommitter->eid); + memset(committer, 0, sizeof(committer[0])); - // start loop - pCommitter->nextKey = pTsdb->imem->minKey; // TODO + committer->tsdb = tsdb; + committer->minutes = tsdb->keepCfg.days; + committer->precision = tsdb->keepCfg.precision; + committer->minRow = info->info.config.tsdbCfg.minRows; + committer->maxRow = info->info.config.tsdbCfg.maxRows; + committer->cmprAlg = info->info.config.tsdbCfg.compression; + committer->sttTrigger = info->info.config.sttTrigger; + committer->compactVersion = INT64_MAX; // TODO: use a function + + TARRAY2_INIT(committer->fopArray); + tsdbFSAllocEid(tsdb->pFS, &committer->eid); + + committer->ctx->now = taosGetTimestampSec(); + committer->ctx->nextKey = tsdb->imem->minKey; // TODO _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(vid, lino, code); } else { - tsdbDebug("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } return code; } -static int32_t close_committer(SCommitter *pCommiter, int32_t eno) { +static int32_t tsdbCloseCommitter(SCommitter2 *pCommiter, int32_t eno) { int32_t code = 0; int32_t lino = 0; - int32_t vid = TD_VID(pCommiter->pTsdb->pVnode); + int32_t vid = TD_VID(pCommiter->tsdb->pVnode); if (eno == 0) { - code = tsdbFSEditBegin(pCommiter->pTsdb->pFS, &pCommiter->opArray, TSDB_FEDIT_COMMIT); + code = tsdbFSEditBegin(pCommiter->tsdb->pFS, pCommiter->fopArray, TSDB_FEDIT_COMMIT); TSDB_CHECK_CODE(code, lino, _exit); } else { // TODO ASSERT(0); } - ASSERT(pCommiter->pWriter == NULL); - taosArrayDestroy(pCommiter->aTbDataP); - TARRAY2_CLEAR_FREE(&pCommiter->opArray, NULL); + ASSERT(pCommiter->sttWriter == NULL); + TARRAY2_FREE(pCommiter->fopArray); _exit: if (code) { @@ -376,51 +398,53 @@ _exit: return code; } -int32_t tsdbPreCommit(STsdb *pTsdb) { - taosThreadRwlockWrlock(&pTsdb->rwLock); - ASSERT(pTsdb->imem == NULL); - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - taosThreadRwlockUnlock(&pTsdb->rwLock); +int32_t tsdbPreCommit(STsdb *tsdb) { + taosThreadRwlockWrlock(&tsdb->rwLock); + ASSERT(tsdb->imem == NULL); + tsdb->imem = tsdb->mem; + tsdb->mem = NULL; + taosThreadRwlockUnlock(&tsdb->rwLock); return 0; } -int32_t tsdbCommitBegin(STsdb *pTsdb, SCommitInfo *pInfo) { - if (!pTsdb) return 0; +int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) { + if (!tsdb) return 0; int32_t code = 0; int32_t lino = 0; - SMemTable *pMem = pTsdb->imem; - - if (pMem->nRow == 0 && pMem->nDel == 0) { - taosThreadRwlockWrlock(&pTsdb->rwLock); - pTsdb->imem = NULL; - taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbUnrefMemTable(pMem, NULL, true); + int32_t vid = TD_VID(tsdb->pVnode); + SMemTable *memt = tsdb->imem; + int64_t nRow = memt->nRow; + int64_t nDel = memt->nDel; + + if (!nRow && !nDel) { + taosThreadRwlockWrlock(&tsdb->rwLock); + tsdb->imem = NULL; + taosThreadRwlockUnlock(&tsdb->rwLock); + tsdbUnrefMemTable(memt, NULL, true); } else { - SCommitter committer; + SCommitter2 committer[1]; - code = open_committer(pTsdb, pInfo, &committer); + code = tsdbOpenCommitter(tsdb, info, committer); TSDB_CHECK_CODE(code, lino, _exit); - while (committer.nextKey != TSKEY_MAX) { - code = commit_fset(&committer); + while (committer->ctx->nextKey != TSKEY_MAX) { + code = tsdbCommitFileSet(committer); if (code) { lino = __LINE__; break; } } - code = close_committer(&committer, code); + code = tsdbCloseCommitter(committer, code); TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(vid, lino, code); } else { - tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(pTsdb->pVnode), __func__, pMem->nRow, - pMem->nDel); + tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, vid, __func__, nRow, nDel); } return code; } diff --git a/source/dnode/vnode/src/tsdb/dev/tsdbFSet.c b/source/dnode/vnode/src/tsdb/dev/tsdbFSet.c index 849ac7a663fd79966aeec909763241e5fd8b6f19..d7edd72d2214b3af19932782a31194cfec123917 100644 --- a/source/dnode/vnode/src/tsdb/dev/tsdbFSet.c +++ b/source/dnode/vnode/src/tsdb/dev/tsdbFSet.c @@ -37,7 +37,7 @@ static int32_t tsdbSttLvlInitEx(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lvl const STFileObj *fobj1; TARRAY2_FOREACH(&lvl1->farr, fobj1) { STFileObj *fobj; - code = tsdbTFileObjInit(pTsdb, &fobj1->f, &fobj); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj); if (code) { tsdbSttLvlClear(lvl); return code; @@ -66,25 +66,25 @@ static int32_t tsdbSttLvlApplyEdit(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl *l STFileObj *fobj2 = i2 < TARRAY2_SIZE(&lvl2->farr) ? TARRAY2_GET(&lvl2->farr, i2) : NULL; if (fobj1 && fobj2) { - if (fobj1->f.cid < fobj2->f.cid) { + if (fobj1->f->cid < fobj2->f->cid) { // create a file obj - code = tsdbTFileObjInit(pTsdb, &fobj1->f, &fobj2); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); if (code) return code; code = TARRAY2_APPEND(&lvl2->farr, fobj2); if (code) return code; i1++; i2++; - } else if (fobj1->f.cid > fobj2->f.cid) { + } else if (fobj1->f->cid > fobj2->f->cid) { // remove a file obj TARRAY2_REMOVE(&lvl2->farr, i2, tsdbSttLvlRemoveFObj); } else { - if (tsdbIsSameTFile(&fobj1->f, &fobj2->f)) { - if (tsdbIsTFileChanged(&fobj1->f, &fobj2->f)) { - fobj2->f = fobj1->f; + if (tsdbIsSameTFile(fobj1->f, fobj2->f)) { + if (tsdbIsTFileChanged(fobj1->f, fobj2->f)) { + fobj2->f[0] = fobj1->f[0]; } } else { TARRAY2_REMOVE(&lvl2->farr, i2, tsdbSttLvlRemoveFObj); - code = tsdbTFileObjInit(pTsdb, &fobj1->f, &fobj2); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); if (code) return code; code = TARRAY2_SORT_INSERT(&lvl2->farr, fobj2, tsdbTFileObjCmpr); if (code) return code; @@ -94,7 +94,7 @@ static int32_t tsdbSttLvlApplyEdit(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl *l } } else if (fobj1) { // create a file obj - code = tsdbTFileObjInit(pTsdb, &fobj1->f, &fobj2); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj2); if (code) return code; code = TARRAY2_APPEND(&lvl2->farr, fobj2); if (code) return code; @@ -127,7 +127,7 @@ static int32_t tsdbSttLvlToJson(const SSttLvl *lvl, cJSON *json) { if (item == NULL) return TSDB_CODE_OUT_OF_MEMORY; cJSON_AddItemToArray(ajson, item); - int32_t code = tsdbTFileToJson(&fobj->f, item); + int32_t code = tsdbTFileToJson(fobj->f, item); if (code) return code; } @@ -186,7 +186,7 @@ int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json) { for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { if (fset->farr[ftype] == NULL) continue; - code = tsdbTFileToJson(&fset->farr[ftype]->f, json); + code = tsdbTFileToJson(fset->farr[ftype]->f, json); if (code) return code; } @@ -266,10 +266,10 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { code = tsdbTFileObjInit(pTsdb, &op->nf, &fobj); if (code) return code; - if (fobj->f.type == TSDB_FTYPE_STT) { - SSttLvl *lvl = tsdbTFileSetGetLvl(fset, fobj->f.stt->level); + if (fobj->f->type == TSDB_FTYPE_STT) { + SSttLvl *lvl = tsdbTFileSetGetLvl(fset, fobj->f->stt->level); if (!lvl) { - code = tsdbSttLvlInit(fobj->f.stt->level, &lvl); + code = tsdbSttLvlInit(fobj->f->stt->level, &lvl); if (code) return code; code = TARRAY2_SORT_INSERT(&fset->lvlArr, lvl, tsdbSttLvlCmprFn); @@ -279,8 +279,8 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { code = TARRAY2_SORT_INSERT(&lvl->farr, fobj, tsdbTFileObjCmpr); if (code) return code; } else { - ASSERT(fset->farr[fobj->f.type] == NULL); - fset->farr[fobj->f.type] = fobj; + ASSERT(fset->farr[fobj->f->type] == NULL); + fset->farr[fobj->f->type] = fobj; } } else if (op->optype == TSDB_FOP_REMOVE) { // delete a file @@ -288,7 +288,7 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { SSttLvl *lvl = tsdbTFileSetGetLvl(fset, op->of.stt->level); ASSERT(lvl); - STFileObj tfobj = {.f = {.cid = op->of.cid}}; + STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}; STFileObj *tfobjp = &tfobj; int32_t idx = TARRAY2_SEARCH_IDX(&lvl->farr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); ASSERT(idx >= 0); @@ -299,7 +299,7 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { // TARRAY2_REMOVE(&fset->lvlArr, lvl - fset->lvlArr.data, tsdbSttLvlClear); } } else { - ASSERT(tsdbIsSameTFile(&op->of, &fset->farr[op->of.type]->f)); + ASSERT(tsdbIsSameTFile(&op->of, fset->farr[op->of.type]->f)); tsdbTFileObjUnref(fset->farr[op->of.type]); fset->farr[op->of.type] = NULL; } @@ -308,14 +308,14 @@ int32_t tsdbTFileSetEdit(STsdb *pTsdb, STFileSet *fset, const STFileOp *op) { SSttLvl *lvl = tsdbTFileSetGetLvl(fset, op->of.stt->level); ASSERT(lvl); - STFileObj tfobj = {.f = {.cid = op->of.cid}}, *tfobjp = &tfobj; + STFileObj tfobj = {.f[0] = {.cid = op->of.cid}}, *tfobjp = &tfobj; tfobjp = TARRAY2_SEARCH_EX(&lvl->farr, &tfobjp, tsdbTFileObjCmpr, TD_EQ); ASSERT(tfobjp); - tfobjp->f = op->nf; + tfobjp->f[0] = op->nf; } else { - fset->farr[op->nf.type]->f = op->nf; + fset->farr[op->nf.type]->f[0] = op->nf; } } @@ -334,18 +334,18 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f STFileObj *fobj2 = fset2->farr[ftype]; if (fobj1 && fobj2) { - if (tsdbIsSameTFile(&fobj1->f, &fobj2->f)) { - if (tsdbIsTFileChanged(&fobj1->f, &fobj2->f)) { - fobj2->f = fobj1->f; + if (tsdbIsSameTFile(fobj1->f, fobj2->f)) { + if (tsdbIsTFileChanged(fobj1->f, fobj2->f)) { + fobj2->f[0] = fobj1->f[0]; } } else { tsdbTFileObjRemove(fobj2); - code = tsdbTFileObjInit(pTsdb, &fobj1->f, &fset2->farr[ftype]); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); if (code) return code; } } else if (fobj1) { // create a new file - code = tsdbTFileObjInit(pTsdb, &fobj1->f, &fset2->farr[ftype]); + code = tsdbTFileObjInit(pTsdb, fobj1->f, &fset2->farr[ftype]); if (code) return code; } else { // remove the file @@ -412,7 +412,7 @@ int32_t tsdbTFileSetInitEx(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fse for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { if (fset1->farr[ftype] == NULL) continue; - code = tsdbTFileObjInit(pTsdb, &fset1->farr[ftype]->f, &fset[0]->farr[ftype]); + code = tsdbTFileObjInit(pTsdb, fset1->farr[ftype]->f, &fset[0]->farr[ftype]); if (code) { tsdbTFileSetClear(fset); return code; @@ -479,12 +479,12 @@ int64_t tsdbTFileSetMaxCid(const STFileSet *fset) { int64_t maxCid = 0; for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { if (fset->farr[ftype] == NULL) continue; - maxCid = TMAX(maxCid, fset->farr[ftype]->f.cid); + maxCid = TMAX(maxCid, fset->farr[ftype]->f->cid); } const SSttLvl *lvl; const STFileObj *fobj; TARRAY2_FOREACH(&fset->lvlArr, lvl) { - TARRAY2_FOREACH(&lvl->farr, fobj) { maxCid = TMAX(maxCid, fobj->f.cid); } + TARRAY2_FOREACH(&lvl->farr, fobj) { maxCid = TMAX(maxCid, fobj->f->cid); } } return maxCid; } diff --git a/source/dnode/vnode/src/tsdb/dev/tsdbFile.c b/source/dnode/vnode/src/tsdb/dev/tsdbFile.c index 7330e413c2bdf9711b7af44b7ffbdd59432eeca3..1c99cafeb9059158a2a98b9f821e88d5f4c2c2da 100644 --- a/source/dnode/vnode/src/tsdb/dev/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/dev/tsdbFile.c @@ -205,7 +205,7 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { if (!fobj[0]) return TSDB_CODE_OUT_OF_MEMORY; taosThreadMutexInit(&fobj[0]->mutex, NULL); - fobj[0]->f = *f; + fobj[0]->f[0] = f[0]; fobj[0]->state = TSDB_FSTATE_LIVE; fobj[0]->ref = 1; tsdbTFileName(pTsdb, f, fobj[0]->fname); @@ -295,9 +295,9 @@ bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2) { } int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2) { - if (fobj1[0]->f.cid < fobj2[0]->f.cid) { + if (fobj1[0]->f->cid < fobj2[0]->f->cid) { return -1; - } else if (fobj1[0]->f.cid > fobj2[0]->f.cid) { + } else if (fobj1[0]->f->cid > fobj2[0]->f->cid) { return 1; } else { return 0; diff --git a/source/dnode/vnode/src/tsdb/dev/tsdbMerge.c b/source/dnode/vnode/src/tsdb/dev/tsdbMerge.c index 79931df44f075e2eeb9f5a7eec58afda9c1f88a1..40b9e4f25112c33c7b33d6a85b22c9622b183497 100644 --- a/source/dnode/vnode/src/tsdb/dev/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/dev/tsdbMerge.c @@ -91,7 +91,7 @@ static int32_t tsdbMergeToDataWriteTSDataBlock(SMerger *merger) { // code = tsdbDataFWriteTSDataBlock(merger->dataWriter, &merger->ctx->bData); // TSDB_CHECK_CODE(code, lino, _exit); } else { - code = tsdbSttFWriteTSDataBlock(merger->sttWriter, &merger->ctx->bData); + code = tsdbSttFileWriteTSDataBlock(merger->sttWriter, &merger->ctx->bData); TSDB_CHECK_CODE(code, lino, _exit); } @@ -157,7 +157,7 @@ static int32_t tsdbMergeToUpperLevel(SMerger *merger) { if (!merger->ctx->row) break; - code = tsdbSttFWriteTSData(merger->sttWriter, merger->ctx->row); + code = tsdbSttFileWriteTSData(merger->sttWriter, merger->ctx->row); TSDB_CHECK_CODE(code, lino, _exit); } @@ -187,7 +187,7 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) { } fobj = TARRAY2_GET(&lvl->farr, 0); - if (fobj->f.stt->nseg < merger->tsdb->pVnode->config.sttTrigger) { + if (fobj->f->stt->nseg < merger->tsdb->pVnode->config.sttTrigger) { merger->ctx->toData = false; break; } else { @@ -208,9 +208,9 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) { // add the operation STFileOp op = { - .fid = fobj->f.fid, + .fid = fobj->f->fid, .optype = TSDB_FOP_REMOVE, - .of = fobj->f, + .of = fobj->f[0], }; code = TARRAY2_APPEND(&merger->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); @@ -227,9 +227,9 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) { .skmTb = &merger->skmTb, .skmRow = &merger->skmRow, .aBuf = merger->aBuf, - .file = fobj->f, + .file = fobj->f[0], }; - code = tsdbSttFWriterOpen(&config, &merger->sttWriter); + code = tsdbSttFileWriterOpen(&config, &merger->sttWriter); TSDB_CHECK_CODE(code, lino, _exit); } else { SSttFileWriterConfig config = { @@ -250,7 +250,7 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) { .stt = {{.level = merger->ctx->level, .nseg = 0}}, }, }; - code = tsdbSttFWriterOpen(&config, &merger->sttWriter); + code = tsdbSttFileWriterOpen(&config, &merger->sttWriter); TSDB_CHECK_CODE(code, lino, _exit); } @@ -276,7 +276,7 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) { int32_t vid = TD_VID(merger->tsdb->pVnode); STFileOp op; - code = tsdbSttFWriterClose(&merger->sttWriter, 0, &op); + code = tsdbSttFileWriterClose(&merger->sttWriter, 0, &op); TSDB_CHECK_CODE(code, lino, _exit); if (op.optype != TSDB_FOP_NONE) { @@ -356,7 +356,7 @@ int32_t tsdbMerge(STsdb *tsdb) { fobj = TARRAY2_GET(&lvl0->farr, 0); - if (fobj->f.stt->nseg >= sttTrigger) { + if (fobj->f->stt->nseg >= sttTrigger) { code = tsdbMergeFileSet(merger, fset); TSDB_CHECK_CODE(code, lino, _exit); }