Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 48593015 编写于 作者: S Shengliang Guan
浏览文件
操作

Merge branch '3.0' of https://github.com/taosdata/TDengine into 3.0

上级 69cc47e6 67e09ead
显示空白变更内容
内联 并排
Showing with 2684 addition and 1926 deletion
include/common/tmsg.h
浏览文件 @ 48593015
...@@ -1070,6 +1070,7 @@ typedef struct { ...@@ -1070,6 +1070,7 @@ typedef struct {
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int32_t syncState; int32_t syncState;
int64_t cacheUsage;
int64_t numOfTables; int64_t numOfTables;
int64_t numOfTimeSeries; int64_t numOfTimeSeries;
int64_t totalStorage; int64_t totalStorage;
......
include/util/trbtree.h 0 → 100644
浏览文件 @ 48593015
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_RBTREE_H_
#define _TD_UTIL_RBTREE_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SRBTree SRBTree;
typedef struct SRBTreeNode SRBTreeNode;
typedef struct SRBTreeIter SRBTreeIter;
typedef int32_t (*tRBTreeCmprFn)(const void *, const void *);
// SRBTree =============================================
#define tRBTreeMin(T) ((T)->min == ((T)->NIL) ? NULL : (T)->min)
#define tRBTreeMax(T) ((T)->max == ((T)->NIL) ? NULL : (T)->max)
void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn);
SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z);
void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z);
SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey);
SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey);
// SRBTreeIter =============================================
#define tRBTreeIterCreate(tree, ascend) \
(SRBTreeIter) { .asc = (ascend), .pTree = (tree), .pNode = (ascend) ? (tree)->min : (tree)->max }
SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter);
// STRUCT =============================================
typedef enum { RED, BLACK } ECOLOR;
struct SRBTreeNode {
ECOLOR color;
SRBTreeNode *parent;
SRBTreeNode *left;
SRBTreeNode *right;
};
#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1])
struct SRBTree {
tRBTreeCmprFn cmprFn;
int64_t n;
SRBTreeNode *root;
SRBTreeNode *min;
SRBTreeNode *max;
SRBTreeNode *NIL;
SRBTreeNode NILNODE;
};
struct SRBTreeIter {
int8_t asc;
SRBTree *pTree;
SRBTreeNode *pNode;
};
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_RBTREE_H_*/
\ No newline at end of file
source/client/src/clientImpl.c
浏览文件 @ 48593015
...@@ -1672,7 +1672,12 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int ...@@ -1672,7 +1672,12 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
break; break;
} }
} }
if (!needConvert) return TSDB_CODE_SUCCESS;
if (!needConvert) {
return TSDB_CODE_SUCCESS;
}
tscDebug("start to convert form json format string");
char* p = (char*)pResultInfo->pData; char* p = (char*)pResultInfo->pData;
int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows);
......
source/common/src/systable.c
浏览文件 @ 48593015
...@@ -207,6 +207,7 @@ static const SSysDbTableSchema vgroupsSchema[] = { ...@@ -207,6 +207,7 @@ static const SSysDbTableSchema vgroupsSchema[] = {
{.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
......
source/common/src/tdatablock.c
浏览文件 @ 48593015
...@@ -2120,6 +2120,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ ...@@ -2120,6 +2120,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_
int32_t* rows = (int32_t*)data; int32_t* rows = (int32_t*)data;
*rows = pBlock->info.rows; *rows = pBlock->info.rows;
data += sizeof(int32_t); data += sizeof(int32_t);
ASSERT(*rows > 0);
int32_t* cols = (int32_t*)data; int32_t* cols = (int32_t*)data;
*cols = numOfCols; *cols = numOfCols;
...@@ -2183,6 +2184,8 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ ...@@ -2183,6 +2184,8 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_
*actualLen = *dataLen; *actualLen = *dataLen;
*groupId = pBlock->info.groupId; *groupId = pBlock->info.groupId;
ASSERT(*dataLen > 0);
uDebug("build data block, actualLen:%d, rows:%d, cols:%d", *dataLen, *rows, *cols);
} }
const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
......
source/common/src/tglobal.c
浏览文件 @ 48593015
...@@ -129,10 +129,6 @@ int32_t tsMinIntervalTime = 1; ...@@ -129,10 +129,6 @@ int32_t tsMinIntervalTime = 1;
int32_t tsQueryBufferSize = -1; int32_t tsQueryBufferSize = -1;
int64_t tsQueryBufferSizeBytes = -1; int64_t tsQueryBufferSizeBytes = -1;
// tsdb config
// For backward compatibility
bool tsdbForceKeepFile = false;
int32_t tsDiskCfgNum = 0; int32_t tsDiskCfgNum = 0;
SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0};
......
source/common/src/tmsg.c
浏览文件 @ 48593015
...@@ -994,6 +994,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { ...@@ -994,6 +994,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i); SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i);
if (tEncodeI32(&encoder, pload->vgId) < 0) return -1; if (tEncodeI32(&encoder, pload->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pload->syncState) < 0) return -1; if (tEncodeI32(&encoder, pload->syncState) < 0) return -1;
if (tEncodeI64(&encoder, pload->cacheUsage) < 0) return -1;
if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1;
if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1;
if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1; if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1;
...@@ -1063,6 +1064,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { ...@@ -1063,6 +1064,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
SVnodeLoad vload = {0}; SVnodeLoad vload = {0};
if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1; if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1;
if (tDecodeI32(&decoder, &vload.syncState) < 0) return -1; if (tDecodeI32(&decoder, &vload.syncState) < 0) return -1;
if (tDecodeI64(&decoder, &vload.cacheUsage) < 0) return -1;
if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1;
if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1;
if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1;
......
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @ 48593015
...@@ -343,6 +343,7 @@ typedef struct { ...@@ -343,6 +343,7 @@ typedef struct {
uint32_t hashEnd; uint32_t hashEnd;
char dbName[TSDB_DB_FNAME_LEN]; char dbName[TSDB_DB_FNAME_LEN];
int64_t dbUid; int64_t dbUid;
int64_t cacheUsage;
int64_t numOfTables; int64_t numOfTables;
int64_t numOfTimeSeries; int64_t numOfTimeSeries;
int64_t totalStorage; int64_t totalStorage;
......
source/dnode/mnode/impl/src/mndDnode.c
浏览文件 @ 48593015
...@@ -347,6 +347,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { ...@@ -347,6 +347,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId); SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId);
if (pVgroup != NULL) { if (pVgroup != NULL) {
if (pVload->syncState == TAOS_SYNC_STATE_LEADER) { if (pVload->syncState == TAOS_SYNC_STATE_LEADER) {
pVgroup->cacheUsage = pVload->cacheUsage;
pVgroup->numOfTables = pVload->numOfTables; pVgroup->numOfTables = pVload->numOfTables;
pVgroup->numOfTimeSeries = pVload->numOfTimeSeries; pVgroup->numOfTimeSeries = pVload->numOfTimeSeries;
pVgroup->totalStorage = pVload->totalStorage; pVgroup->totalStorage = pVload->totalStorage;
......
source/dnode/mnode/impl/src/mndVgroup.c
浏览文件 @ 48593015
...@@ -696,6 +696,9 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p ...@@ -696,6 +696,9 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppendNULL(pColInfo, numOfRows); colDataAppendNULL(pColInfo, numOfRows);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->cacheUsage, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppendNULL(pColInfo, numOfRows); colDataAppendNULL(pColInfo, numOfRows);
......
source/dnode/vnode/CMakeLists.txt
浏览文件 @ 48593015
...@@ -49,6 +49,10 @@ target_sources( ...@@ -49,6 +49,10 @@ target_sources(
"src/tsdb/tsdbSnapshot.c" "src/tsdb/tsdbSnapshot.c"
"src/tsdb/tsdbCacheRead.c" "src/tsdb/tsdbCacheRead.c"
"src/tsdb/tsdbRetention.c" "src/tsdb/tsdbRetention.c"
"src/tsdb/tsdbDiskData.c"
"src/tsdb/tsdbCompress.c"
"src/tsdb/tsdbCompact.c"
"src/tsdb/tsdbMergeTree.c"
# tq # tq
"src/tq/tq.c" "src/tq/tq.c"
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ 48593015
...@@ -155,6 +155,7 @@ int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int6 ...@@ -155,6 +155,7 @@ int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int6
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode); size_t tsdbCacheGetCapacity(SVnode *pVnode);
size_t tsdbCacheGetUsage(SVnode *pVnode);
// tq // tq
typedef struct SMetaTableInfo { typedef struct SMetaTableInfo {
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ 48593015
...@@ -42,15 +42,15 @@ typedef struct SMemTable SMemTable; ...@@ -42,15 +42,15 @@ typedef struct SMemTable SMemTable;
typedef struct STbDataIter STbDataIter; typedef struct STbDataIter STbDataIter;
typedef struct SMapData SMapData; typedef struct SMapData SMapData;
typedef struct SBlockIdx SBlockIdx; typedef struct SBlockIdx SBlockIdx;
typedef struct SBlock SBlock; typedef struct SDataBlk SDataBlk;
typedef struct SBlockL SBlockL; typedef struct SSstBlk SSstBlk;
typedef struct SColData SColData; typedef struct SColData SColData;
typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SDiskDataHdr SDiskDataHdr;
typedef struct SBlockData SBlockData; typedef struct SBlockData SBlockData;
typedef struct SDelFile SDelFile; typedef struct SDelFile SDelFile;
typedef struct SHeadFile SHeadFile; typedef struct SHeadFile SHeadFile;
typedef struct SDataFile SDataFile; typedef struct SDataFile SDataFile;
typedef struct SLastFile SLastFile; typedef struct SSstFile SSstFile;
typedef struct SSmaFile SSmaFile; typedef struct SSmaFile SSmaFile;
typedef struct SDFileSet SDFileSet; typedef struct SDFileSet SDFileSet;
typedef struct SDataFWriter SDataFWriter; typedef struct SDataFWriter SDataFWriter;
...@@ -64,9 +64,12 @@ typedef struct STsdbReadSnap STsdbReadSnap; ...@@ -64,9 +64,12 @@ typedef struct STsdbReadSnap STsdbReadSnap;
typedef struct SBlockInfo SBlockInfo; typedef struct SBlockInfo SBlockInfo;
typedef struct SSmaInfo SSmaInfo; typedef struct SSmaInfo SSmaInfo;
typedef struct SBlockCol SBlockCol; typedef struct SBlockCol SBlockCol;
typedef struct SVersionRange SVersionRange;
#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F)
#define TSDB_MAX_SUBBLOCKS 8 #define TSDB_MAX_SUBBLOCKS 8
#define TSDB_MAX_LAST_FILE 16
#define TSDB_DEFAULT_LAST_FILE 8
#define TSDB_FHDR_SIZE 512 #define TSDB_FHDR_SIZE 512
#define HAS_NONE ((int8_t)0x1) #define HAS_NONE ((int8_t)0x1)
...@@ -111,15 +114,15 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2); ...@@ -111,15 +114,15 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
int32_t tPutBlockCol(uint8_t *p, void *ph); int32_t tPutBlockCol(uint8_t *p, void *ph);
int32_t tGetBlockCol(uint8_t *p, void *ph); int32_t tGetBlockCol(uint8_t *p, void *ph);
int32_t tBlockColCmprFn(const void *p1, const void *p2); int32_t tBlockColCmprFn(const void *p1, const void *p2);
// SBlock // SDataBlk
void tBlockReset(SBlock *pBlock); void tDataBlkReset(SDataBlk *pBlock);
int32_t tPutBlock(uint8_t *p, void *ph); int32_t tPutDataBlk(uint8_t *p, void *ph);
int32_t tGetBlock(uint8_t *p, void *ph); int32_t tGetDataBlk(uint8_t *p, void *ph);
int32_t tBlockCmprFn(const void *p1, const void *p2); int32_t tDataBlkCmprFn(const void *p1, const void *p2);
bool tBlockHasSma(SBlock *pBlock); bool tDataBlkHasSma(SDataBlk *pDataBlk);
// SBlockL // SSstBlk
int32_t tPutBlockL(uint8_t *p, void *ph); int32_t tPutSstBlk(uint8_t *p, void *ph);
int32_t tGetBlockL(uint8_t *p, void *ph); int32_t tGetSstBlk(uint8_t *p, void *ph);
// SBlockIdx // SBlockIdx
int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tPutBlockIdx(uint8_t *p, void *ph);
int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph);
...@@ -170,6 +173,7 @@ int32_t tGetDelData(uint8_t *p, void *ph); ...@@ -170,6 +173,7 @@ int32_t tGetDelData(uint8_t *p, void *ph);
void tMapDataReset(SMapData *pMapData); void tMapDataReset(SMapData *pMapData);
void tMapDataClear(SMapData *pMapData); void tMapDataClear(SMapData *pMapData);
int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *)); int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *));
int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo);
void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *)); void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *));
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
int32_t (*tItemCmprFn)(const void *, const void *), void *pItem); int32_t (*tItemCmprFn)(const void *, const void *), void *pItem);
...@@ -215,7 +219,7 @@ bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); ...@@ -215,7 +219,7 @@ bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile); int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile);
int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile); int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile);
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile);
...@@ -224,7 +228,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet); ...@@ -224,7 +228,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet);
void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]); void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]);
void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]); void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]);
void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]); void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]);
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]); void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]);
// SDelFile // SDelFile
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]); void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]);
...@@ -250,7 +254,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); ...@@ -250,7 +254,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx);
int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx);
int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL); int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk);
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
int8_t cmprAlg, int8_t toLast); int8_t cmprAlg, int8_t toLast);
...@@ -260,10 +264,10 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS ...@@ -260,10 +264,10 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx);
int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData);
int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk);
int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData); int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData);
// SDelFWriter // SDelFWriter
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
...@@ -278,6 +282,8 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); ...@@ -278,6 +282,8 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx);
// tsdbRead.c ============================================================================================== // tsdbRead.c ==============================================================================================
int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap); int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap);
void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap); void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap);
// tsdbMerge.c ==============================================================================================
int32_t tsdbMerge(STsdb *pTsdb);
#define TSDB_CACHE_NO(c) ((c).cacheLast == 0) #define TSDB_CACHE_NO(c) ((c).cacheLast == 0)
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
...@@ -324,6 +330,11 @@ struct TSDBKEY { ...@@ -324,6 +330,11 @@ struct TSDBKEY {
TSKEY ts; TSKEY ts;
}; };
struct SVersionRange {
uint64_t minVer;
uint64_t maxVer;
};
typedef struct SMemSkipListNode SMemSkipListNode; typedef struct SMemSkipListNode SMemSkipListNode;
struct SMemSkipListNode { struct SMemSkipListNode {
int8_t level; int8_t level;
...@@ -416,7 +427,7 @@ struct SSmaInfo { ...@@ -416,7 +427,7 @@ struct SSmaInfo {
int32_t size; int32_t size;
}; };
struct SBlock { struct SDataBlk {
TSDBKEY minKey; TSDBKEY minKey;
TSDBKEY maxKey; TSDBKEY maxKey;
int64_t minVer; int64_t minVer;
...@@ -428,7 +439,7 @@ struct SBlock { ...@@ -428,7 +439,7 @@ struct SBlock {
SSmaInfo smaInfo; SSmaInfo smaInfo;
}; };
struct SBlockL { struct SSstBlk {
int64_t suid; int64_t suid;
int64_t minUid; int64_t minUid;
int64_t maxUid; int64_t maxUid;
...@@ -467,12 +478,6 @@ struct SBlockData { ...@@ -467,12 +478,6 @@ struct SBlockData {
SArray *aColData; // SArray<SColData> SArray *aColData; // SArray<SColData>
}; };
// ================== TSDB global config
extern bool tsdbForceKeepFile;
#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
struct TABLEID { struct TABLEID {
tb_uid_t suid; tb_uid_t suid;
tb_uid_t uid; tb_uid_t uid;
...@@ -536,7 +541,7 @@ struct SDataFile { ...@@ -536,7 +541,7 @@ struct SDataFile {
int64_t size; int64_t size;
}; };
struct SLastFile { struct SSstFile {
volatile int32_t nRef; volatile int32_t nRef;
int64_t commitID; int64_t commitID;
...@@ -556,8 +561,9 @@ struct SDFileSet { ...@@ -556,8 +561,9 @@ struct SDFileSet {
int32_t fid; int32_t fid;
SHeadFile *pHeadF; SHeadFile *pHeadF;
SDataFile *pDataF; SDataFile *pDataF;
SLastFile *pLastF;
SSmaFile *pSmaF; SSmaFile *pSmaF;
uint8_t nSstF;
SSstFile *aSstF[TSDB_MAX_LAST_FILE];
}; };
struct SRowIter { struct SRowIter {
...@@ -586,13 +592,13 @@ struct SDataFWriter { ...@@ -586,13 +592,13 @@ struct SDataFWriter {
TdFilePtr pHeadFD; TdFilePtr pHeadFD;
TdFilePtr pDataFD; TdFilePtr pDataFD;
TdFilePtr pLastFD;
TdFilePtr pSmaFD; TdFilePtr pSmaFD;
TdFilePtr pLastFD;
SHeadFile fHead; SHeadFile fHead;
SDataFile fData; SDataFile fData;
SLastFile fLast;
SSmaFile fSma; SSmaFile fSma;
SSstFile fSst[TSDB_MAX_LAST_FILE];
uint8_t *aBuf[4]; uint8_t *aBuf[4];
}; };
...@@ -603,6 +609,36 @@ struct STsdbReadSnap { ...@@ -603,6 +609,36 @@ struct STsdbReadSnap {
STsdbFS fs; STsdbFS fs;
}; };
struct SDataFReader {
STsdb *pTsdb;
SDFileSet *pSet;
TdFilePtr pHeadFD;
TdFilePtr pDataFD;
TdFilePtr pSmaFD;
TdFilePtr aLastFD[TSDB_MAX_LAST_FILE];
uint8_t *aBuf[3];
};
typedef struct {
int64_t suid;
int64_t uid;
TSDBROW row;
} SRowInfo;
typedef struct SMergeTree {
int8_t backward;
SRBTree rbt;
SArray *pIterList;
struct SLDataIter *pIter;
} SMergeTree;
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange);
void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter);
bool tMergeTreeNext(SMergeTree *pMTree);
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree);
// ========== inline functions ========== // ========== inline functions ==========
static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
TSDBKEY *pKey1 = (TSDBKEY *)p1; TSDBKEY *pKey1 = (TSDBKEY *)p1;
......
source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @ 48593015
...@@ -36,6 +36,7 @@ ...@@ -36,6 +36,7 @@
#include "tlosertree.h" #include "tlosertree.h"
#include "tlrucache.h" #include "tlrucache.h"
#include "tmsgcb.h" #include "tmsgcb.h"
#include "trbtree.h"
#include "tref.h" #include "tref.h"
#include "tskiplist.h" #include "tskiplist.h"
#include "tstream.h" #include "tstream.h"
......
source/dnode/vnode/src/tsdb/tsdbCache.c
浏览文件 @ 48593015
...@@ -420,29 +420,14 @@ typedef enum { ...@@ -420,29 +420,14 @@ typedef enum {
typedef struct { typedef struct {
SFSLASTNEXTROWSTATES state; // [input] SFSLASTNEXTROWSTATES state; // [input]
STsdb *pTsdb; // [input] STsdb *pTsdb; // [input]
SBlockIdx *pBlockIdxExp; // [input]
STSchema *pTSchema; // [input]
tb_uid_t suid;
tb_uid_t uid; tb_uid_t uid;
int32_t nFileSet; int32_t nFileSet;
int32_t iFileSet; int32_t iFileSet;
SArray *aDFileSet; SArray *aDFileSet;
SDataFReader *pDataFReader; SDataFReader *pDataFReader;
SArray *aBlockL;
SBlockL *pBlockL;
SBlockData *pBlockDataL;
SBlockData blockDataL;
int32_t nRow;
int32_t iRow;
TSDBROW row; TSDBROW row;
/*
SArray *aBlockIdx; SMergeTree mergeTree;
SBlockIdx *pBlockIdx;
SMapData blockMap;
int32_t nBlock;
int32_t iBlock;
SBlock block;
*/
} SFSLastNextRowIter; } SFSLastNextRowIter;
static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
...@@ -451,22 +436,16 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { ...@@ -451,22 +436,16 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
switch (state->state) { switch (state->state) {
case SFSLASTNEXTROW_FS: case SFSLASTNEXTROW_FS:
// state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet;
state->nFileSet = taosArrayGetSize(state->aDFileSet); state->nFileSet = taosArrayGetSize(state->aDFileSet);
state->iFileSet = state->nFileSet; state->iFileSet = state->nFileSet;
state->pBlockDataL = NULL;
case SFSLASTNEXTROW_FILESET: { case SFSLASTNEXTROW_FILESET: {
SDFileSet *pFileSet = NULL; SDFileSet *pFileSet = NULL;
_next_fileset: _next_fileset:
if (--state->iFileSet >= 0) { if (--state->iFileSet >= 0) {
pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet);
} else { } else {
if (state->pBlockDataL) { // tMergeTreeClose(&state->mergeTree);
tBlockDataDestroy(state->pBlockDataL, 1);
state->pBlockDataL = NULL;
}
*ppRow = NULL; *ppRow = NULL;
return code; return code;
...@@ -475,68 +454,24 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { ...@@ -475,68 +454,24 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet); code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet);
if (code) goto _err; if (code) goto _err;
if (!state->aBlockL) { tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->uid,
state->aBlockL = taosArrayInit(0, sizeof(SBlockL)); &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX},
} else { &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX});
taosArrayClear(state->aBlockL); bool hasVal = tMergeTreeNext(&state->mergeTree);
} if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET;
code = tsdbReadBlockL(state->pDataFReader, state->aBlockL); // tMergeTreeClose(&state->mergeTree);
if (code) goto _err;
// SBlockL *pBlockL = (SBlockL *)taosArrayGet(state->aBlockL, state->iBlockL);
state->pBlockL = taosArraySearch(state->aBlockL, state->pBlockIdxExp, tCmprBlockL, TD_EQ);
if (!state->pBlockL) {
goto _next_fileset; goto _next_fileset;
} }
int64_t suid = state->pBlockL->suid;
int64_t uid = state->pBlockL->maxUid;
if (!state->pBlockDataL) {
state->pBlockDataL = &state->blockDataL;
tBlockDataCreate(state->pBlockDataL);
}
code = tBlockDataInit(state->pBlockDataL, suid, suid ? 0 : uid, state->pTSchema);
if (code) goto _err;
}
case SFSLASTNEXTROW_BLOCKDATA:
code = tsdbReadLastBlock(state->pDataFReader, state->pBlockL, state->pBlockDataL);
if (code) goto _err;
state->nRow = state->blockDataL.nRow;
state->iRow = state->nRow - 1;
if (!state->pBlockDataL->uid) {
while (state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) {
--state->iRow;
}
}
state->state = SFSLASTNEXTROW_BLOCKROW; state->state = SFSLASTNEXTROW_BLOCKROW;
case SFSLASTNEXTROW_BLOCKROW:
if (state->pBlockDataL->uid) {
if (state->iRow >= 0) {
state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow);
*ppRow = &state->row;
if (--state->iRow < 0) {
state->state = SFSLASTNEXTROW_FILESET;
} }
} case SFSLASTNEXTROW_BLOCKROW:
} else { state->row = tMergeTreeGetRow(&state->mergeTree);
if (state->iRow >= 0 && state->pBlockIdxExp->uid == state->pBlockDataL->aUid[state->iRow]) {
state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow);
*ppRow = &state->row; *ppRow = &state->row;
bool hasVal = tMergeTreeNext(&state->mergeTree);
if (--state->iRow < 0 || state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET; state->state = SFSLASTNEXTROW_FILESET;
} }
}
}
return code; return code;
default: default:
ASSERT(0); ASSERT(0);
...@@ -548,15 +483,6 @@ _err: ...@@ -548,15 +483,6 @@ _err:
tsdbDataFReaderClose(&state->pDataFReader); tsdbDataFReaderClose(&state->pDataFReader);
state->pDataFReader = NULL; state->pDataFReader = NULL;
} }
if (state->aBlockL) {
taosArrayDestroy(state->aBlockL);
state->aBlockL = NULL;
}
if (state->pBlockDataL) {
tBlockDataDestroy(state->pBlockDataL, 1);
state->pBlockDataL = NULL;
}
*ppRow = NULL; *ppRow = NULL;
return code; return code;
...@@ -574,14 +500,6 @@ int32_t clearNextRowFromFSLast(void *iter) { ...@@ -574,14 +500,6 @@ int32_t clearNextRowFromFSLast(void *iter) {
tsdbDataFReaderClose(&state->pDataFReader); tsdbDataFReaderClose(&state->pDataFReader);
state->pDataFReader = NULL; state->pDataFReader = NULL;
} }
if (state->aBlockL) {
taosArrayDestroy(state->aBlockL);
state->aBlockL = NULL;
}
if (state->pBlockDataL) {
tBlockDataDestroy(state->pBlockDataL, 1);
state->pBlockDataL = NULL;
}
return code; return code;
} }
...@@ -609,7 +527,7 @@ typedef struct SFSNextRowIter { ...@@ -609,7 +527,7 @@ typedef struct SFSNextRowIter {
SMapData blockMap; SMapData blockMap;
int32_t nBlock; int32_t nBlock;
int32_t iBlock; int32_t iBlock;
SBlock block; SDataBlk block;
SBlockData blockData; SBlockData blockData;
SBlockData *pBlockData; SBlockData *pBlockData;
int32_t nRow; int32_t nRow;
...@@ -684,13 +602,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { ...@@ -684,13 +602,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
} }
case SFSNEXTROW_BLOCKDATA: case SFSNEXTROW_BLOCKDATA:
if (state->iBlock >= 0) { if (state->iBlock >= 0) {
SBlock block = {0}; SDataBlk block = {0};
tBlockReset(&block); tDataBlkReset(&block);
// tBlockDataReset(&state->blockData); // tBlockDataReset(&state->blockData);
tBlockDataReset(state->pBlockData); tBlockDataReset(state->pBlockData);
tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk);
/* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */
tBlockDataReset(state->pBlockData); tBlockDataReset(state->pBlockData);
code = tBlockDataInit(state->pBlockData, state->suid, state->uid, state->pTSchema); code = tBlockDataInit(state->pBlockData, state->suid, state->uid, state->pTSchema);
...@@ -972,9 +890,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs ...@@ -972,9 +890,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS;
pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.pTsdb = pTsdb;
pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet;
pIter->fsLastState.pBlockIdxExp = &pIter->idx;
pIter->fsLastState.pTSchema = pTSchema;
pIter->fsLastState.suid = suid;
pIter->fsLastState.uid = uid; pIter->fsLastState.uid = uid;
pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.state = SFSNEXTROW_FS;
...@@ -1372,8 +1287,11 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand ...@@ -1372,8 +1287,11 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand
// getTableCacheKeyS(uid, "l", key, &keyLen); // getTableCacheKeyS(uid, "l", key, &keyLen);
getTableCacheKey(uid, 1, key, &keyLen); getTableCacheKey(uid, 1, key, &keyLen);
LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen);
if (h) { if (!h) {
} else { taosThreadMutexLock(&pTsdb->lruMutex);
h = taosLRUCacheLookup(pCache, key, keyLen);
if (!h) {
SArray *pLastArray = NULL; SArray *pLastArray = NULL;
code = mergeLast(uid, pTsdb, &pLastArray); code = mergeLast(uid, pTsdb, &pLastArray);
// if table's empty or error, return code of -1 // if table's empty or error, return code of -1
...@@ -1384,13 +1302,18 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand ...@@ -1384,13 +1302,18 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand
} }
_taos_lru_deleter_t deleter = deleteTableCacheLast; _taos_lru_deleter_t deleter = deleteTableCacheLast;
LRUStatus status = LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, pLastArray, pLastArray->capacity, deleter, NULL,
taosLRUCacheInsert(pCache, key, keyLen, pLastArray, pLastArray->capacity, deleter, NULL, TAOS_LRU_PRIORITY_LOW); TAOS_LRU_PRIORITY_LOW);
if (status != TAOS_LRU_STATUS_OK) { if (status != TAOS_LRU_STATUS_OK) {
code = -1; code = -1;
} }
taosThreadMutexUnlock(&pTsdb->lruMutex);
h = taosLRUCacheLookup(pCache, key, keyLen); h = taosLRUCacheLookup(pCache, key, keyLen);
} else {
taosThreadMutexUnlock(&pTsdb->lruMutex);
}
} }
*handle = h; *handle = h;
...@@ -1411,3 +1334,5 @@ void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity) { ...@@ -1411,3 +1334,5 @@ void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity) {
} }
size_t tsdbCacheGetCapacity(SVnode *pVnode) { return taosLRUCacheGetCapacity(pVnode->pTsdb->lruCache); } size_t tsdbCacheGetCapacity(SVnode *pVnode) { return taosLRUCacheGetCapacity(pVnode->pTsdb->lruCache); }
size_t tsdbCacheGetUsage(SVnode *pVnode) { return taosLRUCacheGetUsage(pVnode->pTsdb->lruCache); }
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ 48593015
...@@ -20,11 +20,26 @@ typedef struct { ...@@ -20,11 +20,26 @@ typedef struct {
STSchema *pTSchema; STSchema *pTSchema;
} SSkmInfo; } SSkmInfo;
typedef enum { MEMORY_DATA_ITER = 0, LAST_DATA_ITER } EDataIterT;
typedef struct { typedef struct {
int64_t suid; SRBTreeNode n;
int64_t uid; SRowInfo r;
TSDBROW row; EDataIterT type;
} SRowInfo; union {
struct {
int32_t iTbDataP;
STbDataIter iter;
}; // memory data iter
struct {
int32_t iSst;
SArray *aSstBlk;
int32_t iSstBlk;
SBlockData bData;
int32_t iRow;
}; // sst file data iter
};
} SDataIter;
typedef struct { typedef struct {
STsdb *pTsdb; STsdb *pTsdb;
...@@ -35,8 +50,9 @@ typedef struct { ...@@ -35,8 +50,9 @@ typedef struct {
int32_t minRow; int32_t minRow;
int32_t maxRow; int32_t maxRow;
int8_t cmprAlg; int8_t cmprAlg;
SArray *aTbDataP; int8_t maxLast;
STsdbFS fs; SArray *aTbDataP; // memory
STsdbFS fs; // disk
// -------------- // --------------
TSKEY nextKey; // reset by each table commit TSKEY nextKey; // reset by each table commit
int32_t commitFid; int32_t commitFid;
...@@ -45,25 +61,24 @@ typedef struct { ...@@ -45,25 +61,24 @@ typedef struct {
// commit file data // commit file data
struct { struct {
SDataFReader *pReader; SDataFReader *pReader;
// data
SArray *aBlockIdx; // SArray<SBlockIdx> SArray *aBlockIdx; // SArray<SBlockIdx>
int32_t iBlockIdx; int32_t iBlockIdx;
SBlockIdx *pBlockIdx; SBlockIdx *pBlockIdx;
SMapData mBlock; // SMapData<SBlock> SMapData mBlock; // SMapData<SDataBlk>
SBlockData bData; SBlockData bData;
// last
SArray *aBlockL; // SArray<SBlockL>
int32_t iBlockL;
SBlockData bDatal;
int32_t iRow;
SRowInfo *pRowInfo;
SRowInfo rowInfo;
} dReader; } dReader;
struct {
SDataIter *pIter;
SRBTree rbt;
SDataIter dataIter;
SDataIter aDataIter[TSDB_MAX_LAST_FILE];
int8_t toLastOnly;
};
struct { struct {
SDataFWriter *pWriter; SDataFWriter *pWriter;
SArray *aBlockIdx; // SArray<SBlockIdx> SArray *aBlockIdx; // SArray<SBlockIdx>
SArray *aBlockL; // SArray<SBlockL> SArray *aSstBlk; // SArray<SSstBlk>
SMapData mBlock; // SMapData<SBlock> SMapData mBlock; // SMapData<SDataBlk>
SBlockData bData; SBlockData bData;
SBlockData bDatal; SBlockData bDatal;
} dWriter; } dWriter;
...@@ -77,11 +92,34 @@ typedef struct { ...@@ -77,11 +92,34 @@ typedef struct {
SArray *aDelData; // SArray<SDelData> SArray *aDelData; // SArray<SDelData>
} SCommitter; } SCommitter;
extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *aSstBlk,
SBlockData *pBlockData); // todo
static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter); static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter);
static int32_t tsdbCommitData(SCommitter *pCommitter); static int32_t tsdbCommitData(SCommitter *pCommitter);
static int32_t tsdbCommitDel(SCommitter *pCommitter); static int32_t tsdbCommitDel(SCommitter *pCommitter);
static int32_t tsdbCommitCache(SCommitter *pCommitter); static int32_t tsdbCommitCache(SCommitter *pCommitter);
static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno); static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno);
static int32_t tsdbNextCommitRow(SCommitter *pCommitter);
static int32_t tRowInfoCmprFn(const void *p1, const void *p2) {
SRowInfo *pInfo1 = (SRowInfo *)p1;
SRowInfo *pInfo2 = (SRowInfo *)p2;
if (pInfo1->suid < pInfo2->suid) {
return -1;
} else if (pInfo1->suid > pInfo2->suid) {
return 1;
}
if (pInfo1->uid < pInfo2->uid) {
return -1;
} else if (pInfo1->uid > pInfo2->uid) {
return 1;
}
return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row);
}
int32_t tsdbBegin(STsdb *pTsdb) { int32_t tsdbBegin(STsdb *pTsdb) {
int32_t code = 0; int32_t code = 0;
...@@ -294,7 +332,10 @@ static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t su ...@@ -294,7 +332,10 @@ static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t su
int32_t code = 0; int32_t code = 0;
if (suid) { if (suid) {
if (pCommitter->skmTable.suid == suid) goto _exit; if (pCommitter->skmTable.suid == suid) {
pCommitter->skmTable.uid = uid;
goto _exit;
}
} else { } else {
if (pCommitter->skmTable.uid == uid) goto _exit; if (pCommitter->skmTable.uid == uid) goto _exit;
} }
...@@ -334,73 +375,104 @@ _exit: ...@@ -334,73 +375,104 @@ _exit:
return code; return code;
} }
static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
ASSERT(pCommitter->dReader.pReader); ASSERT(pCommitter->dReader.pBlockIdx);
ASSERT(pCommitter->dReader.pRowInfo);
SBlockData *pBlockDatal = &pCommitter->dReader.bDatal;
pCommitter->dReader.iRow++;
if (pCommitter->dReader.iRow < pBlockDatal->nRow) {
if (pBlockDatal->uid) {
pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid;
} else {
pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow];
}
pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow);
} else {
pCommitter->dReader.iBlockL++;
if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) {
SBlockL *pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL);
int64_t suid = pBlockL->suid;
int64_t uid = pBlockL->maxUid;
code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid);
if (code) goto _exit;
code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); pCommitter->dReader.iBlockIdx++;
if (code) goto _exit; if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) {
pCommitter->dReader.pBlockIdx =
(SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx);
code = tsdbReadLastBlock(pCommitter->dReader.pReader, pBlockL, pBlockDatal); code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
if (code) goto _exit; if (code) goto _exit;
pCommitter->dReader.iRow = 0; ASSERT(pCommitter->dReader.mBlock.nItem > 0);
pCommitter->dReader.pRowInfo->suid = pBlockDatal->suid;
if (pBlockDatal->uid) {
pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid;
} else {
pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[0];
}
pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow);
} else { } else {
pCommitter->dReader.pRowInfo = NULL; pCommitter->dReader.pBlockIdx = NULL;
}
} }
_exit: _exit:
return code; return code;
} }
static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
ASSERT(pCommitter->dReader.pBlockIdx); pCommitter->pIter = NULL;
tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn);
pCommitter->dReader.iBlockIdx++; // memory
if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN};
pCommitter->dReader.pBlockIdx = SDataIter *pIter = &pCommitter->dataIter;
(SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); pIter->type = MEMORY_DATA_ITER;
pIter->iTbDataP = 0;
for (; pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP); pIter->iTbDataP++) {
STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP);
tsdbTbDataIterOpen(pTbData, &tKey, 0, &pIter->iter);
TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter);
if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow));
pRow = NULL;
}
code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (pRow == NULL) continue;
if (code) goto _exit;
ASSERT(pCommitter->dReader.mBlock.nItem > 0); pIter->r.suid = pTbData->suid;
pIter->r.uid = pTbData->uid;
pIter->r.row = *pRow;
break;
}
ASSERT(pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP));
tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter);
// disk
pCommitter->toLastOnly = 0;
SDataFReader *pReader = pCommitter->dReader.pReader;
if (pReader) {
if (pReader->pSet->nSstF >= pCommitter->maxLast) {
int8_t iIter = 0;
for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) {
pIter = &pCommitter->aDataIter[iIter];
pIter->type = LAST_DATA_ITER;
pIter->iSst = iSst;
code = tsdbReadSstBlk(pCommitter->dReader.pReader, iSst, pIter->aSstBlk);
if (code) goto _err;
if (taosArrayGetSize(pIter->aSstBlk) == 0) continue;
pIter->iSstBlk = 0;
SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, 0);
code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData);
if (code) goto _err;
pIter->iRow = 0;
pIter->r.suid = pIter->bData.suid;
pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0];
pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0);
tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter);
iIter++;
}
} else { } else {
pCommitter->dReader.pBlockIdx = NULL; for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) {
SSstFile *pSstFile = pReader->pSet->aSstF[iSst];
if (pSstFile->size > pSstFile->offset) {
pCommitter->toLastOnly = 1;
break;
}
}
}
} }
_exit: code = tsdbNextCommitRow(pCommitter);
if (code) goto _err;
return code;
_err:
return code; return code;
} }
...@@ -416,8 +488,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { ...@@ -416,8 +488,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
pCommitter->nextKey = TSKEY_MAX; pCommitter->nextKey = TSKEY_MAX;
// Reader // Reader
pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &(SDFileSet){.fid = pCommitter->commitFid}, SDFileSet tDFileSet = {.fid = pCommitter->commitFid};
tDFileSetCmprFn, TD_EQ); pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ);
if (pRSet) { if (pRSet) {
code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet);
if (code) goto _err; if (code) goto _err;
...@@ -427,68 +499,58 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { ...@@ -427,68 +499,58 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
if (code) goto _err; if (code) goto _err;
pCommitter->dReader.iBlockIdx = 0; pCommitter->dReader.iBlockIdx = 0;
if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) {
pCommitter->dReader.pBlockIdx = pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0);
(SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx);
code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
if (code) goto _err; if (code) goto _err;
} else { } else {
pCommitter->dReader.pBlockIdx = NULL; pCommitter->dReader.pBlockIdx = NULL;
} }
tBlockDataReset(&pCommitter->dReader.bData); tBlockDataReset(&pCommitter->dReader.bData);
// last
code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL);
if (code) goto _err;
pCommitter->dReader.iBlockL = -1;
pCommitter->dReader.iRow = -1;
pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo;
tBlockDataReset(&pCommitter->dReader.bDatal);
code = tsdbCommitterNextLastRow(pCommitter);
if (code) goto _err;
} else { } else {
pCommitter->dReader.pBlockIdx = NULL; pCommitter->dReader.pBlockIdx = NULL;
pCommitter->dReader.pRowInfo = NULL;
} }
// Writer // Writer
SHeadFile fHead; SHeadFile fHead = {.commitID = pCommitter->commitID};
SDataFile fData; SDataFile fData = {.commitID = pCommitter->commitID};
SLastFile fLast; SSmaFile fSma = {.commitID = pCommitter->commitID};
SSmaFile fSma; SSstFile fSst = {.commitID = pCommitter->commitID};
SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma}; SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
if (pRSet) { if (pRSet) {
wSet.diskId = pRSet->diskId; ASSERT(pRSet->nSstF <= pCommitter->maxLast);
wSet.fid = pCommitter->commitFid;
fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0};
fData = *pRSet->pDataF; fData = *pRSet->pDataF;
fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0};
fSma = *pRSet->pSmaF; fSma = *pRSet->pSmaF;
wSet.diskId = pRSet->diskId;
if (pRSet->nSstF < pCommitter->maxLast) {
for (int32_t iSst = 0; iSst < pRSet->nSstF; iSst++) {
wSet.aSstF[iSst] = pRSet->aSstF[iSst];
}
wSet.nSstF = pRSet->nSstF + 1;
} else {
wSet.nSstF = 1;
}
} else { } else {
SDiskID did = {0}; SDiskID did = {0};
tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did);
tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did);
wSet.diskId = did; wSet.diskId = did;
wSet.fid = pCommitter->commitFid; wSet.nSstF = 1;
fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0};
fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0};
fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0};
fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0};
} }
wSet.aSstF[wSet.nSstF - 1] = &fSst;
code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet);
if (code) goto _err; if (code) goto _err;
taosArrayClear(pCommitter->dWriter.aBlockIdx); taosArrayClear(pCommitter->dWriter.aBlockIdx);
taosArrayClear(pCommitter->dWriter.aBlockL); taosArrayClear(pCommitter->dWriter.aSstBlk);
tMapDataReset(&pCommitter->dWriter.mBlock); tMapDataReset(&pCommitter->dWriter.mBlock);
tBlockDataReset(&pCommitter->dWriter.bData); tBlockDataReset(&pCommitter->dWriter.bData);
tBlockDataReset(&pCommitter->dWriter.bDatal); tBlockDataReset(&pCommitter->dWriter.bDatal);
// open iter
code = tsdbOpenCommitIter(pCommitter);
if (code) goto _err;
_exit: _exit:
return code; return code;
...@@ -497,18 +559,14 @@ _err: ...@@ -497,18 +559,14 @@ _err:
return code; return code;
} }
static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
SBlockData *pBlockData = &pCommitter->dWriter.bData; SBlockData *pBlockData = &pCommitter->dWriter.bData;
SBlock block; SDataBlk block;
ASSERT(pBlockData->nRow > 0); ASSERT(pBlockData->nRow > 0);
if (pBlock) { tDataBlkReset(&block);
block = *pBlock; // as a subblock
} else {
tBlockReset(&block); // as a new block
}
// info // info
block.nRow += pBlockData->nRow; block.nRow += pBlockData->nRow;
...@@ -539,8 +597,8 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { ...@@ -539,8 +597,8 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) {
((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0); ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0);
if (code) goto _err; if (code) goto _err;
// put SBlock // put SDataBlk
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
// clear // clear
...@@ -555,7 +613,7 @@ _err: ...@@ -555,7 +613,7 @@ _err:
static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
SBlockL blockL; SSstBlk blockL;
SBlockData *pBlockData = &pCommitter->dWriter.bDatal; SBlockData *pBlockData = &pCommitter->dWriter.bDatal;
ASSERT(pBlockData->nRow > 0); ASSERT(pBlockData->nRow > 0);
...@@ -580,8 +638,8 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { ...@@ -580,8 +638,8 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1); code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1);
if (code) goto _err; if (code) goto _err;
// push SBlockL // push SSstBlk
if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { if (taosArrayPush(pCommitter->dWriter.aSstBlk, &blockL) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
...@@ -596,929 +654,833 @@ _err: ...@@ -596,929 +654,833 @@ _err:
return code; return code;
} }
static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
STbData *pTbData = pIter->pTbData;
SBlockData *pBlockDataR = &pCommitter->dReader.bData;
SBlockData *pBlockDataW = &pCommitter->dWriter.bData;
code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR); // write aBlockIdx
code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx);
if (code) goto _err; if (code) goto _err;
tBlockDataClear(pBlockDataW); // write aSstBlk
int32_t iRow = 0; code = tsdbWriteSstBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSstBlk);
TSDBROW row;
TSDBROW *pRow1 = tsdbTbDataIterGet(pIter);
TSDBROW *pRow2 = &row;
*pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow);
while (pRow1 && pRow2) {
int32_t c = tsdbRowCmprFn(pRow1, pRow2);
if (c < 0) {
code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow1));
if (code) goto _err; if (code) goto _err;
code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema, pTbData->uid); // update file header
code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter);
if (code) goto _err; if (code) goto _err;
// next // upsert SDFileSet
tsdbTbDataIterNext(pIter); code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet);
pRow1 = tsdbTbDataIterGet(pIter);
} else if (c > 0) {
code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid);
if (code) goto _err; if (code) goto _err;
iRow++; // close and sync
if (iRow < pBlockDataR->nRow) { code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1);
*pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); if (code) goto _err;
} else {
pRow2 = NULL;
}
} else {
ASSERT(0);
}
// check if (pCommitter->dReader.pReader) {
if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { code = tsdbDataFReaderClose(&pCommitter->dReader.pReader);
code = tsdbCommitDataBlock(pCommitter, NULL);
if (code) goto _err; if (code) goto _err;
} }
}
while (pRow2) { _exit:
code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); return code;
if (code) goto _err;
iRow++; _err:
if (iRow < pBlockDataR->nRow) { tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
*pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); return code;
} else { }
pRow2 = NULL;
}
// check static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) {
if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { int32_t code = 0;
code = tsdbCommitDataBlock(pCommitter, NULL);
while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) {
SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx;
code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
if (code) goto _err; if (code) goto _err;
}
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
} }
// check code = tsdbCommitterNextTableData(pCommitter);
if (pBlockDataW->nRow > 0) {
code = tsdbCommitDataBlock(pCommitter, NULL);
if (code) goto _err; if (code) goto _err;
} }
return code; return code;
_err: _err:
tsdbError("vgId:%d, tsdb merge commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey) { static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter);
static int32_t tsdbCommitFileData(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
STbData *pTbData = pIter->pTbData; STsdb *pTsdb = pCommitter->pTsdb;
SBlockData *pBlockData = &pCommitter->dWriter.bData; SMemTable *pMemTable = pTsdb->imem;
tBlockDataClear(pBlockData);
TSDBROW *pRow = tsdbTbDataIterGet(pIter);
while (true) {
if (pRow == NULL) {
if (pBlockData->nRow > 0) {
goto _write_block;
} else {
break;
}
}
// update schema // commit file data start
code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); code = tsdbCommitFileDataStart(pCommitter);
if (code) goto _err; if (code) goto _err;
// append // impl
code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); code = tsdbCommitFileDataImpl(pCommitter);
if (code) goto _err; if (code) goto _err;
tsdbTbDataIterNext(pIter); // commit file data end
pRow = tsdbTbDataIterGet(pIter); code = tsdbCommitFileDataEnd(pCommitter);
if (pRow) {
TSDBKEY rowKey = TSDBROW_KEY(pRow);
if (tsdbKeyCmprFn(&rowKey, &toKey) >= 0) {
pRow = NULL;
}
}
if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) {
_write_block:
code = tsdbCommitDataBlock(pCommitter, NULL);
if (code) goto _err; if (code) goto _err;
}
}
return code; return code;
_err: _err:
tsdbError("vgId:%d, tsdb commit table mem data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
tsdbDataFReaderClose(&pCommitter->dReader.pReader);
tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0);
return code; return code;
} }
static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { // ----------------------------------------------------------------------------
int32_t nRow = 0; static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
STbDataIter iter = *pIter;
while (true) {
TSDBROW *pRow = tsdbTbDataIterGet(&iter);
if (pRow == NULL) break;
int32_t c = tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &key);
if (c < 0) {
nRow++;
tsdbTbDataIterNext(&iter);
} else if (c > 0) {
break;
} else {
ASSERT(0);
}
}
return nRow;
}
static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) {
int32_t code = 0; int32_t code = 0;
STbData *pTbData = pIter->pTbData;
SBlockData *pBlockData = &pCommitter->dWriter.bData;
tBlockDataClear(pBlockData); memset(pCommitter, 0, sizeof(*pCommitter));
TSDBROW *pRow = tsdbTbDataIterGet(pIter); ASSERT(pTsdb->mem && pTsdb->imem == NULL);
while (true) {
if (pRow == NULL) break;
code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow));
if (code) goto _err;
code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); taosThreadRwlockWrlock(&pTsdb->rwLock);
if (code) goto _err; pTsdb->imem = pTsdb->mem;
pTsdb->mem = NULL;
taosThreadRwlockUnlock(&pTsdb->rwLock);
tsdbTbDataIterNext(pIter); pCommitter->pTsdb = pTsdb;
pRow = tsdbTbDataIterGet(pIter); pCommitter->commitID = pTsdb->pVnode->state.commitID;
if (pRow) { pCommitter->minutes = pTsdb->keepCfg.days;
TSDBKEY rowKey = TSDBROW_KEY(pRow); pCommitter->precision = pTsdb->keepCfg.precision;
if (tsdbKeyCmprFn(&rowKey, &pBlock->maxKey) > 0) { pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
pRow = NULL; pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
} pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
} pCommitter->maxLast = TSDB_DEFAULT_LAST_FILE; // TODO: make it as a config
pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
if (pCommitter->aTbDataP == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
} }
code = tsdbFSCopy(pTsdb, &pCommitter->fs);
ASSERT(pBlockData->nRow > 0 && pBlock->nRow + pBlockData->nRow <= pCommitter->maxRow);
code = tsdbCommitDataBlock(pCommitter, pBlock);
if (code) goto _err; if (code) goto _err;
return code; return code;
_err: _err:
tsdbError("vgId:%d, tsdb merge as subblock failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
STbData *pTbData = pIter->pTbData;
int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN});
if (pCommitter->dReader.pRowInfo && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pRowInfo) == 0) { // reader
if (pCommitter->dReader.pRowInfo->suid) { // super table pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { if (pCommitter->dReader.aBlockIdx == NULL) {
if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; code = TSDB_CODE_OUT_OF_MEMORY;
nRow++; goto _exit;
}
} else { // normal table
ASSERT(pCommitter->dReader.iRow == 0);
nRow += pCommitter->dReader.bDatal.nRow;
}
} }
if (nRow == 0) goto _exit; code = tBlockDataCreate(&pCommitter->dReader.bData);
if (code) goto _exit;
TSDBROW *pRow = tsdbTbDataIterGet(pIter); // merger
if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) {
pRow = NULL; SDataIter *pIter = &pCommitter->aDataIter[iSst];
pIter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if (pIter->aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; code = tBlockDataCreate(&pIter->bData);
if (pRowInfo && pRowInfo->uid != pTbData->uid) { if (code) goto _exit;
pRowInfo = NULL;
} }
while (nRow) { // writer
SBlockData *pBlockData; pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
int8_t toData; if (pCommitter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
if (nRow < pCommitter->minRow) { // to .last goto _exit;
toData = 0;
pBlockData = &pCommitter->dWriter.bDatal;
// commit and reset block data schema if need
// QUESTION: Is there a case that pBlockData->nRow == 0 but need to change schema ?
if (pBlockData->suid || pBlockData->uid) {
if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) {
if (pBlockData->nRow > 0) {
code = tsdbCommitLastBlock(pCommitter);
if (code) goto _err;
} }
tBlockDataReset(pBlockData); pCommitter->dWriter.aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
} if (pCommitter->dWriter.aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
// set block data schema if need code = tBlockDataCreate(&pCommitter->dWriter.bData);
if (pBlockData->suid == 0 && pBlockData->uid == 0) { if (code) goto _exit;
code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid);
if (code) goto _err;
code =
tBlockDataInit(pBlockData, pTbData->suid, pTbData->suid ? 0 : pTbData->uid, pCommitter->skmTable.pTSchema);
if (code) goto _err;
}
if (pBlockData->nRow + nRow > pCommitter->maxRow) { code = tBlockDataCreate(&pCommitter->dWriter.bDatal);
code = tsdbCommitLastBlock(pCommitter); if (code) goto _exit;
if (code) goto _err;
}
} else { // to .data
toData = 1;
pBlockData = &pCommitter->dWriter.bData;
ASSERT(pBlockData->nRow == 0);
}
while (pRow && pRowInfo) { _exit:
int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); return code;
if (c < 0) { }
code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow));
if (code) goto _err;
code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); static void tsdbCommitDataEnd(SCommitter *pCommitter) {
if (code) goto _err; // reader
taosArrayDestroy(pCommitter->dReader.aBlockIdx);
tMapDataClear(&pCommitter->dReader.mBlock);
tBlockDataDestroy(&pCommitter->dReader.bData, 1);
tsdbTbDataIterNext(pIter); // merger
pRow = tsdbTbDataIterGet(pIter); for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) {
if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { SDataIter *pIter = &pCommitter->aDataIter[iSst];
pRow = NULL; taosArrayDestroy(pIter->aSstBlk);
tBlockDataDestroy(&pIter->bData, 1);
} }
} else if (c > 0) {
code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid);
if (code) goto _err;
code = tsdbCommitterNextLastRow(pCommitter); // writer
if (code) goto _err; taosArrayDestroy(pCommitter->dWriter.aBlockIdx);
taosArrayDestroy(pCommitter->dWriter.aSstBlk);
tMapDataClear(&pCommitter->dWriter.mBlock);
tBlockDataDestroy(&pCommitter->dWriter.bData, 1);
tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
tTSchemaDestroy(pCommitter->skmTable.pTSchema);
tTSchemaDestroy(pCommitter->skmRow.pTSchema);
}
pRowInfo = pCommitter->dReader.pRowInfo; static int32_t tsdbCommitData(SCommitter *pCommitter) {
if (pRowInfo && pRowInfo->uid != pTbData->uid) { int32_t code = 0;
pRowInfo = NULL; STsdb *pTsdb = pCommitter->pTsdb;
} SMemTable *pMemTable = pTsdb->imem;
} else {
ASSERT(0);
}
nRow--; // check
if (toData) { if (pMemTable->nRow == 0) goto _exit;
if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) {
code = tsdbCommitDataBlock(pCommitter, NULL);
if (code) goto _err;
goto _outer_break;
}
}
}
while (pRow) { // start ====================
code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); code = tsdbCommitDataStart(pCommitter);
if (code) goto _err; if (code) goto _err;
code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); // impl ====================
pCommitter->nextKey = pMemTable->minKey;
while (pCommitter->nextKey < TSKEY_MAX) {
code = tsdbCommitFileData(pCommitter);
if (code) goto _err; if (code) goto _err;
}
tsdbTbDataIterNext(pIter); // end ====================
pRow = tsdbTbDataIterGet(pIter); tsdbCommitDataEnd(pCommitter);
if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
pRow = NULL; _exit:
tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow);
return code;
_err:
tsdbCommitDataEnd(pCommitter);
tsdbError("vgId:%d, commit data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
static int32_t tsdbCommitDel(SCommitter *pCommitter) {
int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
if (pMemTable->nDel == 0) {
goto _exit;
} }
nRow--; // start
if (toData) { code = tsdbCommitDelStart(pCommitter);
if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { if (code) {
code = tsdbCommitDataBlock(pCommitter, NULL); goto _err;
if (code) goto _err;
goto _outer_break;
} }
// impl
int32_t iDelIdx = 0;
int32_t nDelIdx = taosArrayGetSize(pCommitter->aDelIdx);
int32_t iTbData = 0;
int32_t nTbData = taosArrayGetSize(pCommitter->aTbDataP);
STbData *pTbData;
SDelIdx *pDelIdx;
ASSERT(nTbData > 0);
pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData);
pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL;
while (true) {
if (pTbData == NULL && pDelIdx == NULL) break;
if (pTbData && pDelIdx) {
int32_t c = tTABLEIDCmprFn(pTbData, pDelIdx);
if (c == 0) {
goto _commit_mem_and_disk_del;
} else if (c < 0) {
goto _commit_mem_del;
} else {
goto _commit_disk_del;
} }
} else if (pTbData) {
goto _commit_mem_del;
} else {
goto _commit_disk_del;
} }
while (pRowInfo) { _commit_mem_del:
code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); code = tsdbCommitTableDel(pCommitter, pTbData, NULL);
if (code) goto _err; if (code) goto _err;
code = tsdbCommitterNextLastRow(pCommitter); iTbData++;
pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL;
continue;
_commit_disk_del:
code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx);
if (code) goto _err; if (code) goto _err;
pRowInfo = pCommitter->dReader.pRowInfo; iDelIdx++;
if (pRowInfo && pRowInfo->uid != pTbData->uid) { pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL;
pRowInfo = NULL; continue;
}
nRow--; _commit_mem_and_disk_del:
if (toData) { code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx);
if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) {
code = tsdbCommitDataBlock(pCommitter, NULL);
if (code) goto _err; if (code) goto _err;
goto _outer_break;
} iTbData++;
} pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL;
iDelIdx++;
pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL;
continue;
} }
_outer_break: // end
ASSERT(nRow >= 0); code = tsdbCommitDelEnd(pCommitter);
if (code) {
goto _err;
} }
_exit: _exit:
tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel);
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb merge commit last failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d, commit del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); ASSERT(eno == 0);
ASSERT(pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, pTbData) >= 0);
// merge commit table data
STbDataIter iter = {0};
STbDataIter *pIter = &iter;
TSDBROW *pRow;
tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter);
pRow = tsdbTbDataIterGet(pIter);
if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
pRow = NULL;
}
if (pRow == NULL) { code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) == 0) {
SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid};
code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
if (code) goto _err; if (code) goto _err;
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { // lock
code = TSDB_CODE_OUT_OF_MEMORY; taosThreadRwlockWrlock(&pTsdb->rwLock);
// commit or rollback
code = tsdbFSCommit2(pTsdb, &pCommitter->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _err; goto _err;
} }
}
goto _exit; pTsdb->imem = NULL;
}
int32_t iBlock = 0; // unlock
SBlock block; taosThreadRwlockUnlock(&pTsdb->rwLock);
SBlock *pBlock = &block;
if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) {
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock);
} else {
pBlock = NULL;
}
code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid); tsdbUnrefMemTable(pMemTable);
if (code) goto _err; tsdbFSDestroy(&pCommitter->fs);
taosArrayDestroy(pCommitter->aTbDataP);
tMapDataReset(&pCommitter->dWriter.mBlock); // if (pCommitter->toMerge) {
code = tBlockDataInit(&pCommitter->dReader.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); // code = tsdbMerge(pTsdb);
if (code) goto _err; // if (code) goto _err;
code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); // }
if (code) goto _err;
// .data merge tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode));
while (pBlock && pRow) { return code;
int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)});
if (c < 0) { // disk
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock);
if (code) goto _err;
// next _err:
iBlock++; tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
if (iBlock < pCommitter->dReader.mBlock.nItem) { return code;
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); }
} else {
pBlock = NULL; // ================================================================================
}
} else if (c > 0) { // memory static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) {
code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey); return (pCommitter->pIter) ? &pCommitter->pIter->r : NULL;
if (code) goto _err; }
static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
int32_t code = 0;
// next if (pCommitter->pIter) {
pRow = tsdbTbDataIterGet(pIter); SDataIter *pIter = pCommitter->pIter;
if (pCommitter->pIter->type == MEMORY_DATA_ITER) { // memory
tsdbTbDataIterNext(&pIter->iter);
TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter);
while (true) {
if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow));
pRow = NULL; pRow = NULL;
} }
} else { // merge
int32_t nOvlp = tsdbGetNumOfRowsLessThan(pIter, pBlock->maxKey);
ASSERT(nOvlp > 0); if (pRow) {
pIter->r.suid = pIter->iter.pTbData->suid;
pIter->r.uid = pIter->iter.pTbData->uid;
pIter->r.row = *pRow;
break;
}
if (pBlock->nRow + nOvlp <= pCommitter->maxRow && pBlock->nSubBlock < TSDB_MAX_SUBBLOCKS) { pIter->iTbDataP++;
code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); if (pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)) {
if (code) goto _err; STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP);
TSDBKEY keyFrom = {.ts = pCommitter->minKey, .version = VERSION_MIN};
tsdbTbDataIterOpen(pTbData, &keyFrom, 0, &pIter->iter);
pRow = tsdbTbDataIterGet(&pIter->iter);
continue;
} else { } else {
code = tsdbMergeCommitData(pCommitter, pIter, pBlock); pCommitter->pIter = NULL;
if (code) goto _err; break;
} }
// next
pRow = tsdbTbDataIterGet(pIter);
if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) {
pRow = NULL;
} }
iBlock++; } else if (pCommitter->pIter->type == LAST_DATA_ITER) { // last file
if (iBlock < pCommitter->dReader.mBlock.nItem) { pIter->iRow++;
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); if (pIter->iRow < pIter->bData.nRow) {
pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow];
pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
} else {
pIter->iSstBlk++;
if (pIter->iSstBlk < taosArrayGetSize(pIter->aSstBlk)) {
SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk);
code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData);
if (code) goto _exit;
pIter->iRow = 0;
pIter->r.suid = pIter->bData.suid;
pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0];
pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0);
} else { } else {
pBlock = NULL; pCommitter->pIter = NULL;
} }
} }
} else {
ASSERT(0);
} }
while (pBlock) { // compare with min in RB Tree
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt);
if (code) goto _err; if (pCommitter->pIter && pIter) {
int32_t c = tRowInfoCmprFn(&pCommitter->pIter->r, &pIter->r);
// next if (c > 0) {
iBlock++; tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter);
if (iBlock < pCommitter->dReader.mBlock.nItem) { pCommitter->pIter = NULL;
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock);
} else { } else {
pBlock = NULL; ASSERT(c);
}
} }
} }
// .data append and .last merge if (pCommitter->pIter == NULL) {
code = tsdbMergeCommitLast(pCommitter, pIter); pCommitter->pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt);
if (code) goto _err; if (pCommitter->pIter) {
tRBTreeDrop(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter);
// end
if (pCommitter->dWriter.mBlock.nItem > 0) {
SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid};
code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx);
if (code) goto _err;
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
} }
} }
_exit: _exit:
pRow = tsdbTbDataIterGet(pIter);
if (pRow) {
pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow));
}
return code;
_err:
tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
int32_t code = 0; int32_t code = 0;
SBlockData *pBlockData = &pCommitter->dWriter.bData;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
// write aBlockIdx tBlockDataClear(pBlockData);
code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); while (pRowInfo) {
ASSERT(pRowInfo->row.type == 0);
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
if (code) goto _err; if (code) goto _err;
// write aBlockL code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid);
code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL);
if (code) goto _err; if (code) goto _err;
// update file header code = tsdbNextCommitRow(pCommitter);
code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter);
if (code) goto _err; if (code) goto _err;
// upsert SDFileSet pRowInfo = tsdbGetCommitRow(pCommitter);
code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); if (pRowInfo) {
if (code) goto _err; if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) {
pRowInfo = NULL;
} else {
TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row);
if (tsdbKeyCmprFn(&tKey, &pDataBlk->minKey) >= 0) pRowInfo = NULL;
}
}
// close and sync if (pBlockData->nRow >= pCommitter->maxRow) {
code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); code = tsdbCommitDataBlock(pCommitter);
if (code) goto _err; if (code) goto _err;
}
}
if (pCommitter->dReader.pReader) { if (pBlockData->nRow) {
code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); code = tsdbCommitDataBlock(pCommitter);
if (code) goto _err; if (code) goto _err;
} }
_exit:
return code; return code;
_err: _err:
tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
int32_t code = 0; int32_t code = 0;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
SBlockData *pBDataR = &pCommitter->dReader.bData;
SBlockData *pBDataW = &pCommitter->dWriter.bData;
// .data code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR);
while (true) {
if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break;
SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx;
code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
if (code) goto _err; if (code) goto _err;
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { tBlockDataClear(pBDataW);
code = TSDB_CODE_OUT_OF_MEMORY; int32_t iRow = 0;
goto _err; TSDBROW row = tsdbRowFromBlockData(pBDataR, 0);
} TSDBROW *pRow = &row;
code = tsdbCommitterNextTableData(pCommitter); while (pRow && pRowInfo) {
int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row);
if (c < 0) {
code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid);
if (code) goto _err; if (code) goto _err;
}
// .last iRow++;
while (true) { if (iRow < pBDataR->nRow) {
if (pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, &toTable) >= 0) break; row = tsdbRowFromBlockData(pBDataR, iRow);
} else {
SBlockData *pBlockDataR = &pCommitter->dReader.bDatal; pRow = NULL;
SBlockData *pBlockDataW = &pCommitter->dWriter.bDatal; }
tb_uid_t suid = pCommitter->dReader.pRowInfo->suid; } else if (c > 0) {
tb_uid_t uid = pCommitter->dReader.pRowInfo->uid; ASSERT(pRowInfo->row.type == 0);
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
if (code) goto _err;
ASSERT((pBlockDataR->suid && !pBlockDataR->uid) || (!pBlockDataR->suid && pBlockDataR->uid)); code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid);
ASSERT(pBlockDataR->nRow > 0); if (code) goto _err;
// commit and reset block data schema if need code = tsdbNextCommitRow(pCommitter);
if (pBlockDataW->suid || pBlockDataW->uid) {
if (pBlockDataW->suid != suid || pBlockDataW->suid == 0) {
if (pBlockDataW->nRow > 0) {
code = tsdbCommitLastBlock(pCommitter);
if (code) goto _err; if (code) goto _err;
pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo) {
if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) {
pRowInfo = NULL;
} else {
TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row);
if (tsdbKeyCmprFn(&tKey, &pDataBlk->maxKey) > 0) pRowInfo = NULL;
} }
tBlockDataReset(pBlockDataW);
} }
} else {
ASSERT(0);
} }
// set block data schema if need if (pBDataW->nRow >= pCommitter->maxRow) {
if (pBlockDataW->suid == 0 && pBlockDataW->uid == 0) { code = tsdbCommitDataBlock(pCommitter);
code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid);
if (code) goto _err; if (code) goto _err;
}
}
code = tBlockDataInit(pBlockDataW, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); while (pRow) {
code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid);
if (code) goto _err; if (code) goto _err;
}
// check if it can make sure that one table data in one block
int32_t nRow = 0;
if (pBlockDataR->suid) {
int32_t iRow = pCommitter->dReader.iRow;
while ((iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid)) {
nRow++;
iRow++; iRow++;
} if (iRow < pBDataR->nRow) {
row = tsdbRowFromBlockData(pBDataR, iRow);
} else { } else {
ASSERT(pCommitter->dReader.iRow == 0); pRow = NULL;
nRow = pBlockDataR->nRow;
} }
ASSERT(nRow > 0 && nRow < pCommitter->minRow); if (pBDataW->nRow >= pCommitter->maxRow) {
code = tsdbCommitDataBlock(pCommitter);
if (pBlockDataW->nRow + nRow > pCommitter->maxRow) {
ASSERT(pBlockDataW->nRow > 0);
code = tsdbCommitLastBlock(pCommitter);
if (code) goto _err; if (code) goto _err;
} }
}
while (nRow > 0) { if (pBDataW->nRow) {
code = tBlockDataAppendRow(pBlockDataW, &pCommitter->dReader.pRowInfo->row, NULL, uid); code = tsdbCommitDataBlock(pCommitter);
if (code) goto _err;
code = tsdbCommitterNextLastRow(pCommitter);
if (code) goto _err; if (code) goto _err;
nRow--;
}
} }
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbCommitFileData(SCommitter *pCommitter) { static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb; SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx;
SMemTable *pMemTable = pTsdb->imem;
// commit file data start ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0);
code = tsdbCommitFileDataStart(pCommitter); if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) {
if (code) goto _err; int32_t iBlock = 0;
SDataBlk block;
SDataBlk *pDataBlk = &block;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
// commit file data impl ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid);
for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) {
STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData);
// move commit until current (suid, uid) tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
code = tsdbMoveCommitData(pCommitter, *(TABLEID *)pTbData); while (pDataBlk && pRowInfo) {
if (code) goto _err; SDataBlk tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)};
int32_t c = tDataBlkCmprFn(pDataBlk, &tBlock);
// commit current table data if (c < 0) {
code = tsdbCommitTableData(pCommitter, pTbData); code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
// move next reader table data if need iBlock++;
if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { if (iBlock < pCommitter->dReader.mBlock.nItem) {
code = tsdbCommitterNextTableData(pCommitter); tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
if (code) goto _err; } else {
} pDataBlk = NULL;
} }
} else if (c > 0) {
code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); code = tsdbCommitAheadBlock(pCommitter, pDataBlk);
if (code) goto _err; if (code) goto _err;
if (pCommitter->dWriter.bDatal.nRow > 0) { pRowInfo = tsdbGetCommitRow(pCommitter);
code = tsdbCommitLastBlock(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL;
} else {
code = tsdbCommitMergeBlock(pCommitter, pDataBlk);
if (code) goto _err; if (code) goto _err;
iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) {
tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
} else {
pDataBlk = NULL;
}
pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL;
}
} }
// commit file data end while (pDataBlk) {
code = tsdbCommitFileDataEnd(pCommitter); code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
return code; iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) {
_err: tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk);
tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); } else {
tsdbDataFReaderClose(&pCommitter->dReader.pReader); pDataBlk = NULL;
tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); }
return code;
}
// ----------------------------------------------------------------------------
static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
int32_t code = 0;
memset(pCommitter, 0, sizeof(*pCommitter));
ASSERT(pTsdb->mem && pTsdb->imem == NULL);
taosThreadRwlockWrlock(&pTsdb->rwLock);
pTsdb->imem = pTsdb->mem;
pTsdb->mem = NULL;
taosThreadRwlockUnlock(&pTsdb->rwLock);
pCommitter->pTsdb = pTsdb;
pCommitter->commitID = pTsdb->pVnode->state.commitID;
pCommitter->minutes = pTsdb->keepCfg.days;
pCommitter->precision = pTsdb->keepCfg.precision;
pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
if (pCommitter->aTbDataP == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
} }
code = tsdbFSCopy(pTsdb, &pCommitter->fs); code = tsdbCommitterNextTableData(pCommitter);
if (code) goto _err; if (code) goto _err;
}
_exit:
return code; return code;
_err: _err:
tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) {
int32_t code = 0; int32_t code = 0;
// Reader SBlockData *pBDatal = &pCommitter->dWriter.bDatal;
pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pBDatal->suid || pBDatal->uid) {
if (pCommitter->dReader.aBlockIdx == NULL) { if ((pBDatal->suid != id.suid) || (id.suid == 0)) {
code = TSDB_CODE_OUT_OF_MEMORY; if (pBDatal->nRow) {
goto _exit; code = tsdbCommitLastBlock(pCommitter);
}
code = tBlockDataCreate(&pCommitter->dReader.bData);
if (code) goto _exit; if (code) goto _exit;
pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL));
if (pCommitter->dReader.aBlockL == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
tBlockDataReset(pBDatal);
code = tBlockDataCreate(&pCommitter->dReader.bDatal);
if (code) goto _exit;
// Writer
pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCommitter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
pCommitter->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL));
if (pCommitter->dWriter.aBlockL == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
code = tBlockDataCreate(&pCommitter->dWriter.bData); if (!pBDatal->suid && !pBDatal->uid) {
if (code) goto _exit; ASSERT(pCommitter->skmTable.suid == id.suid);
ASSERT(pCommitter->skmTable.uid == id.uid);
code = tBlockDataCreate(&pCommitter->dWriter.bDatal); code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema);
if (code) goto _exit; if (code) goto _exit;
}
_exit: _exit:
return code; return code;
} }
static void tsdbCommitDataEnd(SCommitter *pCommitter) { static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) {
// Reader
taosArrayDestroy(pCommitter->dReader.aBlockIdx);
tMapDataClear(&pCommitter->dReader.mBlock);
tBlockDataDestroy(&pCommitter->dReader.bData, 1);
taosArrayDestroy(pCommitter->dReader.aBlockL);
tBlockDataDestroy(&pCommitter->dReader.bDatal, 1);
// Writer
taosArrayDestroy(pCommitter->dWriter.aBlockIdx);
taosArrayDestroy(pCommitter->dWriter.aBlockL);
tMapDataClear(&pCommitter->dWriter.mBlock);
tBlockDataDestroy(&pCommitter->dWriter.bData, 1);
tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
tTSchemaDestroy(pCommitter->skmTable.pTSchema);
tTSchemaDestroy(pCommitter->skmRow.pTSchema);
}
static int32_t tsdbCommitData(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
// check SBlockData *pBData = &pCommitter->dWriter.bData;
if (pMemTable->nRow == 0) goto _exit; SBlockData *pBDatal = &pCommitter->dWriter.bDatal;
// start ==================== TABLEID id = {.suid = pBData->suid, .uid = pBData->uid};
code = tsdbCommitDataStart(pCommitter); code = tsdbInitLastBlockIfNeed(pCommitter, id);
if (code) goto _err; if (code) goto _err;
// impl ==================== for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) {
pCommitter->nextKey = pMemTable->minKey; TSDBROW row = tsdbRowFromBlockData(pBData, iRow);
while (pCommitter->nextKey < TSKEY_MAX) { code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid);
code = tsdbCommitFileData(pCommitter);
if (code) goto _err; if (code) goto _err;
}
// end ==================== if (pBDatal->nRow >= pCommitter->maxRow) {
tsdbCommitDataEnd(pCommitter); code = tsdbCommitLastBlock(pCommitter);
if (code) goto _err;
}
}
_exit:
tsdbDebug("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow);
return code; return code;
_err: _err:
tsdbCommitDataEnd(pCommitter);
tsdbError("vgId:%d, commit data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbCommitDel(SCommitter *pCommitter) { static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
if (pMemTable->nDel == 0) { SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
goto _exit; if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
pRowInfo = NULL;
} }
// start if (pRowInfo == NULL) goto _exit;
code = tsdbCommitDelStart(pCommitter);
if (code) {
goto _err;
}
// impl SBlockData *pBData;
int32_t iDelIdx = 0; if (pCommitter->toLastOnly) {
int32_t nDelIdx = taosArrayGetSize(pCommitter->aDelIdx); pBData = &pCommitter->dWriter.bDatal;
int32_t iTbData = 0; code = tsdbInitLastBlockIfNeed(pCommitter, id);
int32_t nTbData = taosArrayGetSize(pCommitter->aTbDataP); if (code) goto _err;
STbData *pTbData;
SDelIdx *pDelIdx;
ASSERT(nTbData > 0);
pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData);
pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL;
while (true) {
if (pTbData == NULL && pDelIdx == NULL) break;
if (pTbData && pDelIdx) {
int32_t c = tTABLEIDCmprFn(pTbData, pDelIdx);
if (c == 0) {
goto _commit_mem_and_disk_del;
} else if (c < 0) {
goto _commit_mem_del;
} else {
goto _commit_disk_del;
}
} else if (pTbData) {
goto _commit_mem_del;
} else { } else {
goto _commit_disk_del; pBData = &pCommitter->dWriter.bData;
ASSERT(pBData->nRow == 0);
} }
_commit_mem_del: while (pRowInfo) {
code = tsdbCommitTableDel(pCommitter, pTbData, NULL); STSchema *pTSchema = NULL;
if (pRowInfo->row.type == 0) {
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
if (code) goto _err; if (code) goto _err;
pTSchema = pCommitter->skmRow.pTSchema;
}
iTbData++; code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid);
pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; if (code) goto _err;
continue;
_commit_disk_del: code = tsdbNextCommitRow(pCommitter);
code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx);
if (code) goto _err; if (code) goto _err;
iDelIdx++; pRowInfo = tsdbGetCommitRow(pCommitter);
pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
continue; pRowInfo = NULL;
}
_commit_mem_and_disk_del: if (pBData->nRow >= pCommitter->maxRow) {
code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx); if (pCommitter->toLastOnly) {
code = tsdbCommitLastBlock(pCommitter);
if (code) goto _err;
} else {
code = tsdbCommitDataBlock(pCommitter);
if (code) goto _err; if (code) goto _err;
}
iTbData++; }
pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL;
iDelIdx++;
pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL;
continue;
} }
// end if (!pCommitter->toLastOnly && pBData->nRow) {
code = tsdbCommitDelEnd(pCommitter); if (pBData->nRow > pCommitter->minRow) {
if (code) { code = tsdbCommitDataBlock(pCommitter);
goto _err; if (code) goto _err;
} else {
code = tsdbAppendLastBlock(pCommitter);
if (code) goto _err;
}
} }
_exit: _exit:
tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel);
return code; return code;
_err: _err:
tsdbError("vgId:%d, commit del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
ASSERT(eno == 0); SRowInfo *pRowInfo;
TABLEID id = {0};
while ((pRowInfo = tsdbGetCommitRow(pCommitter)) != NULL) {
ASSERT(pRowInfo->suid != id.suid || pRowInfo->uid != id.uid);
id.suid = pRowInfo->suid;
id.uid = pRowInfo->uid;
code = tsdbFSCommit1(pTsdb, &pCommitter->fs); code = tsdbMoveCommitData(pCommitter, id);
if (code) goto _err; if (code) goto _err;
// lock // start
taosThreadRwlockWrlock(&pTsdb->rwLock); tMapDataReset(&pCommitter->dWriter.mBlock);
// commit or rollback // impl
code = tsdbFSCommit2(pTsdb, &pCommitter->fs); code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid);
if (code) { if (code) goto _err;
taosThreadRwlockUnlock(&pTsdb->rwLock); code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema);
if (code) goto _err;
code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema);
if (code) goto _err;
/* merge with data in .data file */
code = tsdbMergeTableData(pCommitter, id);
if (code) goto _err;
/* handle remain table data */
code = tsdbCommitTableData(pCommitter, id);
if (code) goto _err;
// end
if (pCommitter->dWriter.mBlock.nItem > 0) {
SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid};
code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx);
if (code) goto _err;
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
}
}
pTsdb->imem = NULL; id.suid = INT64_MAX;
id.uid = INT64_MAX;
// unlock code = tsdbMoveCommitData(pCommitter, id);
taosThreadRwlockUnlock(&pTsdb->rwLock); if (code) goto _err;
tsdbUnrefMemTable(pMemTable); if (pCommitter->dWriter.bDatal.nRow > 0) {
tsdbFSDestroy(&pCommitter->fs); code = tsdbCommitLastBlock(pCommitter);
taosArrayDestroy(pCommitter->aTbDataP); if (code) goto _err;
}
tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode));
return code; return code;
_err: _err:
tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
source/dnode/vnode/src/tsdb/tsdbCompact.c 0 → 100644
浏览文件 @ 48593015
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
typedef struct {
STsdb *pTsdb;
STsdbFS fs;
} STsdbCompactor;
int32_t tsdbCompact(STsdb *pTsdb) {
int32_t code = 0;
// TODO
return code;
}
source/dnode/vnode/src/tsdb/tsdbCompress.c 0 → 100644
浏览文件 @ 48593015
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
// Integer =====================================================
typedef struct {
int8_t rawCopy;
int64_t prevVal;
int32_t nVal;
int32_t nBuf;
uint8_t *pBuf;
} SIntCompressor;
#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
static int32_t tsdbCmprI64(SIntCompressor *pCompressor, int64_t val) {
int32_t code = 0;
// raw copy
if (pCompressor->rawCopy) {
memcpy(pCompressor->pBuf + pCompressor->nBuf, &val, sizeof(val));
pCompressor->nBuf += sizeof(val);
pCompressor->nVal++;
goto _exit;
}
if (!I64_SAFE_ADD(val, pCompressor->prevVal)) {
pCompressor->rawCopy = 1;
// TODO: decompress and copy
pCompressor->nVal++;
goto _exit;
}
int64_t diff = val - pCompressor->prevVal;
uint8_t zigzag = ZIGZAGE(int64_t, diff);
if (zigzag >= SIMPLE8B_MAX) {
pCompressor->rawCopy = 1;
// TODO: decompress and copy
pCompressor->nVal++;
goto _exit;
}
_exit:
return code;
}
// Timestamp =====================================================
// Float =====================================================
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbDiskData.c 0 → 100644
浏览文件 @ 48593015
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
typedef struct SDiskColBuilder SDiskColBuilder;
struct SDiskColBuilder {
uint8_t flags;
uint8_t *pBitMap;
int32_t *aOffset;
int32_t nData;
uint8_t *pData;
};
int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
// TODO
return code;
}
// ================================================================
typedef struct SDiskDataBuilder SDiskDataBuilder;
struct SDiskDataBuilder {
SDiskDataHdr hdr;
SArray *aBlockCol; // SArray<SBlockCol>
};
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) {
int32_t code = 0;
// TODO
return code;
}
void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
// TODO
}
void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) {
pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, //
.fmtVer = 0,
.suid = suid,
.uid = uid,
.cmprAlg = cmprAlg};
}
void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) {
// TODO
}
int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) {
int32_t code = 0;
// uid (todo)
// version (todo)
// TSKEY (todo)
SRowIter iter = {0};
tRowIterInit(&iter, pRow, pTSchema);
for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) {
}
return code;
}
int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) {
int32_t code = 0;
// TODO
return code;
}
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbFS.c
浏览文件 @ 48593015
...@@ -110,7 +110,7 @@ _err: ...@@ -110,7 +110,7 @@ _err:
// taosRemoveFile(fname); // taosRemoveFile(fname);
// } // }
// // last // // sst
// if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) { // if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) {
// if (pFrom->pLastF->size > pTo->pLastF->size) { // if (pFrom->pLastF->size > pTo->pLastF->size) {
// code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE); // code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE);
...@@ -140,7 +140,7 @@ _err: ...@@ -140,7 +140,7 @@ _err:
// tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname); // tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname);
// taosRemoveFile(fname); // taosRemoveFile(fname);
// // last // // sst
// tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname); // tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname);
// taosRemoveFile(fname); // taosRemoveFile(fname);
...@@ -254,8 +254,10 @@ void tsdbFSDestroy(STsdbFS *pFS) { ...@@ -254,8 +254,10 @@ void tsdbFSDestroy(STsdbFS *pFS) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet); SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet);
taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pDataF);
taosMemoryFree(pSet->pLastF);
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->pSmaF);
for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
taosMemoryFree(pSet->aSstF[iSst]);
}
} }
taosArrayDestroy(pFS->aDFileSet); taosArrayDestroy(pFS->aDFileSet);
...@@ -309,29 +311,31 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { ...@@ -309,29 +311,31 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
if (code) goto _err; if (code) goto _err;
} }
// last =========== // sma =============
tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname); tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (size != pSet->pLastF->size) { if (size < pSet->pSmaF->size) {
code = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
goto _err; goto _err;
} else if (size > pSet->pSmaF->size) {
code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE);
if (code) goto _err;
} }
// sma ============= // sst ===========
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (size < pSet->pSmaF->size) { if (size != pSet->aSstF[iSst]->size) {
code = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
goto _err; goto _err;
} else if (size > pSet->pSmaF->size) { }
code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE);
if (code) goto _err;
} }
} }
...@@ -382,41 +386,15 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) { ...@@ -382,41 +386,15 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) {
taosArrayClear(pTsdb->fs.aDFileSet); taosArrayClear(pTsdb->fs.aDFileSet);
n += tGetU32v(pData + n, &nSet); n += tGetU32v(pData + n, &nSet);
for (uint32_t iSet = 0; iSet < nSet; iSet++) { for (uint32_t iSet = 0; iSet < nSet; iSet++) {
SDFileSet fSet; SDFileSet fSet = {0};
// head int32_t nt = tGetDFileSet(pData + n, &fSet);
fSet.pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); if (nt < 0) {
if (fSet.pHeadF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
fSet.pHeadF->nRef = 1;
// data n += nt;
fSet.pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile));
if (fSet.pDataF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
fSet.pDataF->nRef = 1;
// last
fSet.pLastF = (SLastFile *)taosMemoryCalloc(1, sizeof(SLastFile));
if (fSet.pLastF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
fSet.pLastF->nRef = 1;
// sma
fSet.pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile));
if (fSet.pSmaF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
fSet.pSmaF->nRef = 1;
n += tGetDFileSet(pData + n, &fSet);
if (taosArrayPush(pTsdb->fs.aDFileSet, &fSet) == NULL) { if (taosArrayPush(pTsdb->fs.aDFileSet, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
...@@ -532,13 +510,15 @@ int32_t tsdbFSClose(STsdb *pTsdb) { ...@@ -532,13 +510,15 @@ int32_t tsdbFSClose(STsdb *pTsdb) {
ASSERT(pSet->pDataF->nRef == 1); ASSERT(pSet->pDataF->nRef == 1);
taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pDataF);
// last
ASSERT(pSet->pLastF->nRef == 1);
taosMemoryFree(pSet->pLastF);
// sma // sma
ASSERT(pSet->pSmaF->nRef == 1); ASSERT(pSet->pSmaF->nRef == 1);
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->pSmaF);
// sst
for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
ASSERT(pSet->aSstF[iSst]->nRef == 1);
taosMemoryFree(pSet->aSstF[iSst]);
}
} }
taosArrayDestroy(pTsdb->fs.aDFileSet); taosArrayDestroy(pTsdb->fs.aDFileSet);
...@@ -586,21 +566,23 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { ...@@ -586,21 +566,23 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
} }
*fSet.pDataF = *pSet->pDataF; *fSet.pDataF = *pSet->pDataF;
// data // sma
fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
if (fSet.pLastF == NULL) { if (fSet.pSmaF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
*fSet.pLastF = *pSet->pLastF; *fSet.pSmaF = *pSet->pSmaF;
// last // sst
fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); for (fSet.nSstF = 0; fSet.nSstF < pSet->nSstF; fSet.nSstF++) {
if (fSet.pSmaF == NULL) { fSet.aSstF[fSet.nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (fSet.aSstF[fSet.nSstF] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
*fSet.pSmaF = *pSet->pSmaF; *fSet.aSstF[fSet.nSstF] = *pSet->aSstF[fSet.nSstF];
}
if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
...@@ -651,14 +633,38 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { ...@@ -651,14 +633,38 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
if (c == 0) { if (c == 0) {
*pDFileSet->pHeadF = *pSet->pHeadF; *pDFileSet->pHeadF = *pSet->pHeadF;
*pDFileSet->pDataF = *pSet->pDataF; *pDFileSet->pDataF = *pSet->pDataF;
*pDFileSet->pLastF = *pSet->pLastF;
*pDFileSet->pSmaF = *pSet->pSmaF; *pDFileSet->pSmaF = *pSet->pSmaF;
// sst
if (pSet->nSstF > pDFileSet->nSstF) {
ASSERT(pSet->nSstF == pDFileSet->nSstF + 1);
pDFileSet->aSstF[pDFileSet->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pDFileSet->aSstF[pDFileSet->nSstF] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
*pDFileSet->aSstF[pDFileSet->nSstF] = *pSet->aSstF[pSet->nSstF - 1];
pDFileSet->nSstF++;
} else if (pSet->nSstF < pDFileSet->nSstF) {
ASSERT(pSet->nSstF == 1);
for (int32_t iSst = 1; iSst < pDFileSet->nSstF; iSst++) {
taosMemoryFree(pDFileSet->aSstF[iSst]);
} }
SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid}; *pDFileSet->aSstF[0] = *pSet->aSstF[0];
pDFileSet->nSstF = 1;
} else {
for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
*pDFileSet->aSstF[iSst] = *pSet->aSstF[iSst];
}
}
goto _exit;
}
}
ASSERT(pSet->nSstF == 1);
SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSstF = 1};
// head // head
fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
...@@ -676,21 +682,21 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { ...@@ -676,21 +682,21 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
} }
*fSet.pDataF = *pSet->pDataF; *fSet.pDataF = *pSet->pDataF;
// data // sma
fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
if (fSet.pLastF == NULL) { if (fSet.pSmaF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
*fSet.pLastF = *pSet->pLastF; *fSet.pSmaF = *pSet->pSmaF;
// last // sst
fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (fSet.pSmaF == NULL) { if (fSet.aSstF[0] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
*fSet.pSmaF = *pSet->pSmaF; *fSet.aSstF[0] = *pSet->aSstF[0];
if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) { if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
...@@ -836,27 +842,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { ...@@ -836,27 +842,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
pSetOld->pDataF->size = pSetNew->pDataF->size; pSetOld->pDataF->size = pSetNew->pDataF->size;
} }
// last
fSet.pLastF = pSetOld->pLastF;
if ((!sameDisk) || (pSetOld->pLastF->commitID != pSetNew->pLastF->commitID)) {
pSetOld->pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
if (pSetOld->pLastF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->pLastF = *pSetNew->pLastF;
pSetOld->pLastF->nRef = 1;
nRef = atomic_sub_fetch_32(&fSet.pLastF->nRef, 1);
if (nRef == 0) {
tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pLastF, fname);
taosRemoveFile(fname);
taosMemoryFree(fSet.pLastF);
}
} else {
ASSERT(pSetOld->pLastF->size == pSetNew->pLastF->size);
}
// sma // sma
fSet.pSmaF = pSetOld->pSmaF; fSet.pSmaF = pSetOld->pSmaF;
if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) { if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) {
...@@ -879,6 +864,84 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { ...@@ -879,6 +864,84 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
pSetOld->pSmaF->size = pSetNew->pSmaF->size; pSetOld->pSmaF->size = pSetNew->pSmaF->size;
} }
// sst
if (sameDisk) {
if (pSetNew->nSstF > pSetOld->nSstF) {
ASSERT(pSetNew->nSstF = pSetOld->nSstF + 1);
pSetOld->aSstF[pSetOld->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pSetOld->aSstF[pSetOld->nSstF] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->aSstF[pSetOld->nSstF] = *pSetNew->aSstF[pSetOld->nSstF];
pSetOld->aSstF[pSetOld->nSstF]->nRef = 1;
pSetOld->nSstF++;
} else if (pSetNew->nSstF < pSetOld->nSstF) {
ASSERT(pSetNew->nSstF == 1);
for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
SSstFile *pSstFile = pSetOld->aSstF[iSst];
nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
if (nRef == 0) {
tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
taosRemoveFile(fname);
taosMemoryFree(pSstFile);
}
pSetOld->aSstF[iSst] = NULL;
}
pSetOld->nSstF = 1;
pSetOld->aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pSetOld->aSstF[0] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->aSstF[0] = *pSetNew->aSstF[0];
pSetOld->aSstF[0]->nRef = 1;
} else {
for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
if (pSetOld->aSstF[iSst]->commitID != pSetNew->aSstF[iSst]->commitID) {
SSstFile *pSstFile = pSetOld->aSstF[iSst];
nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
if (nRef == 0) {
tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
taosRemoveFile(fname);
taosMemoryFree(pSstFile);
}
pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pSetOld->aSstF[iSst] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst];
pSetOld->aSstF[iSst]->nRef = 1;
} else {
ASSERT(pSetOld->aSstF[iSst]->size == pSetOld->aSstF[iSst]->size);
ASSERT(pSetOld->aSstF[iSst]->offset == pSetOld->aSstF[iSst]->offset);
}
}
}
} else {
ASSERT(pSetOld->nSstF == pSetNew->nSstF);
for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
SSstFile *pSstFile = pSetOld->aSstF[iSst];
nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
if (nRef == 0) {
tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
taosRemoveFile(fname);
taosMemoryFree(pSstFile);
}
pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (pSetOld->aSstF[iSst] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst];
pSetOld->aSstF[iSst]->nRef = 1;
}
}
if (!sameDisk) { if (!sameDisk) {
pSetOld->diskId = pSetNew->diskId; pSetOld->diskId = pSetNew->diskId;
} }
...@@ -902,26 +965,27 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { ...@@ -902,26 +965,27 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
taosMemoryFree(pSetOld->pDataF); taosMemoryFree(pSetOld->pDataF);
} }
nRef = atomic_sub_fetch_32(&pSetOld->pLastF->nRef, 1); nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1);
if (nRef == 0) { if (nRef == 0) {
tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pLastF, fname); tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pSetOld->pLastF); taosMemoryFree(pSetOld->pSmaF);
} }
nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1); for (int8_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
nRef = atomic_sub_fetch_32(&pSetOld->aSstF[iSst]->nRef, 1);
if (nRef == 0) { if (nRef == 0) {
tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname); tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSstF[iSst], fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pSetOld->pSmaF); taosMemoryFree(pSetOld->aSstF[iSst]);
}
} }
taosArrayRemove(pTsdb->fs.aDFileSet, iOld); taosArrayRemove(pTsdb->fs.aDFileSet, iOld);
continue; continue;
_add_new: _add_new:
fSet.diskId = pSetNew->diskId; fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSstF = 1};
fSet.fid = pSetNew->fid;
// head // head
fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
...@@ -941,15 +1005,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { ...@@ -941,15 +1005,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
*fSet.pDataF = *pSetNew->pDataF; *fSet.pDataF = *pSetNew->pDataF;
fSet.pDataF->nRef = 1; fSet.pDataF->nRef = 1;
// last
fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
if (fSet.pLastF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*fSet.pLastF = *pSetNew->pLastF;
fSet.pLastF->nRef = 1;
// sma // sma
fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
if (fSet.pSmaF == NULL) { if (fSet.pSmaF == NULL) {
...@@ -959,6 +1014,16 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { ...@@ -959,6 +1014,16 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
*fSet.pSmaF = *pSetNew->pSmaF; *fSet.pSmaF = *pSetNew->pSmaF;
fSet.pSmaF->nRef = 1; fSet.pSmaF->nRef = 1;
// sst
ASSERT(pSetNew->nSstF == 1);
fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
if (fSet.aSstF[0] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*fSet.aSstF[0] = *pSetNew->aSstF[0];
fSet.aSstF[0]->nRef = 1;
if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
...@@ -1002,11 +1067,13 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { ...@@ -1002,11 +1067,13 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) {
nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1); nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1);
ASSERT(nRef > 0); ASSERT(nRef > 0);
nRef = atomic_fetch_add_32(&pSet->pLastF->nRef, 1); nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1);
ASSERT(nRef > 0); ASSERT(nRef > 0);
nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
nRef = atomic_fetch_add_32(&pSet->aSstF[iSst]->nRef, 1);
ASSERT(nRef > 0); ASSERT(nRef > 0);
}
if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
...@@ -1053,22 +1120,25 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { ...@@ -1053,22 +1120,25 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pDataF);
} }
// last // sma
nRef = atomic_sub_fetch_32(&pSet->pLastF->nRef, 1); nRef = atomic_sub_fetch_32(&pSet->pSmaF->nRef, 1);
ASSERT(nRef >= 0); ASSERT(nRef >= 0);
if (nRef == 0) { if (nRef == 0) {
tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname); tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pSet->pLastF); taosMemoryFree(pSet->pSmaF);
} }
// sma // sst
nRef = atomic_sub_fetch_32(&pSet->pSmaF->nRef, 1); for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
nRef = atomic_sub_fetch_32(&pSet->aSstF[iSst]->nRef, 1);
ASSERT(nRef >= 0); ASSERT(nRef >= 0);
if (nRef == 0) { if (nRef == 0) {
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname);
taosRemoveFile(fname); taosRemoveFile(fname);
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->aSstF[iSst]);
/* code */
}
} }
} }
......
source/dnode/vnode/src/tsdb/tsdbFile.c
浏览文件 @ 48593015
...@@ -53,22 +53,22 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) { ...@@ -53,22 +53,22 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) {
return n; return n;
} }
int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) { int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pLastFile->commitID); n += tPutI64v(p ? p + n : p, pSstFile->commitID);
n += tPutI64v(p ? p + n : p, pLastFile->size); n += tPutI64v(p ? p + n : p, pSstFile->size);
n += tPutI64v(p ? p + n : p, pLastFile->offset); n += tPutI64v(p ? p + n : p, pSstFile->offset);
return n; return n;
} }
static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) { static int32_t tGetSstFile(uint8_t *p, SSstFile *pSstFile) {
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p + n, &pLastFile->commitID); n += tGetI64v(p + n, &pSstFile->commitID);
n += tGetI64v(p + n, &pLastFile->size); n += tGetI64v(p + n, &pSstFile->size);
n += tGetI64v(p + n, &pLastFile->offset); n += tGetI64v(p + n, &pSstFile->offset);
return n; return n;
} }
...@@ -102,9 +102,9 @@ void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, ...@@ -102,9 +102,9 @@ void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF,
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data"); TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data");
} }
void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]) { void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]) {
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did), snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pLastF->commitID, ".last"); TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSstF->commitID, ".sst");
} }
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) {
...@@ -194,9 +194,11 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { ...@@ -194,9 +194,11 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
n += tPutDataFile(p ? p + n : p, pSet->pDataF); n += tPutDataFile(p ? p + n : p, pSet->pDataF);
n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF);
// last // sst
n += tPutU8(p ? p + n : p, 1); // for future compatibility n += tPutU8(p ? p + n : p, pSet->nSstF);
n += tPutLastFile(p ? p + n : p, pSet->pLastF); for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
n += tPutSstFile(p ? p + n : p, pSet->aSstF[iSst]);
}
return n; return n;
} }
...@@ -208,15 +210,40 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { ...@@ -208,15 +210,40 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
n += tGetI32v(p + n, &pSet->diskId.id); n += tGetI32v(p + n, &pSet->diskId.id);
n += tGetI32v(p + n, &pSet->fid); n += tGetI32v(p + n, &pSet->fid);
// data // head
pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile));
if (pSet->pHeadF == NULL) {
return -1;
}
pSet->pHeadF->nRef = 1;
n += tGetHeadFile(p + n, pSet->pHeadF); n += tGetHeadFile(p + n, pSet->pHeadF);
// data
pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile));
if (pSet->pDataF == NULL) {
return -1;
}
pSet->pDataF->nRef = 1;
n += tGetDataFile(p + n, pSet->pDataF); n += tGetDataFile(p + n, pSet->pDataF);
// sma
pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile));
if (pSet->pSmaF == NULL) {
return -1;
}
pSet->pSmaF->nRef = 1;
n += tGetSmaFile(p + n, pSet->pSmaF); n += tGetSmaFile(p + n, pSet->pSmaF);
// last // sst
uint8_t nLast; n += tGetU8(p + n, &pSet->nSstF);
n += tGetU8(p + n, &nLast); for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
n += tGetLastFile(p + n, pSet->pLastF); pSet->aSstF[iSst] = (SSstFile *)taosMemoryCalloc(1, sizeof(SSstFile));
if (pSet->aSstF[iSst] == NULL) {
return -1;
}
pSet->aSstF[iSst]->nRef = 1;
n += tGetSstFile(p + n, pSet->aSstF[iSst]);
}
return n; return n;
} }
......
source/dnode/vnode/src/tsdb/tsdbMergeTree.c 0 → 100644
浏览文件 @ 48593015
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
// SLDataIter =================================================
typedef struct SLDataIter {
SRBTreeNode node;
SSstBlk *pSstBlk;
SDataFReader *pReader;
int32_t iSst;
int8_t backward;
SArray *aSstBlk;
int32_t iSstBlk;
SBlockData bData[2];
int32_t loadIndex;
int32_t iRow;
SRowInfo rInfo;
uint64_t uid;
STimeWindow timeWindow;
SVersionRange verRange;
} SLDataIter;
static SBlockData* getCurrentBlock(SLDataIter* pIter) {
return &pIter->bData[pIter->loadIndex];
}
static SBlockData* getNextBlock(SLDataIter* pIter) {
pIter->loadIndex ^= 1;
return getCurrentBlock(pIter);
}
int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pRange) {
int32_t code = 0;
*pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
if (*pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(*pIter)->uid = uid;
(*pIter)->timeWindow = *pTimeWindow;
(*pIter)->verRange = *pRange;
(*pIter)->pReader = pReader;
(*pIter)->iSst = iSst;
(*pIter)->backward = backward;
(*pIter)->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if ((*pIter)->aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
code = tBlockDataCreate(&(*pIter)->bData[0]);
if (code) {
goto _exit;
}
code = tBlockDataCreate(&(*pIter)->bData[1]);
if (code) {
goto _exit;
}
code = tsdbReadSstBlk(pReader, iSst, (*pIter)->aSstBlk);
if (code) {
goto _exit;
}
size_t size = taosArrayGetSize((*pIter)->aSstBlk);
// find the start block
int32_t index = -1;
if (!backward) { // asc
for (int32_t i = 0; i < size; ++i) {
SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i);
if (p->minUid <= uid && p->maxUid >= uid) {
index = i;
break;
}
}
} else { // desc
for (int32_t i = size - 1; i >= 0; --i) {
SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i);
if (p->minUid <= uid && p->maxUid >= uid) {
index = i;
break;
}
}
}
(*pIter)->iSstBlk = index;
if (index != -1) {
(*pIter)->pSstBlk = taosArrayGet((*pIter)->aSstBlk, (*pIter)->iSstBlk);
}
_exit:
return code;
}
void tLDataIterClose(SLDataIter *pIter) {
tBlockDataDestroy(&pIter->bData[0], 1);
tBlockDataDestroy(&pIter->bData[1], 1);
taosArrayDestroy(pIter->aSstBlk);
taosMemoryFree(pIter);
}
extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData);
void tLDataIterNextBlock(SLDataIter *pIter) {
int32_t step = pIter->backward ? -1 : 1;
pIter->iSstBlk += step;
int32_t index = -1;
size_t size = taosArrayGetSize(pIter->aSstBlk);
for (int32_t i = pIter->iSstBlk; i < size && i >= 0; i += step) {
SSstBlk *p = taosArrayGet(pIter->aSstBlk, i);
if ((!pIter->backward) && p->minUid > pIter->uid) {
break;
}
if (pIter->backward && p->maxUid < pIter->uid) {
break;
}
if (p->minUid <= pIter->uid && p->maxUid >= pIter->uid) {
index = i;
break;
}
}
if (index == -1) {
pIter->pSstBlk = NULL;
} else {
pIter->pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk);
}
}
static void findNextValidRow(SLDataIter *pIter) {
int32_t step = pIter->backward ? -1 : 1;
bool hasVal = false;
int32_t i = pIter->iRow;
SBlockData* pBlockData = getCurrentBlock(pIter);
for (; i < pBlockData->nRow && i >= 0; i += step) {
if (pBlockData->aUid != NULL) {
if (!pIter->backward) {
if (pBlockData->aUid[i] < pIter->uid) {
continue;
} else if (pBlockData->aUid[i] > pIter->uid) {
break;
}
} else {
if (pBlockData->aUid[i] > pIter->uid) {
continue;
} else if (pBlockData->aUid[i] < pIter->uid) {
break;
}
}
}
int64_t ts = pBlockData->aTSKEY[i];
if (!pIter->backward) { // asc
if (ts > pIter->timeWindow.ekey) { // no more data
break;
} else if (ts < pIter->timeWindow.skey) {
continue;
}
} else {
if (ts < pIter->timeWindow.skey) {
break;
} else if (ts > pIter->timeWindow.ekey) {
continue;
}
}
int64_t ver = pBlockData->aVersion[i];
if (ver < pIter->verRange.minVer) {
continue;
}
// todo opt handle desc case
if (ver > pIter->verRange.maxVer) {
continue;
}
// todo handle delete soon
#if 0
TSDBKEY k = {.ts = ts, .version = ver};
if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
continue;
}
#endif
hasVal = true;
break;
}
pIter->iRow = (hasVal) ? i : -1;
}
bool tLDataIterNextRow(SLDataIter *pIter) {
int32_t code = 0;
int32_t step = pIter->backward ? -1 : 1;
// no qualified last file block in current file, no need to fetch row
if (pIter->pSstBlk == NULL) {
return false;
}
int32_t iBlockL = pIter->iSstBlk;
SBlockData* pBlockData = getCurrentBlock(pIter);
if (pBlockData->nRow == 0 && pIter->pSstBlk != NULL) { // current block not loaded yet
pBlockData = getNextBlock(pIter);
code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
pIter->iRow = (pIter->backward) ? pBlockData->nRow : -1;
}
pIter->iRow += step;
while (1) {
findNextValidRow(pIter);
if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {
tLDataIterNextBlock(pIter);
if (pIter->pSstBlk == NULL) { // no more data
goto _exit;
}
} else {
break;
}
if (iBlockL != pIter->iSstBlk) {
pBlockData = getNextBlock(pIter);
code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData);
if (code) {
goto _exit;
}
pIter->iRow = pIter->backward ? (pBlockData->nRow - 1) : 0;
}
}
pIter->rInfo.suid = pBlockData->suid;
pIter->rInfo.uid = pBlockData->uid;
pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow);
_exit:
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
}
return (code == TSDB_CODE_SUCCESS) && (pIter->pSstBlk != NULL);
}
SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; }
// SMergeTree =================================================
static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) {
SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode));
SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode));
TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row);
TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row);
if (key1.ts < key2.ts) {
return -1;
} else if (key1.ts > key2.ts) {
return 1;
} else {
if (key1.version < key2.version) {
return -1;
} else if (key1.version > key2.version) {
return 1;
} else {
return 0;
}
}
}
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange) {
pMTree->backward = backward;
pMTree->pIter = NULL;
pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
if (pMTree->pIterList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
int32_t code = TSDB_CODE_OUT_OF_MEMORY;
struct SLDataIter *pIterList[TSDB_DEFAULT_LAST_FILE] = {0};
for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) { // open all last file
code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
}
bool hasVal = tLDataIterNextRow(pIterList[i]);
if (hasVal) {
taosArrayPush(pMTree->pIterList, &pIterList[i]);
tMergeTreeAddIter(pMTree, pIterList[i]);
} else {
tLDataIterClose(pIterList[i]);
}
}
return code;
_end:
tMergeTreeClose(pMTree);
return code;
}
void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); }
bool tMergeTreeNext(SMergeTree *pMTree) {
int32_t code = TSDB_CODE_SUCCESS;
if (pMTree->pIter) {
SLDataIter *pIter = pMTree->pIter;
bool hasVal = tLDataIterNextRow(pIter);
if (!hasVal) {
pMTree->pIter = NULL;
}
// compare with min in RB Tree
pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt);
if (pMTree->pIter && pIter) {
int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node));
if (c > 0) {
tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter);
pMTree->pIter = NULL;
} else {
ASSERT(c);
}
}
}
if (pMTree->pIter == NULL) {
pMTree->pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt);
if (pMTree->pIter) {
tRBTreeDrop(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter);
}
}
return pMTree->pIter != NULL;
}
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; }
void tMergeTreeClose(SMergeTree *pMTree) {
size_t size = taosArrayGetSize(pMTree->pIterList);
for (int32_t i = 0; i < size; ++i) {
SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i);
tLDataIterClose(pIter);
}
pMTree->pIterList = taosArrayDestroy(pMTree->pIterList);
pMTree->pIter = NULL;
}
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ 48593015
...@@ -34,7 +34,7 @@ typedef struct { ...@@ -34,7 +34,7 @@ typedef struct {
typedef struct { typedef struct {
int32_t numOfBlocks; int32_t numOfBlocks;
int32_t numOfLastBlocks; int32_t numOfLastFiles;
} SBlockNumber; } SBlockNumber;
typedef struct STableBlockScanInfo { typedef struct STableBlockScanInfo {
...@@ -46,9 +46,8 @@ typedef struct STableBlockScanInfo { ...@@ -46,9 +46,8 @@ typedef struct STableBlockScanInfo {
SIterInfo iiter; // imem buffer skip list iterator SIterInfo iiter; // imem buffer skip list iterator
SArray* delSkyline; // delete info for this table SArray* delSkyline; // delete info for this table
int32_t fileDelIndex; // file block delete index int32_t fileDelIndex; // file block delete index
int32_t lastBlockDelIndex;// delete index for last block int32_t lastBlockDelIndex; // delete index for last block
bool iterInit; // whether to initialize the in-memory skip list iterator or not bool iterInit; // whether to initialize the in-memory skip list iterator or not
int16_t indexInBlockL;// row position in last block
} STableBlockScanInfo; } STableBlockScanInfo;
typedef struct SBlockOrderWrapper { typedef struct SBlockOrderWrapper {
...@@ -83,20 +82,12 @@ typedef struct SBlockLoadSuppInfo { ...@@ -83,20 +82,12 @@ typedef struct SBlockLoadSuppInfo {
char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated.
} SBlockLoadSuppInfo; } SBlockLoadSuppInfo;
typedef struct SVersionRange {
uint64_t minVer;
uint64_t maxVer;
} SVersionRange;
typedef struct SLastBlockReader { typedef struct SLastBlockReader {
SArray* pBlockL;
int32_t currentBlockIndex;
SBlockData lastBlockData;
STimeWindow window; STimeWindow window;
SVersionRange verRange; SVersionRange verRange;
int32_t order; int32_t order;
uint64_t uid; uint64_t uid;
int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL SMergeTree mergeTree;
} SLastBlockReader; } SLastBlockReader;
typedef struct SFilesetIter { typedef struct SFilesetIter {
...@@ -118,7 +109,7 @@ typedef struct SDataBlockIter { ...@@ -118,7 +109,7 @@ typedef struct SDataBlockIter {
int32_t index; int32_t index;
SArray* blockList; // SArray<SFileDataBlockInfo> SArray* blockList; // SArray<SFileDataBlockInfo>
int32_t order; int32_t order;
SBlock block; // current SBlock data SDataBlk block; // current SDataBlk data
SHashObj* pTableMap; SHashObj* pTableMap;
} SDataBlockIter; } SDataBlockIter;
...@@ -175,7 +166,8 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i ...@@ -175,7 +166,8 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i
static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader);
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader,
SRowMerger* pMerger); SRowMerger* pMerger);
static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger); static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
SRowMerger* pMerger);
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
STsdbReader* pReader); STsdbReader* pReader);
static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid);
...@@ -184,11 +176,12 @@ static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pR ...@@ -184,11 +176,12 @@ static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pR
static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order);
static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
STsdbReader* pReader, bool* freeTSRow); STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow);
static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo,
STSRow** pTSRow); STsdbReader* pReader, STSRow** pTSRow);
static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
STsdbReader* pReader);
static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData,
STbData* piMemTbData); STbData* piMemTbData);
...@@ -234,7 +227,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK ...@@ -234,7 +227,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
} }
for (int32_t j = 0; j < numOfTables; ++j) { for (int32_t j = 0; j < numOfTables; ++j) {
STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = INITIAL_ROW_INDEX_VAL}; STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
if (ASCENDING_TRAVERSE(pTsdbReader->order)) { if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) {
info.lastKey = pTsdbReader->window.skey; info.lastKey = pTsdbReader->window.skey;
...@@ -266,9 +259,7 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap) { ...@@ -266,9 +259,7 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap) {
p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter); p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter);
} }
p->fileDelIndex = -1;
p->delSkyline = taosArrayDestroy(p->delSkyline); p->delSkyline = taosArrayDestroy(p->delSkyline);
p->lastBlockDelIndex = INITIAL_ROW_INDEX_VAL;
} }
} }
...@@ -330,7 +321,8 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap ...@@ -330,7 +321,8 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap
} }
// init file iterator // init file iterator
static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader/*int32_t order, const char* idstr*/) { static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet,
STsdbReader* pReader /*int32_t order, const char* idstr*/) {
size_t numOfFileset = taosArrayGetSize(aDFileSet); size_t numOfFileset = taosArrayGetSize(aDFileSet);
pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset; pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset;
...@@ -345,19 +337,15 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb ...@@ -345,19 +337,15 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr);
return code; return code;
} }
}
SLastBlockReader* pLReader = pIter->pLastBlockReader; SLastBlockReader* pLReader = pIter->pLastBlockReader;
pLReader->pBlockL = taosArrayInit(4, sizeof(SBlockL));
pLReader->order = pReader->order; pLReader->order = pReader->order;
pLReader->window = pReader->window; pLReader->window = pReader->window;
pLReader->verRange = pReader->verRange; pLReader->verRange = pReader->verRange;
pLReader->currentBlockIndex = -1;
int32_t code = tBlockDataCreate(&pLReader->lastBlockData); pLReader->uid = 0;
if (code != TSDB_CODE_SUCCESS) { tMergeTreeClose(&pLReader->mergeTree);
return code;
}
}
tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr); tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -372,6 +360,9 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { ...@@ -372,6 +360,9 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
return false; return false;
} }
pIter->pLastBlockReader->uid = 0;
tMergeTreeClose(&pIter->pLastBlockReader->mergeTree);
// check file the time range of coverage // check file the time range of coverage
STimeWindow win = {0}; STimeWindow win = {0};
...@@ -580,14 +571,12 @@ static void cleanupTableScanInfo(SHashObj* pTableMap) { ...@@ -580,14 +571,12 @@ static void cleanupTableScanInfo(SHashObj* pTableMap) {
} }
// reset the index in last block when handing a new file // reset the index in last block when handing a new file
px->indexInBlockL = INITIAL_ROW_INDEX_VAL;
tMapDataClear(&px->mapData); tMapDataClear(&px->mapData);
taosArrayClear(px->pBlockList); taosArrayClear(px->pBlockList);
} }
} }
static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum) {
SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) {
int32_t numOfQTable = 0; int32_t numOfQTable = 0;
size_t sizeInDisk = 0; size_t sizeInDisk = 0;
size_t numOfTables = taosArrayGetSize(pIndexList); size_t numOfTables = taosArrayGetSize(pIndexList);
...@@ -605,8 +594,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* ...@@ -605,8 +594,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray*
sizeInDisk += pScanInfo->mapData.nData; sizeInDisk += pScanInfo->mapData.nData;
for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) {
SBlock block = {0}; SDataBlk block = {0};
tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetBlock); tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetDataBlk);
// 1. time range check // 1. time range check
if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) {
...@@ -632,36 +621,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* ...@@ -632,36 +621,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray*
} }
} }
size_t numOfLast = taosArrayGetSize(pLastBlockIndex); pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSstF;
for(int32_t i = 0; i < numOfLast; ++i) { int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks;
SBlockL* pLastBlock = taosArrayGet(pLastBlockIndex, i);
if (pLastBlock->suid != pReader->suid) {
continue;
}
{
// 1. time range check
if (pLastBlock->minKey > pReader->window.ekey || pLastBlock->maxKey < pReader->window.skey) {
continue;
}
// 2. version range check
if (pLastBlock->minVer > pReader->verRange.maxVer || pLastBlock->maxVer < pReader->verRange.minVer) {
continue;
}
pBlockNum->numOfLastBlocks += 1;
taosArrayPush(pQualifiedLastBlock, pLastBlock);
}
}
int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks;
double el = (taosGetTimestampUs() - st) / 1000.0; double el = (taosGetTimestampUs() - st) / 1000.0;
tsdbDebug( tsdbDebug(
"load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, block-info-size:%.2f Kb, elapsed " "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed "
"time:%.2f ms %s", "time:%.2f ms %s",
numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk / 1000.0, el, numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el,
pReader->idStr); pReader->idStr);
pReader->cost.numOfBlocks += total; pReader->cost.numOfBlocks += total;
...@@ -702,7 +669,7 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { ...@@ -702,7 +669,7 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) {
return pBlockInfo; return pBlockInfo;
} }
static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } static SDataBlk* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; }
static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
...@@ -710,7 +677,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn ...@@ -710,7 +677,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
SBlockData* pBlockData = &pStatus->fileBlockData; SBlockData* pBlockData = &pStatus->fileBlockData;
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter);
SBlock* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
SSDataBlock* pResBlock = pReader->pResBlock; SSDataBlock* pResBlock = pReader->pResBlock;
int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock);
...@@ -795,7 +762,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI ...@@ -795,7 +762,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
ASSERT(pBlockInfo != NULL); ASSERT(pBlockInfo != NULL);
SBlock* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
...@@ -873,7 +840,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { ...@@ -873,7 +840,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) {
if (pBlockInfo != NULL) { if (pBlockInfo != NULL) {
STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx);
tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetDataBlk);
} }
#if 0 #if 0
...@@ -924,12 +891,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte ...@@ -924,12 +891,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte
} }
sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf;
SBlock block = {0}; SDataBlk block = {0};
for (int32_t k = 0; k < num; ++k) { for (int32_t k = 0; k < num; ++k) {
SBlockOrderWrapper wrapper = {0}; SBlockOrderWrapper wrapper = {0};
int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k); int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k);
tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetBlock); tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetDataBlk);
wrapper.uid = pTableScanInfo->uid; wrapper.uid = pTableScanInfo->uid;
wrapper.offset = block.aSubBlock[0].offset; wrapper.offset = block.aSubBlock[0].offset;
...@@ -990,8 +957,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte ...@@ -990,8 +957,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte
} }
int64_t et = taosGetTimestampUs(); int64_t et = taosGetTimestampUs();
tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, (et - st) / 1000.0, tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks,
pReader->idStr); (et - st) / 1000.0, pReader->idStr);
cleanupBlockOrderSupporter(&sup); cleanupBlockOrderSupporter(&sup);
taosMemoryFree(pTree); taosMemoryFree(pTree);
...@@ -1018,14 +985,14 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { ...@@ -1018,14 +985,14 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) {
/** /**
* This is an two rectangles overlap cases. * This is an two rectangles overlap cases.
*/ */
static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SBlock* pBlock) { static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SDataBlk* pBlock) {
return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) || return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) ||
(pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) || (pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) ||
(pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) ||
(pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer);
} }
static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, static SDataBlk* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo,
int32_t* nextIndex, int32_t order) { int32_t* nextIndex, int32_t order) {
bool asc = ASCENDING_TRAVERSE(order); bool asc = ASCENDING_TRAVERSE(order);
if (asc && pFBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { if (asc && pFBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) {
...@@ -1039,10 +1006,10 @@ static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STab ...@@ -1039,10 +1006,10 @@ static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STab
int32_t step = asc ? 1 : -1; int32_t step = asc ? 1 : -1;
*nextIndex = pFBlockInfo->tbBlockIdx + step; *nextIndex = pFBlockInfo->tbBlockIdx + step;
SBlock* pBlock = taosMemoryCalloc(1, sizeof(SBlock)); SDataBlk* pBlock = taosMemoryCalloc(1, sizeof(SDataBlk));
int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex);
tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetBlock); tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetDataBlk);
return pBlock; return pBlock;
} }
...@@ -1085,7 +1052,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t ...@@ -1085,7 +1052,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t order) { static bool overlapWithNeighborBlock(SDataBlk* pBlock, SDataBlk* pNeighbor, int32_t order) {
// it is the last block in current file, no chance to overlap with neighbor blocks. // it is the last block in current file, no chance to overlap with neighbor blocks.
if (ASCENDING_TRAVERSE(order)) { if (ASCENDING_TRAVERSE(order)) {
return pBlock->maxKey.ts == pNeighbor->minKey.ts; return pBlock->maxKey.ts == pNeighbor->minKey.ts;
...@@ -1094,19 +1061,19 @@ static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t ...@@ -1094,19 +1061,19 @@ static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t
} }
} }
static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SBlock* pBlock) { static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SDataBlk* pBlock) {
bool ascScan = ASCENDING_TRAVERSE(order); bool ascScan = ASCENDING_TRAVERSE(order);
return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->minKey.ts)) || return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->minKey.ts)) ||
(!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->maxKey.ts)); (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->maxKey.ts));
} }
static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVerRange) { static bool keyOverlapFileBlock(TSDBKEY key, SDataBlk* pBlock, SVersionRange* pVerRange) {
return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) && return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) &&
(pBlock->minVer <= pVerRange->maxVer); (pBlock->minVer <= pVerRange->maxVer);
} }
static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock) {
size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) {
...@@ -1140,7 +1107,7 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons ...@@ -1140,7 +1107,7 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons
return false; return false;
} }
static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock, int32_t order) {
if (pBlockScanInfo->delSkyline == NULL) { if (pBlockScanInfo->delSkyline == NULL) {
return false; return false;
} }
...@@ -1175,10 +1142,10 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl ...@@ -1175,10 +1142,10 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl
// 3. current timestamp should not be overlap with each other // 3. current timestamp should not be overlap with each other
// 4. output buffer should be large enough to hold all rows in current block // 4. output buffer should be large enough to hold all rows in current block
// 5. delete info should not overlap with current block data // 5. delete info should not overlap with current block data
static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SDataBlk* pBlock,
STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) { STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) {
int32_t neighborIndex = 0; int32_t neighborIndex = 0;
SBlock* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order);
// overlap with neighbor // overlap with neighbor
bool overlapWithNeighbor = false; bool overlapWithNeighbor = false;
...@@ -1192,11 +1159,14 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc ...@@ -1192,11 +1159,14 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc
bool overlapWithDel = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); bool overlapWithDel = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order);
// todo here we need to each key in the last files to identify if it is really overlapped with last block // todo here we need to each key in the last files to identify if it is really overlapped with last block
// todo
bool overlapWithlastBlock = false; bool overlapWithlastBlock = false;
if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { #if 0
SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); if (taosArrayGetSize(pLastBlockReader->pSstBlk) > 0 && (pLastBlockReader->currentBlockIndex != -1)) {
overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); SSstBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex);
overlapWithlastBlock = !(pBlock->maxKey.ts < pSstBlk->minKey || pBlock->minKey.ts > pSstBlk->maxKey);
} }
#endif
bool moreThanOutputCapacity = pBlock->nRow > pReader->capacity; bool moreThanOutputCapacity = pBlock->nRow > pReader->capacity;
bool partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); bool partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock);
...@@ -1295,15 +1265,13 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* ...@@ -1295,15 +1265,13 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
int64_t tsLast = INT64_MIN; int64_t tsLast = INT64_MIN;
if ((pLastBlockReader->lastBlockData.nRow > 0) && hasDataInLastBlock(pLastBlockReader)) { if (hasDataInLastBlock(pLastBlockReader)) {
tsLast = getCurrentKeyInLastBlock(pLastBlockReader); tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
} }
TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY k = TSDBROW_KEY(pRow);
TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData;
int64_t minKey = 0; int64_t minKey = 0;
if (pReader->order == TSDB_ORDER_ASC) { if (pReader->order == TSDB_ORDER_ASC) {
minKey = INT64_MAX; // chosen the minimum value minKey = INT64_MAX; // chosen the minimum value
...@@ -1336,7 +1304,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* ...@@ -1336,7 +1304,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
bool init = false; bool init = false;
// ASC: file block ---> last block -----> imem -----> mem // ASC: file block ---> last block -----> imem -----> mem
//DESC: mem -----> imem -----> last block -----> file block // DESC: mem -----> imem -----> last block -----> file block
if (pReader->order == TSDB_ORDER_ASC) { if (pReader->order == TSDB_ORDER_ASC) {
if (minKey == key) { if (minKey == key) {
init = true; init = true;
...@@ -1345,7 +1313,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* ...@@ -1345,7 +1313,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
} }
if (minKey == tsLast) { if (minKey == tsLast) {
TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
if (init) { if (init) {
tRowMerge(&merge, &fRow1); tRowMerge(&merge, &fRow1);
} else { } else {
...@@ -1374,7 +1342,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* ...@@ -1374,7 +1342,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
} }
if (minKey == tsLast) { if (minKey == tsLast) {
TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
if (init) { if (init) {
tRowMerge(&merge, &fRow1); tRowMerge(&merge, &fRow1);
} else { } else {
...@@ -1411,14 +1379,13 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, ...@@ -1411,14 +1379,13 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData,
bool mergeBlockData) { bool mergeBlockData) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; // SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData;
int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader);
STSRow* pTSRow = NULL; STSRow* pTSRow = NULL;
SRowMerger merge = {0}; SRowMerger merge = {0};
TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
tRowMergerInit(&merge, &fRow, pReader->pSchema); tRowMergerInit(&merge, &fRow, pReader->pSchema);
doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
...@@ -1445,7 +1412,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader ...@@ -1445,7 +1412,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
if (pBlockData->nRow > 0) { if (pBlockData->nRow > 0) {
// no last block available, only data block exists // no last block available, only data block exists
if (pLastBlockReader->lastBlockData.nRow == 0 || (!hasDataInLastBlock(pLastBlockReader))) { if (!hasDataInLastBlock(pLastBlockReader)) {
return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
} }
...@@ -1499,7 +1466,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* ...@@ -1499,7 +1466,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader);
ASSERT(pRow != NULL && piRow != NULL); ASSERT(pRow != NULL && piRow != NULL);
SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData;
int64_t tsLast = INT64_MIN; int64_t tsLast = INT64_MIN;
if (hasDataInLastBlock(pLastBlockReader)) { if (hasDataInLastBlock(pLastBlockReader)) {
tsLast = getCurrentKeyInLastBlock(pLastBlockReader); tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
...@@ -1560,7 +1526,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* ...@@ -1560,7 +1526,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
} }
if (minKey == tsLast) { if (minKey == tsLast) {
TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
if (init) { if (init) {
tRowMerge(&merge, &fRow1); tRowMerge(&merge, &fRow1);
} else { } else {
...@@ -1610,7 +1576,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* ...@@ -1610,7 +1576,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
} }
if (minKey == tsLast) { if (minKey == tsLast) {
TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
if (init) { if (init) {
tRowMerge(&merge, &fRow1); tRowMerge(&merge, &fRow1);
} else { } else {
...@@ -1788,6 +1754,70 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* ...@@ -1788,6 +1754,70 @@ static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo*
} }
#endif #endif
static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
if (pBlockScanInfo->iterInit) {
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
TSDBKEY startKey = {0};
if (ASCENDING_TRAVERSE(pReader->order)) {
startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer};
} else {
startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer};
}
int32_t backward = (!ASCENDING_TRAVERSE(pReader->order));
STbData* d = NULL;
if (pReader->pReadSnap->pMem != NULL) {
d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid);
if (d != NULL) {
code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter);
if (code == TSDB_CODE_SUCCESS) {
pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL);
tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 " %s",
pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr);
} else {
tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid,
tstrerror(code), pReader->idStr);
return code;
}
}
} else {
tsdbDebug("%p uid:%" PRId64 ", no data in mem, %s", pReader, pBlockScanInfo->uid, pReader->idStr);
}
STbData* di = NULL;
if (pReader->pReadSnap->pIMem != NULL) {
di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid);
if (di != NULL) {
code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter);
if (code == TSDB_CODE_SUCCESS) {
pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL);
tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 " %s",
pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr);
} else {
tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid,
tstrerror(code), pReader->idStr);
return code;
}
}
} else {
tsdbDebug("%p uid:%" PRId64 ", no data in imem, %s", pReader, pBlockScanInfo->uid, pReader->idStr);
}
initDelSkylineIterator(pBlockScanInfo, pReader, d, di);
pBlockScanInfo->iterInit = true;
return TSDB_CODE_SUCCESS;
}
static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo,
STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
// it is an multi-table data block // it is an multi-table data block
...@@ -1819,34 +1849,25 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum ...@@ -1819,34 +1849,25 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, int16_t* startPos) { static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
pLastBlockReader->uid = uid; while(1) {
pLastBlockReader->rowIndex = startPos; bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
if (!hasVal) {
if (*startPos == -1) { return false;
if (ASCENDING_TRAVERSE(pLastBlockReader->order)) {
// do nothing
} else {
*startPos = pLastBlockReader->lastBlockData.nRow;
}
} }
}
static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
*pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; TSDBKEY k = TSDBROW_KEY(&row);
} if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
return true;
static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { }
bool asc = ASCENDING_TRAVERSE(pLastBlockReader->order);
int32_t step = (asc) ? 1 : -1;
if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) {
return false;
} }
#if 0
*(pLastBlockReader->rowIndex) += step; *(pLastBlockReader->rowIndex) += step;
SBlockData* pBlockData = &pLastBlockReader->lastBlockData; SBlockData* pBlockData = &pLastBlockReader->lastBlockData;
for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) { for (int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) {
if (pBlockData->aUid != NULL) { if (pBlockData->aUid != NULL) {
if (asc) { if (asc) {
if (pBlockData->aUid[i] < pLastBlockReader->uid) { if (pBlockData->aUid[i] < pLastBlockReader->uid) {
...@@ -1895,23 +1916,42 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockSc ...@@ -1895,23 +1916,42 @@ static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockSc
// set all data is consumed in last block // set all data is consumed in last block
setAllRowsChecked(pLastBlockReader); setAllRowsChecked(pLastBlockReader);
return false; return false;
#endif
} }
static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo,
SBlockData* pBlockData = &pLastBlockReader->lastBlockData; STsdbReader* pReader) {
return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; // the last block reader has been initialized for this table.
} if (pLastBlockReader->uid == pBlockScanInfo->uid) {
return true;
}
if (pLastBlockReader->uid != 0) {
tMergeTreeClose(&pLastBlockReader->mergeTree);
}
static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { initMemDataIterator(pBlockScanInfo, pReader);
if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { pLastBlockReader->uid = pBlockScanInfo->uid;
int32_t code =
tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
pBlockScanInfo->uid, &pLastBlockReader->window, &pLastBlockReader->verRange);
if (code != TSDB_CODE_SUCCESS) {
return false; return false;
} }
ASSERT(pLastBlockReader->lastBlockData.nRow > 0); return nextRowFromLastBlocks(pLastBlockReader, pBlockScanInfo);
return true;
} }
int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) {
TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
TSDBKEY key = TSDBROW_KEY(&row);
return key.ts;
}
static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; }
int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
STsdbReader* pReader) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
...@@ -1943,7 +1983,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI ...@@ -1943,7 +1983,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI
SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
int64_t key = (pBlockData->nRow > 0)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; int64_t key = (pBlockData->nRow > 0) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN;
TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader);
...@@ -1996,7 +2036,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { ...@@ -1996,7 +2036,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
pDumpInfo->rowIndex += step; pDumpInfo->rowIndex += step;
SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) {
setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
break; break;
...@@ -2015,7 +2055,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { ...@@ -2015,7 +2055,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
// currently loaded file data block is consumed // currently loaded file data block is consumed
if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) {
SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
break; break;
} }
...@@ -2043,70 +2083,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { ...@@ -2043,70 +2083,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; } void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; }
static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
if (pBlockScanInfo->iterInit) {
return TSDB_CODE_SUCCESS;
}
int32_t code = TSDB_CODE_SUCCESS;
TSDBKEY startKey = {0};
if (ASCENDING_TRAVERSE(pReader->order)) {
startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer};
} else {
startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer};
}
int32_t backward = (!ASCENDING_TRAVERSE(pReader->order));
STbData* d = NULL;
if (pReader->pReadSnap->pMem != NULL) {
d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid);
if (d != NULL) {
code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter);
if (code == TSDB_CODE_SUCCESS) {
pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL);
tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 " %s",
pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr);
} else {
tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid,
tstrerror(code), pReader->idStr);
return code;
}
}
} else {
tsdbDebug("%p uid:%" PRId64 ", no data in mem, %s", pReader, pBlockScanInfo->uid, pReader->idStr);
}
STbData* di = NULL;
if (pReader->pReadSnap->pIMem != NULL) {
di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid);
if (di != NULL) {
code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter);
if (code == TSDB_CODE_SUCCESS) {
pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL);
tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 " %s",
pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr);
} else {
tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid,
tstrerror(code), pReader->idStr);
return code;
}
}
} else {
tsdbDebug("%p uid:%" PRId64 ", no data in imem, %s", pReader, pBlockScanInfo->uid, pReader->idStr);
}
initDelSkylineIterator(pBlockScanInfo, pReader, d, di);
pBlockScanInfo->iterInit = true;
return TSDB_CODE_SUCCESS;
}
int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData,
STbData* piMemTbData) { STbData* piMemTbData) {
if (pBlockScanInfo->delSkyline != NULL) { if (pBlockScanInfo->delSkyline != NULL) {
...@@ -2191,8 +2167,6 @@ _err: ...@@ -2191,8 +2167,6 @@ _err:
static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; TSDBKEY key = {.ts = TSKEY_INITIAL_VAL};
initMemDataIterator(pScanInfo, pReader);
TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader);
if (pRow != NULL) { if (pRow != NULL) {
key = TSDBROW_KEY(pRow); key = TSDBROW_KEY(pRow);
...@@ -2212,12 +2186,10 @@ static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* p ...@@ -2212,12 +2186,10 @@ static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* p
static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
pBlockNum->numOfBlocks = 0; pBlockNum->numOfBlocks = 0;
pBlockNum->numOfLastBlocks = 0; pBlockNum->numOfLastFiles = 0;
size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap);
SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx));
SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL;
taosArrayClear(pLastBlocks);
while (1) { while (1) {
bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader);
...@@ -2232,32 +2204,16 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { ...@@ -2232,32 +2204,16 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) {
return code; return code;
} }
code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSstF > 0) {
code = doLoadFileBlock(pReader, pIndexList, pBlockNum);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pIndexList); taosArrayDestroy(pIndexList);
return code; return code;
} }
if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pLastBlocks) > 0) { if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) {
SArray* pQLastBlock = taosArrayInit(4, sizeof(SBlockL));
code = doLoadFileBlock(pReader, pIndexList, pLastBlocks, pBlockNum, pQLastBlock);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pIndexList);
taosArrayDestroy(pQLastBlock);
return code;
}
if (pBlockNum->numOfBlocks + pBlockNum->numOfLastBlocks > 0) {
ASSERT(taosArrayGetSize(pQLastBlock) == pBlockNum->numOfLastBlocks);
taosArrayClear(pLastBlocks);
taosArrayAddAll(pLastBlocks, pQLastBlock);
taosArrayDestroy(pQLastBlock);
break; break;
} }
taosArrayDestroy(pQLastBlock);
} }
// no blocks in current file, try next files // no blocks in current file, try next files
...@@ -2267,81 +2223,22 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { ...@@ -2267,81 +2223,22 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo *pBlockScanInfo, STsdbReader* pReader) {
SArray* pBlocks = pLastBlockReader->pBlockL;
SBlockL* pBlock = NULL;
uint64_t uid = pBlockScanInfo->uid;
int32_t totalLastBlocks = (int32_t)taosArrayGetSize(pBlocks);
initMemDataIterator(pBlockScanInfo, pReader);
// find the correct SBlockL. todo binary search
int32_t index = -1;
for (int32_t i = 0; i < totalLastBlocks; ++i) {
SBlockL* p = taosArrayGet(pBlocks, i);
if (p->minUid <= uid && p->maxUid >= uid) {
index = i;
pBlock = p;
break;
}
}
if (index == -1) {
pLastBlockReader->currentBlockIndex = index;
tBlockDataReset(&pLastBlockReader->lastBlockData);
return TSDB_CODE_SUCCESS;
}
// the required last datablock has already loaded
if (index == pLastBlockReader->currentBlockIndex) {
return TSDB_CODE_SUCCESS;
}
int64_t st = taosGetTimestampUs();
int32_t code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr);
return code;
}
code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData);
double el = (taosGetTimestampUs() - st) / 1000.0;
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p error occurs in loading last block into buffer, last block index:%d, total:%d code:%s %s", pReader,
pLastBlockReader->currentBlockIndex, totalLastBlocks, tstrerror(code), pReader->idStr);
} else {
tsdbDebug("%p load last block completed, uid:%" PRIu64
" last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 "-%" PRId64
" elapsed time:%.2f ms, %s",
pReader, uid, index, totalLastBlocks, pBlock->nRow, pBlock->minVer, pBlock->maxVer, pBlock->minKey,
pBlock->maxKey, el, pReader->idStr);
}
pLastBlockReader->currentBlockIndex = index;
pReader->cost.lastBlockLoad += 1;
pReader->cost.lastBlockLoadTime += el;
return TSDB_CODE_SUCCESS;
}
static int32_t uidComparFunc(const void* p1, const void* p2) { static int32_t uidComparFunc(const void* p1, const void* p2) {
uint64_t pu1 = *(uint64_t*) p1; uint64_t pu1 = *(uint64_t*)p1;
uint64_t pu2 = *(uint64_t*) p2; uint64_t pu2 = *(uint64_t*)p2;
if (pu1 == pu2) { if (pu1 == pu2) {
return 0; return 0;
} else { } else {
return (pu1 < pu2)? -1:1; return (pu1 < pu2) ? -1 : 1;
} }
} }
static void extractOrderedTableUidList(SUidOrderCheckInfo *pOrderCheckInfo, SReaderStatus* pStatus) { static void extractOrderedTableUidList(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) {
int32_t index = 0; int32_t index = 0;
int32_t total = taosHashGetSize(pStatus->pTableMap); int32_t total = taosHashGetSize(pStatus->pTableMap);
void* p = taosHashIterate(pStatus->pTableMap, NULL); void* p = taosHashIterate(pStatus->pTableMap, NULL);
while(p != NULL) { while (p != NULL) {
STableBlockScanInfo* pScanInfo = p; STableBlockScanInfo* pScanInfo = p;
pOrderCheckInfo->tableUidList[index++] = pScanInfo->uid; pOrderCheckInfo->tableUidList[index++] = pScanInfo->uid;
p = taosHashIterate(pStatus->pTableMap, p); p = taosHashIterate(pStatus->pTableMap, p);
...@@ -2351,9 +2248,12 @@ static void extractOrderedTableUidList(SUidOrderCheckInfo *pOrderCheckInfo, SRea ...@@ -2351,9 +2248,12 @@ static void extractOrderedTableUidList(SUidOrderCheckInfo *pOrderCheckInfo, SRea
} }
static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) { static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) {
if (pOrderCheckInfo->tableUidList == NULL) {
int32_t total = taosHashGetSize(pStatus->pTableMap); int32_t total = taosHashGetSize(pStatus->pTableMap);
if (total == 0) {
return TSDB_CODE_SUCCESS;
}
if (pOrderCheckInfo->tableUidList == NULL) {
pOrderCheckInfo->currentIndex = 0; pOrderCheckInfo->currentIndex = 0;
pOrderCheckInfo->tableUidList = taosMemoryMalloc(total * sizeof(uint64_t)); pOrderCheckInfo->tableUidList = taosMemoryMalloc(total * sizeof(uint64_t));
if (pOrderCheckInfo->tableUidList == NULL) { if (pOrderCheckInfo->tableUidList == NULL) {
...@@ -2361,20 +2261,17 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt ...@@ -2361,20 +2261,17 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt
} }
extractOrderedTableUidList(pOrderCheckInfo, pStatus); extractOrderedTableUidList(pOrderCheckInfo, pStatus);
uint64_t uid = pOrderCheckInfo->tableUidList[0]; uint64_t uid = pOrderCheckInfo->tableUidList[0];
pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid));
} else { } else {
if (pStatus->pTableIter == NULL) { // it is the last block of a new file if (pStatus->pTableIter == NULL) { // it is the last block of a new file
// ASSERT(pOrderCheckInfo->currentIndex == taosHashGetSize(pStatus->pTableMap));
pOrderCheckInfo->currentIndex = 0; pOrderCheckInfo->currentIndex = 0;
uint64_t uid = pOrderCheckInfo->tableUidList[pOrderCheckInfo->currentIndex]; uint64_t uid = pOrderCheckInfo->tableUidList[pOrderCheckInfo->currentIndex];
pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid));
// the tableMap has already updated // the tableMap has already updated
if (pStatus->pTableIter == NULL) { if (pStatus->pTableIter == NULL) {
void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, taosHashGetSize(pStatus->pTableMap)*sizeof(uint64_t)); void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, total * sizeof(uint64_t));
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
...@@ -2391,7 +2288,7 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt ...@@ -2391,7 +2288,7 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool moveToNextTable(SUidOrderCheckInfo *pOrderedCheckInfo, SReaderStatus* pStatus) { static bool moveToNextTable(SUidOrderCheckInfo* pOrderedCheckInfo, SReaderStatus* pStatus) {
pOrderedCheckInfo->currentIndex += 1; pOrderedCheckInfo->currentIndex += 1;
if (pOrderedCheckInfo->currentIndex >= taosHashGetSize(pStatus->pTableMap)) { if (pOrderedCheckInfo->currentIndex >= taosHashGetSize(pStatus->pTableMap)) {
pStatus->pTableIter = NULL; pStatus->pTableIter = NULL;
...@@ -2408,35 +2305,17 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { ...@@ -2408,35 +2305,17 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader;
SUidOrderCheckInfo *pOrderedCheckInfo = &pStatus->uidCheckInfo; SUidOrderCheckInfo* pOrderedCheckInfo = &pStatus->uidCheckInfo;
int32_t code = initOrderCheckInfo(pOrderedCheckInfo, pStatus); int32_t code = initOrderCheckInfo(pOrderedCheckInfo, pStatus);
if (code != TSDB_CODE_SUCCESS || (taosHashGetSize(pStatus->pTableMap) == 0)) { if (code != TSDB_CODE_SUCCESS || (taosHashGetSize(pStatus->pTableMap) == 0)) {
return code; return code;
} }
while(1) { while (1) {
// load the last data block of current table // load the last data block of current table
STableBlockScanInfo* pScanInfo = pStatus->pTableIter; STableBlockScanInfo* pScanInfo = pStatus->pTableIter;
code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
if (code != TSDB_CODE_SUCCESS) { if (!hasVal) {
return code;
}
if (pLastBlockReader->currentBlockIndex != -1) {
initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL);
int32_t index = pScanInfo->indexInBlockL;
if (index == INITIAL_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) {
bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo);
if (!hasData) { // current table does not have rows in last block, try next table
bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus);
if (!hasNexTable) {
return TSDB_CODE_SUCCESS;
}
continue;
}
}
} else { // no data in last block, try next table
bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus);
if (!hasNexTable) { if (!hasNexTable) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -2462,9 +2341,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { ...@@ -2462,9 +2341,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
} }
static int32_t doBuildDataBlock(STsdbReader* pReader) { static int32_t doBuildDataBlock(STsdbReader* pReader) {
TSDBKEY key = {0};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SBlock* pBlock = NULL; SDataBlk* pBlock = NULL;
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
SDataBlockIter* pBlockIter = &pStatus->blockIter; SDataBlockIter* pBlockIter = &pStatus->blockIter;
...@@ -2482,21 +2360,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { ...@@ -2482,21 +2360,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
pBlock = getCurrentBlock(pBlockIter); pBlock = getCurrentBlock(pBlockIter);
} }
{ initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
key = getCurrentKeyInBuf(pScanInfo, pReader); TSDBKEY key = getCurrentKeyInBuf(pScanInfo, pReader);
// load the last data block of current table
code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
// note: the lastblock may be null here
initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL);
if (pScanInfo->indexInBlockL == INITIAL_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) {
bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo);
}
}
if (pBlockInfo == NULL) { // build data block from last data file if (pBlockInfo == NULL) { // build data block from last data file
ASSERT(pBlockIter->numOfBlocks == 0); ASSERT(pBlockIter->numOfBlocks == 0);
...@@ -2524,7 +2389,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { ...@@ -2524,7 +2389,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) {
// only return the rows in last block // only return the rows in last block
int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
ASSERT (tsLast >= pBlock->maxKey.ts); ASSERT(tsLast >= pBlock->maxKey.ts);
tBlockDataReset(&pReader->status.fileBlockData); tBlockDataReset(&pReader->status.fileBlockData);
code = buildComposedDataBlock(pReader); code = buildComposedDataBlock(pReader);
...@@ -2575,7 +2440,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { ...@@ -2575,7 +2440,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
// set the correct start position in case of the first/last file block, according to the query time window // set the correct start position in case of the first/last file block, according to the query time window
static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) {
SBlock* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
...@@ -2595,7 +2460,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl ...@@ -2595,7 +2460,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl
} }
// all data files are consumed, try data in buffer // all data files are consumed, try data in buffer
if (num.numOfBlocks + num.numOfLastBlocks == 0) { if (num.numOfBlocks + num.numOfLastFiles == 0) {
pReader->status.loadFromFile = false; pReader->status.loadFromFile = false;
return code; return code;
} }
...@@ -2608,9 +2473,6 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl ...@@ -2608,9 +2473,6 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl
resetDataBlockIterator(pBlockIter, pReader->order); resetDataBlockIterator(pBlockIter, pReader->order);
} }
SLastBlockReader* pLReader = pReader->status.fileIter.pLastBlockReader;
pLReader->currentBlockIndex = -1;
// set the correct start position according to the query time window // set the correct start position according to the query time window
initBlockDumpInfo(pReader, pBlockIter); initBlockDumpInfo(pReader, pBlockIter);
return code; return code;
...@@ -2675,7 +2537,8 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { ...@@ -2675,7 +2537,8 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
bool hasNext = blockIteratorNext(&pReader->status.blockIter); bool hasNext = blockIteratorNext(&pReader->status.blockIter);
if (hasNext) { // check for the next block in the block accessed order list if (hasNext) { // check for the next block in the block accessed order list
initBlockDumpInfo(pReader, pBlockIter); initBlockDumpInfo(pReader, pBlockIter);
} else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now } else if (hasDataInLastBlock(pReader->status.fileIter.pLastBlockReader)) {
// data blocks in current file are exhausted, let's try the next file now
tBlockDataReset(&pReader->status.fileBlockData); tBlockDataReset(&pReader->status.fileBlockData);
resetDataBlockIterator(pBlockIter, pReader->order); resetDataBlockIterator(pBlockIter, pReader->order);
goto _begin; goto _begin;
...@@ -2948,7 +2811,7 @@ typedef enum { ...@@ -2948,7 +2811,7 @@ typedef enum {
CHECK_FILEBLOCK_QUIT = 0x2, CHECK_FILEBLOCK_QUIT = 0x2,
} CHECK_FILEBLOCK_STATE; } CHECK_FILEBLOCK_STATE;
static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SBlock* pBlock, static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SDataBlk* pBlock,
SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key, SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key,
CHECK_FILEBLOCK_STATE* state) { CHECK_FILEBLOCK_STATE* state) {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
...@@ -2958,7 +2821,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn ...@@ -2958,7 +2821,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn
int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
int32_t nextIndex = -1; int32_t nextIndex = -1;
SBlock* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order); SDataBlk* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order);
if (pNeighborBlock == NULL) { // do nothing if (pNeighborBlock == NULL) { // do nothing
return 0; return 0;
} }
...@@ -3022,7 +2885,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc ...@@ -3022,7 +2885,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
CHECK_FILEBLOCK_STATE st; CHECK_FILEBLOCK_STATE st;
SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
SBlock* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter);
checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st);
if (st == CHECK_FILEBLOCK_QUIT) { if (st == CHECK_FILEBLOCK_QUIT) {
break; break;
...@@ -3033,11 +2896,12 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc ...@@ -3033,11 +2896,12 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger) { int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
while(nextRowInLastBlock(pLastBlockReader, pScanInfo)) { SRowMerger* pMerger) {
while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo)) {
int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
if (next1 == ts) { if (next1 == ts) {
TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
tRowMerge(pMerger, &fRow1); tRowMerge(pMerger, &fRow1);
} else { } else {
break; break;
...@@ -3175,7 +3039,8 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR ...@@ -3175,7 +3039,8 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
} }
if (pBlockScanInfo->iter.hasVal && pRow != NULL) { if (pBlockScanInfo->iter.hasVal && pRow != NULL) {
return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader,
freeTSRow);
} }
if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { if (pBlockScanInfo->iiter.hasVal && piRow != NULL) {
...@@ -3229,7 +3094,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* ...@@ -3229,7 +3094,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) { int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
int32_t rowIndex) {
int32_t i = 0, j = 0; int32_t i = 0, j = 0;
int32_t outputRowIndex = pResBlock->info.rows; int32_t outputRowIndex = pResBlock->info.rows;
...@@ -3306,7 +3172,7 @@ int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) { ...@@ -3306,7 +3172,7 @@ int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) {
ASSERT(pReader != NULL); ASSERT(pReader != NULL);
taosHashClear(pReader->status.pTableMap); taosHashClear(pReader->status.pTableMap);
STableBlockScanInfo info = {.lastKey = 0, .uid = uid, .indexInBlockL = INITIAL_ROW_INDEX_VAL}; STableBlockScanInfo info = {.lastKey = 0, .uid = uid};
taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
} }
...@@ -3327,7 +3193,6 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { ...@@ -3327,7 +3193,6 @@ void* tsdbGetIvtIdx(SMeta* pMeta) {
uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; }
// ====================================== EXPOSED APIs ====================================== // ====================================== EXPOSED APIs ======================================
int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader, int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader,
const char* idstr) { const char* idstr) {
...@@ -3450,7 +3315,6 @@ void tsdbReaderClose(STsdbReader* pReader) { ...@@ -3450,7 +3315,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
} }
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap);
taosMemoryFreeClear(pSupInfo->plist); taosMemoryFreeClear(pSupInfo->plist);
taosMemoryFree(pSupInfo->colIds); taosMemoryFree(pSupInfo->colIds);
...@@ -3461,6 +3325,7 @@ void tsdbReaderClose(STsdbReader* pReader) { ...@@ -3461,6 +3325,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
taosMemoryFreeClear(pSupInfo->buildBuf[i]); taosMemoryFreeClear(pSupInfo->buildBuf[i]);
} }
} }
taosMemoryFree(pSupInfo->buildBuf); taosMemoryFree(pSupInfo->buildBuf);
tBlockDataDestroy(&pReader->status.fileBlockData, true); tBlockDataDestroy(&pReader->status.fileBlockData, true);
...@@ -3474,12 +3339,13 @@ void tsdbReaderClose(STsdbReader* pReader) { ...@@ -3474,12 +3339,13 @@ void tsdbReaderClose(STsdbReader* pReader) {
tsdbDataFReaderClose(&pReader->pFileReader); tsdbDataFReaderClose(&pReader->pFileReader);
} }
tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap);
taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList);
SFilesetIter* pFilesetIter = &pReader->status.fileIter; SFilesetIter* pFilesetIter = &pReader->status.fileIter;
if (pFilesetIter->pLastBlockReader != NULL) { if (pFilesetIter->pLastBlockReader != NULL) {
tBlockDataDestroy(&pFilesetIter->pLastBlockReader->lastBlockData, true); tMergeTreeClose(&pFilesetIter->pLastBlockReader->mergeTree);
taosArrayDestroy(pFilesetIter->pLastBlockReader->pBlockL);
taosMemoryFree(pFilesetIter->pLastBlockReader); taosMemoryFree(pFilesetIter->pLastBlockReader);
} }
...@@ -3603,12 +3469,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS ...@@ -3603,12 +3469,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
int64_t stime = taosGetTimestampUs(); int64_t stime = taosGetTimestampUs();
SBlockLoadSuppInfo* pSup = &pReader->suppInfo; SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
if (tBlockHasSma(pBlock)) { if (tDataBlkHasSma(pBlock)) {
code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code),
...@@ -3785,7 +3651,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa ...@@ -3785,7 +3651,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
while (true) { while (true) {
if (hasNext) { if (hasNext) {
SBlock* pBlock = getCurrentBlock(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter);
int32_t numOfRows = pBlock->nRow; int32_t numOfRows = pBlock->nRow;
pTableBlockInfo->totalRows += numOfRows; pTableBlockInfo->totalRows += numOfRows;
......
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
浏览文件 @ 48593015
...@@ -394,17 +394,6 @@ _err: ...@@ -394,17 +394,6 @@ _err:
} }
// SDataFReader ==================================================== // SDataFReader ====================================================
struct SDataFReader {
STsdb *pTsdb;
SDFileSet *pSet;
TdFilePtr pHeadFD;
TdFilePtr pDataFD;
TdFilePtr pLastFD;
TdFilePtr pSmaFD;
uint8_t *aBuf[3];
};
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
int32_t code = 0; int32_t code = 0;
SDataFReader *pReader; SDataFReader *pReader;
...@@ -436,14 +425,6 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS ...@@ -436,14 +425,6 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
goto _err; goto _err;
} }
// last
tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname);
pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ);
if (pReader->pLastFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// sma // sma
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ); pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ);
...@@ -452,6 +433,16 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS ...@@ -452,6 +433,16 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
goto _err; goto _err;
} }
// sst
for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname);
pReader->aLastFD[iSst] = taosOpenFile(fname, TD_FILE_READ);
if (pReader->aLastFD[iSst] == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
}
*ppReader = pReader; *ppReader = pReader;
return code; return code;
...@@ -465,30 +456,35 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { ...@@ -465,30 +456,35 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
int32_t code = 0; int32_t code = 0;
if (*ppReader == NULL) goto _exit; if (*ppReader == NULL) goto _exit;
// head
if (taosCloseFile(&(*ppReader)->pHeadFD) < 0) { if (taosCloseFile(&(*ppReader)->pHeadFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
// data
if (taosCloseFile(&(*ppReader)->pDataFD) < 0) { if (taosCloseFile(&(*ppReader)->pDataFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (taosCloseFile(&(*ppReader)->pLastFD) < 0) { // sma
if (taosCloseFile(&(*ppReader)->pSmaFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (taosCloseFile(&(*ppReader)->pSmaFD) < 0) { // sst
for (int32_t iSst = 0; iSst < (*ppReader)->pSet->nSstF; iSst++) {
if (taosCloseFile(&(*ppReader)->aLastFD[iSst]) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
}
for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) {
tFree((*ppReader)->aBuf[iBuf]); tFree((*ppReader)->aBuf[iBuf]);
} }
taosMemoryFree(*ppReader); taosMemoryFree(*ppReader);
_exit: _exit:
...@@ -563,14 +559,14 @@ _err: ...@@ -563,14 +559,14 @@ _err:
return code; return code;
} }
int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) {
int32_t code = 0; int32_t code = 0;
int64_t offset = pReader->pSet->pLastF->offset; int64_t offset = pReader->pSet->aSstF[iSst]->offset;
int64_t size = pReader->pSet->pLastF->size - offset; int64_t size = pReader->pSet->aSstF[iSst]->size - offset;
int64_t n; int64_t n;
uint32_t delimiter; uint32_t delimiter;
taosArrayClear(aBlockL); taosArrayClear(aSstBlk);
if (size == 0) { if (size == 0) {
goto _exit; goto _exit;
} }
...@@ -580,13 +576,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { ...@@ -580,13 +576,13 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) {
if (code) goto _err; if (code) goto _err;
// seek // seek
if (taosLSeekFile(pReader->pLastFD, offset, SEEK_SET) < 0) { if (taosLSeekFile(pReader->aLastFD[iSst], offset, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
// read // read
n = taosReadFile(pReader->pLastFD, pReader->aBuf[0], size); n = taosReadFile(pReader->aLastFD[iSst], pReader->aBuf[0], size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
...@@ -607,10 +603,10 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { ...@@ -607,10 +603,10 @@ int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) {
ASSERT(delimiter == TSDB_FILE_DLMT); ASSERT(delimiter == TSDB_FILE_DLMT);
while (n < size - sizeof(TSCKSUM)) { while (n < size - sizeof(TSCKSUM)) {
SBlockL blockl; SSstBlk blockl;
n += tGetBlockL(pReader->aBuf[0] + n, &blockl); n += tGetSstBlk(pReader->aBuf[0] + n, &blockl);
if (taosArrayPush(aBlockL, &blockl) == NULL) { if (taosArrayPush(aSstBlk, &blockl) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
...@@ -681,9 +677,9 @@ _err: ...@@ -681,9 +677,9 @@ _err:
return code; return code;
} }
int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aColumnDataAgg) {
int32_t code = 0; int32_t code = 0;
SSmaInfo *pSmaInfo = &pBlock->smaInfo; SSmaInfo *pSmaInfo = &pDataBlk->smaInfo;
ASSERT(pSmaInfo->size > 0); ASSERT(pSmaInfo->size > 0);
...@@ -745,7 +741,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo ...@@ -745,7 +741,7 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
tBlockDataClear(pBlockData); tBlockDataClear(pBlockData);
TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; TdFilePtr pFD = fromLast ? pReader->aLastFD[0] : pReader->pDataFD; // (todo)
// uid + version + tskey // uid + version + tskey
code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1); code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1);
...@@ -847,13 +843,13 @@ _err: ...@@ -847,13 +843,13 @@ _err:
return code; return code;
} }
int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData) { int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) {
int32_t code = 0; int32_t code = 0;
code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData); code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], 0, pBlockData);
if (code) goto _err; if (code) goto _err;
if (pBlock->nSubBlock > 1) { if (pDataBlk->nSubBlock > 1) {
SBlockData bData1; SBlockData bData1;
SBlockData bData2; SBlockData bData2;
...@@ -867,8 +863,8 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl ...@@ -867,8 +863,8 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl
tBlockDataInitEx(&bData1, pBlockData); tBlockDataInitEx(&bData1, pBlockData);
tBlockDataInitEx(&bData2, pBlockData); tBlockDataInitEx(&bData2, pBlockData);
for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[iSubBlock], 0, &bData1); code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], 0, &bData1);
if (code) { if (code) {
tBlockDataDestroy(&bData1, 1); tBlockDataDestroy(&bData1, 1);
tBlockDataDestroy(&bData2, 1); tBlockDataDestroy(&bData2, 1);
...@@ -901,10 +897,10 @@ _err: ...@@ -901,10 +897,10 @@ _err:
return code; return code;
} }
int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData) { int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) {
int32_t code = 0; int32_t code = 0;
code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); code = tsdbReadBlockDataImpl(pReader, &pSstBlk->bInfo, 1, pBlockData);
if (code) goto _err; if (code) goto _err;
return code; return code;
...@@ -914,6 +910,21 @@ _err: ...@@ -914,6 +910,21 @@ _err:
return code; return code;
} }
int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) {
int32_t code = 0;
// read
code = tsdbReadAndCheck(pReader->aLastFD[iSst], pSstBlk->bInfo.offset, &pReader->aBuf[0], pSstBlk->bInfo.szBlock, 0);
if (code) goto _exit;
// decmpr
code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]);
if (code) goto _exit;
_exit:
return code;
}
// SDataFWriter ==================================================== // SDataFWriter ====================================================
int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) {
int32_t code = 0; int32_t code = 0;
...@@ -929,18 +940,22 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ...@@ -929,18 +940,22 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
if (code) goto _err;
pWriter->pTsdb = pTsdb; pWriter->pTsdb = pTsdb;
pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, pWriter->wSet = (SDFileSet){
.diskId = pSet->diskId,
.fid = pSet->fid, .fid = pSet->fid,
.pHeadF = &pWriter->fHead, .pHeadF = &pWriter->fHead,
.pDataF = &pWriter->fData, .pDataF = &pWriter->fData,
.pLastF = &pWriter->fLast, .pSmaF = &pWriter->fSma,
.pSmaF = &pWriter->fSma}; .nSstF = pSet->nSstF //
};
pWriter->fHead = *pSet->pHeadF; pWriter->fHead = *pSet->pHeadF;
pWriter->fData = *pSet->pDataF; pWriter->fData = *pSet->pDataF;
pWriter->fLast = *pSet->pLastF;
pWriter->fSma = *pSet->pSmaF; pWriter->fSma = *pSet->pSmaF;
for (int8_t iSst = 0; iSst < pSet->nSstF; iSst++) {
pWriter->wSet.aSstF[iSst] = &pWriter->fSst[iSst];
pWriter->fSst[iSst] = *pSet->aSstF[iSst];
}
// head // head
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
...@@ -991,36 +1006,6 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ...@@ -991,36 +1006,6 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
ASSERT(n == pWriter->fData.size); ASSERT(n == pWriter->fData.size);
} }
// last
if (pWriter->fLast.size == 0) {
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
} else {
flag = TD_FILE_WRITE;
}
tsdbLastFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fLast, fname);
pWriter->pLastFD = taosOpenFile(fname, flag);
if (pWriter->pLastFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (pWriter->fLast.size == 0) {
n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
pWriter->fLast.size += TSDB_FHDR_SIZE;
} else {
n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_END);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
ASSERT(n == pWriter->fLast.size);
}
// sma // sma
if (pWriter->fSma.size == 0) { if (pWriter->fSma.size == 0) {
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
...@@ -1051,6 +1036,22 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ...@@ -1051,6 +1036,22 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
ASSERT(n == pWriter->fSma.size); ASSERT(n == pWriter->fSma.size);
} }
// sst
ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0);
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname);
pWriter->pLastFD = taosOpenFile(fname, flag);
if (pWriter->pLastFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE;
*ppWriter = pWriter; *ppWriter = pWriter;
return code; return code;
...@@ -1078,12 +1079,12 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { ...@@ -1078,12 +1079,12 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) {
goto _err; goto _err;
} }
if (taosFsyncFile((*ppWriter)->pLastFD) < 0) { if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) { if (taosFsyncFile((*ppWriter)->pLastFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
...@@ -1099,12 +1100,12 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { ...@@ -1099,12 +1100,12 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) {
goto _err; goto _err;
} }
if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) { if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) { if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
...@@ -1161,35 +1162,35 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { ...@@ -1161,35 +1162,35 @@ int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) {
goto _err; goto _err;
} }
// last ============== // sma ==============
memset(hdr, 0, TSDB_FHDR_SIZE); memset(hdr, 0, TSDB_FHDR_SIZE);
tPutLastFile(hdr, &pWriter->fLast); tPutSmaFile(hdr, &pWriter->fSma);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
// sma ============== // sst ==============
memset(hdr, 0, TSDB_FHDR_SIZE); memset(hdr, 0, TSDB_FHDR_SIZE);
tPutSmaFile(hdr, &pWriter->fSma); tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET); n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
...@@ -1300,22 +1301,22 @@ _err: ...@@ -1300,22 +1301,22 @@ _err:
return code; return code;
} }
int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) {
int32_t code = 0; int32_t code = 0;
SLastFile *pLastFile = &pWriter->fLast; SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1];
int64_t size; int64_t size;
int64_t n; int64_t n;
// check // check
if (taosArrayGetSize(aBlockL) == 0) { if (taosArrayGetSize(aSstBlk) == 0) {
pLastFile->offset = pLastFile->size; pSstFile->offset = pSstFile->size;
goto _exit; goto _exit;
} }
// size // size
size = sizeof(uint32_t); // TSDB_FILE_DLMT size = sizeof(uint32_t); // TSDB_FILE_DLMT
for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) {
size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL)); size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL));
} }
size += sizeof(TSCKSUM); size += sizeof(TSCKSUM);
...@@ -1326,8 +1327,8 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { ...@@ -1326,8 +1327,8 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) {
// encode // encode
n = 0; n = 0;
n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) {
n += tPutBlockL(pWriter->aBuf[0] + n, taosArrayGet(aBlockL, iBlockL)); n += tPutSstBlk(pWriter->aBuf[0] + n, taosArrayGet(aSstBlk, iBlockL));
} }
taosCalcChecksumAppend(0, pWriter->aBuf[0], size); taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
...@@ -1341,12 +1342,12 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { ...@@ -1341,12 +1342,12 @@ int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) {
} }
// update // update
pLastFile->offset = pLastFile->size; pSstFile->offset = pSstFile->size;
pLastFile->size += size; pSstFile->size += size;
_exit: _exit:
tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode),
pLastFile->offset, size); pSstFile->offset, size);
return code; return code;
_err: _err:
...@@ -1354,28 +1355,28 @@ _err: ...@@ -1354,28 +1355,28 @@ _err:
return code; return code;
} }
static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SDataBlk *pDataBlk) {
for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]};
if (iRow == 0) { if (iRow == 0) {
if (tsdbKeyCmprFn(&pBlock->minKey, &key) > 0) { if (tsdbKeyCmprFn(&pDataBlk->minKey, &key) > 0) {
pBlock->minKey = key; pDataBlk->minKey = key;
} }
} else { } else {
if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) {
pBlock->hasDup = 1; pDataBlk->hasDup = 1;
} }
} }
if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pBlock->maxKey, &key) < 0) { if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pDataBlk->maxKey, &key) < 0) {
pBlock->maxKey = key; pDataBlk->maxKey = key;
} }
pBlock->minVer = TMIN(pBlock->minVer, key.version); pDataBlk->minVer = TMIN(pDataBlk->minVer, key.version);
pBlock->maxVer = TMAX(pBlock->maxVer, key.version); pDataBlk->maxVer = TMAX(pDataBlk->maxVer, key.version);
} }
pBlock->nRow += pBlockData->nRow; pDataBlk->nRow += pBlockData->nRow;
} }
static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) {
...@@ -1430,7 +1431,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock ...@@ -1430,7 +1431,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock
ASSERT(pBlockData->nRow > 0); ASSERT(pBlockData->nRow > 0);
pBlkInfo->offset = toLast ? pWriter->fLast.size : pWriter->fData.size; pBlkInfo->offset = toLast ? pWriter->fSst[pWriter->wSet.nSstF - 1].size : pWriter->fData.size;
pBlkInfo->szBlock = 0; pBlkInfo->szBlock = 0;
pBlkInfo->szKey = 0; pBlkInfo->szKey = 0;
...@@ -1474,7 +1475,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock ...@@ -1474,7 +1475,7 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlock
// update info // update info
if (toLast) { if (toLast) {
pWriter->fLast.size += pBlkInfo->szBlock; pWriter->fSst[pWriter->wSet.nSstF - 1].size += pBlkInfo->szBlock;
} else { } else {
pWriter->fData.size += pBlkInfo->szBlock; pWriter->fData.size += pBlkInfo->szBlock;
} }
...@@ -1553,9 +1554,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1553,9 +1554,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
taosCloseFile(&pOutFD); taosCloseFile(&pOutFD);
taosCloseFile(&PInFD); taosCloseFile(&PInFD);
// last // sst
tsdbLastFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pLastF, fNameFrom); tsdbSstFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSstF[0], fNameFrom);
tsdbLastFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pLastF, fNameTo); tsdbSstFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSstF[0], fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
...@@ -1569,7 +1570,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1569,7 +1570,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pLastF->size); n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSstF[0]->size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
......
source/dnode/vnode/src/tsdb/tsdbRetention.c
浏览文件 @ 48593015
...@@ -60,7 +60,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { ...@@ -60,7 +60,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
if (expLevel < 0) { if (expLevel < 0) {
taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF); taosMemoryFree(pSet->pDataF);
taosMemoryFree(pSet->pLastF); taosMemoryFree(pSet->aSstF[0]);
taosMemoryFree(pSet->pSmaF); taosMemoryFree(pSet->pSmaF);
taosArrayRemove(fs.aDFileSet, iSet); taosArrayRemove(fs.aDFileSet, iSet);
iSet--; iSet--;
......
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
浏览文件 @ 48593015
...@@ -27,13 +27,13 @@ struct STsdbSnapReader { ...@@ -27,13 +27,13 @@ struct STsdbSnapReader {
int32_t fid; int32_t fid;
SDataFReader* pDataFReader; SDataFReader* pDataFReader;
SArray* aBlockIdx; // SArray<SBlockIdx> SArray* aBlockIdx; // SArray<SBlockIdx>
SArray* aBlockL; // SArray<SBlockL> SArray* aSstBlk; // SArray<SSstBlk>
SBlockIdx* pBlockIdx; SBlockIdx* pBlockIdx;
SBlockL* pBlockL; SSstBlk* pSstBlk;
int32_t iBlockIdx; int32_t iBlockIdx;
int32_t iBlockL; int32_t iBlockL;
SMapData mBlock; // SMapData<SBlock> SMapData mBlock; // SMapData<SDataBlk>
int32_t iBlock; int32_t iBlock;
SBlockData oBlockData; SBlockData oBlockData;
SBlockData nBlockData; SBlockData nBlockData;
...@@ -64,7 +64,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { ...@@ -64,7 +64,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx);
if (code) goto _err; if (code) goto _err;
code = tsdbReadBlockL(pReader->pDataFReader, pReader->aBlockL); code = tsdbReadSstBlk(pReader->pDataFReader, 0, pReader->aSstBlk);
if (code) goto _err; if (code) goto _err;
// init // init
...@@ -82,13 +82,13 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { ...@@ -82,13 +82,13 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
pReader->iBlockL = 0; pReader->iBlockL = 0;
while (true) { while (true) {
if (pReader->iBlockL >= taosArrayGetSize(pReader->aBlockL)) { if (pReader->iBlockL >= taosArrayGetSize(pReader->aSstBlk)) {
pReader->pBlockL = NULL; pReader->pSstBlk = NULL;
break; break;
} }
pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL); pReader->pSstBlk = (SSstBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL);
if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
// TODO // TODO
break; break;
} }
...@@ -101,8 +101,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { ...@@ -101,8 +101,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
} }
while (true) { while (true) {
if (pReader->pBlockIdx && pReader->pBlockL) { if (pReader->pBlockIdx && pReader->pSstBlk) {
TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; TABLEID id = {.suid = pReader->pSstBlk->suid, .uid = pReader->pSstBlk->minUid};
ASSERT(0); ASSERT(0);
...@@ -115,8 +115,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { ...@@ -115,8 +115,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
// } // }
} else if (pReader->pBlockIdx) { } else if (pReader->pBlockIdx) {
while (pReader->iBlock < pReader->mBlock.nItem) { while (pReader->iBlock < pReader->mBlock.nItem) {
SBlock block; SDataBlk block;
tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetDataBlk);
if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) { if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) {
// load data (todo) // load data (todo)
...@@ -142,18 +142,18 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { ...@@ -142,18 +142,18 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
} }
if (*ppData) goto _exit; if (*ppData) goto _exit;
} else if (pReader->pBlockL) { } else if (pReader->pSstBlk) {
while (pReader->pBlockL) { while (pReader->pSstBlk) {
if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
// load data (todo) // load data (todo)
} }
// next // next
pReader->iBlockL++; pReader->iBlockL++;
if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { if (pReader->iBlockL < taosArrayGetSize(pReader->aSstBlk)) {
pReader->pBlockL = (SBlockL*)taosArrayGetSize(pReader->aBlockL); pReader->pSstBlk = (SSstBlk*)taosArrayGetSize(pReader->aSstBlk);
} else { } else {
pReader->pBlockL = NULL; pReader->pSstBlk = NULL;
} }
if (*ppData) goto _exit; if (*ppData) goto _exit;
...@@ -298,8 +298,8 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type ...@@ -298,8 +298,8 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
pReader->aBlockL = taosArrayInit(0, sizeof(SBlockL)); pReader->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if (pReader->aBlockL == NULL) { if (pReader->aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
...@@ -338,7 +338,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { ...@@ -338,7 +338,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
if (pReader->pDataFReader) { if (pReader->pDataFReader) {
tsdbDataFReaderClose(&pReader->pDataFReader); tsdbDataFReaderClose(&pReader->pDataFReader);
} }
taosArrayDestroy(pReader->aBlockL); taosArrayDestroy(pReader->aSstBlk);
taosArrayDestroy(pReader->aBlockIdx); taosArrayDestroy(pReader->aBlockIdx);
tMapDataClear(&pReader->mBlock); tMapDataClear(&pReader->mBlock);
tBlockDataDestroy(&pReader->oBlockData, 1); tBlockDataDestroy(&pReader->oBlockData, 1);
...@@ -426,24 +426,24 @@ struct STsdbSnapWriter { ...@@ -426,24 +426,24 @@ struct STsdbSnapWriter {
SArray* aBlockIdx; // SArray<SBlockIdx> SArray* aBlockIdx; // SArray<SBlockIdx>
int32_t iBlockIdx; int32_t iBlockIdx;
SBlockIdx* pBlockIdx; SBlockIdx* pBlockIdx;
SMapData mBlock; // SMapData<SBlock> SMapData mBlock; // SMapData<SDataBlk>
int32_t iBlock; int32_t iBlock;
SBlockData* pBlockData; SBlockData* pBlockData;
int32_t iRow; int32_t iRow;
SBlockData bDataR; SBlockData bDataR;
SArray* aBlockL; // SArray<SBlockL> SArray* aSstBlk; // SArray<SSstBlk>
int32_t iBlockL; int32_t iBlockL;
SBlockData lDataR; SBlockData lDataR;
SDataFWriter* pDataFWriter; SDataFWriter* pDataFWriter;
SBlockIdx* pBlockIdxW; // NULL when no committing table SBlockIdx* pBlockIdxW; // NULL when no committing table
SBlock blockW; SDataBlk blockW;
SBlockData bDataW; SBlockData bDataW;
SBlockIdx blockIdxW; SBlockIdx blockIdxW;
SMapData mBlockW; // SMapData<SBlock> SMapData mBlockW; // SMapData<SDataBlk>
SArray* aBlockIdxW; // SArray<SBlockIdx> SArray* aBlockIdxW; // SArray<SBlockIdx>
SArray* aBlockLW; // SArray<SBlockL> SArray* aBlockLW; // SArray<SSstBlk>
// for del file // for del file
SDelFReader* pDelFReader; SDelFReader* pDelFReader;
...@@ -475,10 +475,10 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { ...@@ -475,10 +475,10 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
// &pWriter->blockW, pWriter->cmprAlg); // &pWriter->blockW, pWriter->cmprAlg);
if (code) goto _err; if (code) goto _err;
code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
tBlockReset(&pWriter->blockW); tDataBlkReset(&pWriter->blockW);
tBlockDataClear(&pWriter->bDataW); tBlockDataClear(&pWriter->bDataW);
} }
...@@ -499,15 +499,15 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { ...@@ -499,15 +499,15 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
// &pWriter->blockW, pWriter->cmprAlg); // &pWriter->blockW, pWriter->cmprAlg);
// if (code) goto _err; // if (code) goto _err;
code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
} }
while (true) { while (true) {
if (pWriter->iBlock >= pWriter->mBlock.nItem) break; if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
SBlock block; SDataBlk block;
tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk);
// if (block.last) { // if (block.last) {
// code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
...@@ -520,13 +520,13 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { ...@@ -520,13 +520,13 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
// if (code) goto _err; // if (code) goto _err;
// } // }
code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
pWriter->iBlock++; pWriter->iBlock++;
} }
// SBlock // SDataBlk
// code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
// if (code) goto _err; // if (code) goto _err;
...@@ -553,10 +553,10 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p ...@@ -553,10 +553,10 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p
if (code) goto _err; if (code) goto _err;
// SBlockData // SBlockData
SBlock block; SDataBlk block;
tMapDataReset(&pWriter->mBlockW); tMapDataReset(&pWriter->mBlockW);
for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetDataBlk);
// if (block.last) { // if (block.last) {
// code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
...@@ -570,11 +570,11 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p ...@@ -570,11 +570,11 @@ static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* p
// if (code) goto _err; // if (code) goto _err;
// } // }
code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
} }
// SBlock // SDataBlk
SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx); code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx);
if (code) goto _err; if (code) goto _err;
...@@ -642,10 +642,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { ...@@ -642,10 +642,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
while (true) { while (true) {
if (pWriter->iBlock >= pWriter->mBlock.nItem) break; if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
SBlock block; SDataBlk block;
int32_t c; int32_t c;
tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk);
// if (block.last) { // if (block.last) {
// pWriter->pBlockData = &pWriter->bDataR; // pWriter->pBlockData = &pWriter->bDataR;
...@@ -668,14 +668,14 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { ...@@ -668,14 +668,14 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
// &pWriter->blockW, pWriter->cmprAlg); // &pWriter->blockW, pWriter->cmprAlg);
// if (code) goto _err; // if (code) goto _err;
code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
tBlockReset(&pWriter->blockW); tDataBlkReset(&pWriter->blockW);
tBlockDataClear(&pWriter->bDataW); tBlockDataClear(&pWriter->bDataW);
} }
code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
pWriter->iBlock++; pWriter->iBlock++;
...@@ -719,10 +719,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { ...@@ -719,10 +719,10 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
// &pWriter->blockW, pWriter->cmprAlg); // &pWriter->blockW, pWriter->cmprAlg);
// if (code) goto _err; // if (code) goto _err;
code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
if (code) goto _err; if (code) goto _err;
tBlockReset(&pWriter->blockW); tDataBlkReset(&pWriter->blockW);
tBlockDataClear(&pWriter->bDataW); tBlockDataClear(&pWriter->bDataW);
} }
...@@ -803,7 +803,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { ...@@ -803,7 +803,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) {
pWriter->pBlockIdxW->suid = id.suid; pWriter->pBlockIdxW->suid = id.suid;
pWriter->pBlockIdxW->uid = id.uid; pWriter->pBlockIdxW->uid = id.uid;
tBlockReset(&pWriter->blockW); tDataBlkReset(&pWriter->blockW);
tBlockDataReset(&pWriter->bDataW); tBlockDataReset(&pWriter->bDataW);
tMapDataReset(&pWriter->mBlockW); tMapDataReset(&pWriter->mBlockW);
} }
...@@ -845,7 +845,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { ...@@ -845,7 +845,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
// write remain stuff // write remain stuff
if (taosArrayGetSize(pWriter->aBlockLW) > 0) { if (taosArrayGetSize(pWriter->aBlockLW) > 0) {
code = tsdbWriteBlockL(pWriter->pDataFWriter, pWriter->aBlockIdxW); code = tsdbWriteSstBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW);
if (code) goto _err; if (code) goto _err;
} }
...@@ -911,12 +911,12 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 ...@@ -911,12 +911,12 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx);
if (code) goto _err; if (code) goto _err;
code = tsdbReadBlockL(pWriter->pDataFReader, pWriter->aBlockL); code = tsdbReadSstBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk);
if (code) goto _err; if (code) goto _err;
} else { } else {
ASSERT(pWriter->pDataFReader == NULL); ASSERT(pWriter->pDataFReader == NULL);
taosArrayClear(pWriter->aBlockIdx); taosArrayClear(pWriter->aBlockIdx);
taosArrayClear(pWriter->aBlockL); taosArrayClear(pWriter->aSstBlk);
} }
pWriter->iBlockIdx = 0; pWriter->iBlockIdx = 0;
pWriter->pBlockIdx = NULL; pWriter->pBlockIdx = NULL;
...@@ -931,23 +931,25 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 ...@@ -931,23 +931,25 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
// write // write
SHeadFile fHead; SHeadFile fHead;
SDataFile fData; SDataFile fData;
SLastFile fLast; SSstFile fLast;
SSmaFile fSma; SSmaFile fSma;
SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma}; SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSstF[0] = &fLast, .pSmaF = &fSma};
if (pSet) { if (pSet) {
wSet.diskId = pSet->diskId; wSet.diskId = pSet->diskId;
wSet.fid = fid; wSet.fid = fid;
wSet.nSstF = 1;
fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
fData = *pSet->pDataF; fData = *pSet->pDataF;
fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0};
fSma = *pSet->pSmaF; fSma = *pSet->pSmaF;
} else { } else {
wSet.diskId = (SDiskID){.level = 0, .id = 0}; wSet.diskId = (SDiskID){.level = 0, .id = 0};
wSet.fid = fid; wSet.fid = fid;
wSet.nSstF = 1;
fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; fData = (SDataFile){.commitID = pWriter->commitID, .size = 0};
fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0, .offset = 0};
fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0};
} }
...@@ -1145,8 +1147,8 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr ...@@ -1145,8 +1147,8 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
code = tBlockDataCreate(&pWriter->bDataR); code = tBlockDataCreate(&pWriter->bDataR);
if (code) goto _err; if (code) goto _err;
pWriter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); pWriter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
if (pWriter->aBlockL == NULL) { if (pWriter->aSstBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
...@@ -1159,7 +1161,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr ...@@ -1159,7 +1161,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
code = tBlockDataCreate(&pWriter->bDataW); code = tBlockDataCreate(&pWriter->bDataW);
if (code) goto _err; if (code) goto _err;
pWriter->aBlockLW = taosArrayInit(0, sizeof(SBlockL)); pWriter->aBlockLW = taosArrayInit(0, sizeof(SSstBlk));
if (pWriter->aBlockLW == NULL) { if (pWriter->aBlockLW == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
......
source/dnode/vnode/src/tsdb/tsdbUtil.c
浏览文件 @ 48593015
...@@ -51,6 +51,22 @@ _exit: ...@@ -51,6 +51,22 @@ _exit:
return code; return code;
} }
int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo) {
int32_t code = 0;
pTo->nItem = pFrom->nItem;
pTo->nData = pFrom->nData;
code = tRealloc((uint8_t **)&pTo->aOffset, sizeof(int32_t) * pFrom->nItem);
if (code) goto _exit;
code = tRealloc(&pTo->pData, pFrom->nData);
if (code) goto _exit;
memcpy(pTo->aOffset, pFrom->aOffset, sizeof(int32_t) * pFrom->nItem);
memcpy(pTo->pData, pFrom->pData, pFrom->nData);
_exit:
return code;
}
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
int32_t (*tItemCmprFn)(const void *, const void *), void *pItem) { int32_t (*tItemCmprFn)(const void *, const void *), void *pItem) {
int32_t code = 0; int32_t code = 0;
...@@ -198,7 +214,7 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) { ...@@ -198,7 +214,7 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) {
int32_t tCmprBlockL(void const *lhs, void const *rhs) { int32_t tCmprBlockL(void const *lhs, void const *rhs) {
SBlockIdx *lBlockIdx = (SBlockIdx *)lhs; SBlockIdx *lBlockIdx = (SBlockIdx *)lhs;
SBlockL *rBlockL = (SBlockL *)rhs; SSstBlk *rBlockL = (SSstBlk *)rhs;
if (lBlockIdx->suid < rBlockL->suid) { if (lBlockIdx->suid < rBlockL->suid) {
return -1; return -1;
...@@ -215,69 +231,69 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs) { ...@@ -215,69 +231,69 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs) {
return 0; return 0;
} }
// SBlock ====================================================== // SDataBlk ======================================================
void tBlockReset(SBlock *pBlock) { void tDataBlkReset(SDataBlk *pDataBlk) {
*pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; *pDataBlk = (SDataBlk){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN};
} }
int32_t tPutBlock(uint8_t *p, void *ph) { int32_t tPutDataBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlock *pBlock = (SBlock *)ph; SDataBlk *pDataBlk = (SDataBlk *)ph;
n += tPutI64v(p ? p + n : p, pBlock->minKey.version); n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version);
n += tPutI64v(p ? p + n : p, pBlock->minKey.ts); n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts);
n += tPutI64v(p ? p + n : p, pBlock->maxKey.version); n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version);
n += tPutI64v(p ? p + n : p, pBlock->maxKey.ts); n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts);
n += tPutI64v(p ? p + n : p, pBlock->minVer); n += tPutI64v(p ? p + n : p, pDataBlk->minVer);
n += tPutI64v(p ? p + n : p, pBlock->maxVer); n += tPutI64v(p ? p + n : p, pDataBlk->maxVer);
n += tPutI32v(p ? p + n : p, pBlock->nRow); n += tPutI32v(p ? p + n : p, pDataBlk->nRow);
n += tPutI8(p ? p + n : p, pBlock->hasDup); n += tPutI8(p ? p + n : p, pDataBlk->hasDup);
n += tPutI8(p ? p + n : p, pBlock->nSubBlock); n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock);
for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].offset); n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset);
n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlock); n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock);
n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szKey); n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey);
} }
if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) {
n += tPutI64v(p ? p + n : p, pBlock->smaInfo.offset); n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset);
n += tPutI32v(p ? p + n : p, pBlock->smaInfo.size); n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size);
} }
return n; return n;
} }
int32_t tGetBlock(uint8_t *p, void *ph) { int32_t tGetDataBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlock *pBlock = (SBlock *)ph; SDataBlk *pDataBlk = (SDataBlk *)ph;
n += tGetI64v(p + n, &pBlock->minKey.version); n += tGetI64v(p + n, &pDataBlk->minKey.version);
n += tGetI64v(p + n, &pBlock->minKey.ts); n += tGetI64v(p + n, &pDataBlk->minKey.ts);
n += tGetI64v(p + n, &pBlock->maxKey.version); n += tGetI64v(p + n, &pDataBlk->maxKey.version);
n += tGetI64v(p + n, &pBlock->maxKey.ts); n += tGetI64v(p + n, &pDataBlk->maxKey.ts);
n += tGetI64v(p + n, &pBlock->minVer); n += tGetI64v(p + n, &pDataBlk->minVer);
n += tGetI64v(p + n, &pBlock->maxVer); n += tGetI64v(p + n, &pDataBlk->maxVer);
n += tGetI32v(p + n, &pBlock->nRow); n += tGetI32v(p + n, &pDataBlk->nRow);
n += tGetI8(p + n, &pBlock->hasDup); n += tGetI8(p + n, &pDataBlk->hasDup);
n += tGetI8(p + n, &pBlock->nSubBlock); n += tGetI8(p + n, &pDataBlk->nSubBlock);
for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].offset); n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset);
n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlock); n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock);
n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szKey); n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey);
} }
if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) {
n += tGetI64v(p + n, &pBlock->smaInfo.offset); n += tGetI64v(p + n, &pDataBlk->smaInfo.offset);
n += tGetI32v(p + n, &pBlock->smaInfo.size); n += tGetI32v(p + n, &pDataBlk->smaInfo.size);
} else { } else {
pBlock->smaInfo.offset = 0; pDataBlk->smaInfo.offset = 0;
pBlock->smaInfo.size = 0; pDataBlk->smaInfo.size = 0;
} }
return n; return n;
} }
int32_t tBlockCmprFn(const void *p1, const void *p2) { int32_t tDataBlkCmprFn(const void *p1, const void *p2) {
SBlock *pBlock1 = (SBlock *)p1; SDataBlk *pBlock1 = (SDataBlk *)p1;
SBlock *pBlock2 = (SBlock *)p2; SDataBlk *pBlock2 = (SDataBlk *)p2;
if (tsdbKeyCmprFn(&pBlock1->maxKey, &pBlock2->minKey) < 0) { if (tsdbKeyCmprFn(&pBlock1->maxKey, &pBlock2->minKey) < 0) {
return -1; return -1;
...@@ -288,48 +304,48 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) { ...@@ -288,48 +304,48 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) {
return 0; return 0;
} }
bool tBlockHasSma(SBlock *pBlock) { bool tDataBlkHasSma(SDataBlk *pDataBlk) {
if (pBlock->nSubBlock > 1) return false; if (pDataBlk->nSubBlock > 1) return false;
if (pBlock->hasDup) return false; if (pDataBlk->hasDup) return false;
return pBlock->smaInfo.size > 0; return pDataBlk->smaInfo.size > 0;
} }
// SBlockL ====================================================== // SSstBlk ======================================================
int32_t tPutBlockL(uint8_t *p, void *ph) { int32_t tPutSstBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlockL *pBlockL = (SBlockL *)ph; SSstBlk *pSstBlk = (SSstBlk *)ph;
n += tPutI64(p ? p + n : p, pBlockL->suid); n += tPutI64(p ? p + n : p, pSstBlk->suid);
n += tPutI64(p ? p + n : p, pBlockL->minUid); n += tPutI64(p ? p + n : p, pSstBlk->minUid);
n += tPutI64(p ? p + n : p, pBlockL->maxUid); n += tPutI64(p ? p + n : p, pSstBlk->maxUid);
n += tPutI64v(p ? p + n : p, pBlockL->minKey); n += tPutI64v(p ? p + n : p, pSstBlk->minKey);
n += tPutI64v(p ? p + n : p, pBlockL->maxKey); n += tPutI64v(p ? p + n : p, pSstBlk->maxKey);
n += tPutI64v(p ? p + n : p, pBlockL->minVer); n += tPutI64v(p ? p + n : p, pSstBlk->minVer);
n += tPutI64v(p ? p + n : p, pBlockL->maxVer); n += tPutI64v(p ? p + n : p, pSstBlk->maxVer);
n += tPutI32v(p ? p + n : p, pBlockL->nRow); n += tPutI32v(p ? p + n : p, pSstBlk->nRow);
n += tPutI64v(p ? p + n : p, pBlockL->bInfo.offset); n += tPutI64v(p ? p + n : p, pSstBlk->bInfo.offset);
n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szBlock); n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szBlock);
n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szKey); n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szKey);
return n; return n;
} }
int32_t tGetBlockL(uint8_t *p, void *ph) { int32_t tGetSstBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlockL *pBlockL = (SBlockL *)ph; SSstBlk *pSstBlk = (SSstBlk *)ph;
n += tGetI64(p + n, &pBlockL->suid); n += tGetI64(p + n, &pSstBlk->suid);
n += tGetI64(p + n, &pBlockL->minUid); n += tGetI64(p + n, &pSstBlk->minUid);
n += tGetI64(p + n, &pBlockL->maxUid); n += tGetI64(p + n, &pSstBlk->maxUid);
n += tGetI64v(p + n, &pBlockL->minKey); n += tGetI64v(p + n, &pSstBlk->minKey);
n += tGetI64v(p + n, &pBlockL->maxKey); n += tGetI64v(p + n, &pSstBlk->maxKey);
n += tGetI64v(p + n, &pBlockL->minVer); n += tGetI64v(p + n, &pSstBlk->minVer);
n += tGetI64v(p + n, &pBlockL->maxVer); n += tGetI64v(p + n, &pSstBlk->maxVer);
n += tGetI32v(p + n, &pBlockL->nRow); n += tGetI32v(p + n, &pSstBlk->nRow);
n += tGetI64v(p + n, &pBlockL->bInfo.offset); n += tGetI64v(p + n, &pSstBlk->bInfo.offset);
n += tGetI32v(p + n, &pBlockL->bInfo.szBlock); n += tGetI32v(p + n, &pSstBlk->bInfo.szBlock);
n += tGetI32v(p + n, &pBlockL->bInfo.szKey); n += tGetI32v(p + n, &pSstBlk->bInfo.szKey);
return n; return n;
} }
...@@ -1603,7 +1619,7 @@ _exit: ...@@ -1603,7 +1619,7 @@ _exit:
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]) { int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]) {
int32_t code = 0; int32_t code = 0;
tBlockDataClear(pBlockData); tBlockDataReset(pBlockData);
int32_t n = 0; int32_t n = 0;
SDiskDataHdr hdr = {0}; SDiskDataHdr hdr = {0};
......
source/dnode/vnode/src/vnd/vnodeQuery.c
浏览文件 @ 48593015
...@@ -368,6 +368,7 @@ _exit: ...@@ -368,6 +368,7 @@ _exit:
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
pLoad->vgId = TD_VID(pVnode); pLoad->vgId = TD_VID(pVnode);
pLoad->syncState = syncGetMyRole(pVnode->sync); pLoad->syncState = syncGetMyRole(pVnode->sync);
pLoad->cacheUsage = tsdbCacheGetUsage(pVnode);
pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta);
pLoad->totalStorage = (int64_t)3 * 1073741824; pLoad->totalStorage = (int64_t)3 * 1073741824;
......
source/libs/executor/inc/executil.h
浏览文件 @ 48593015
...@@ -87,8 +87,6 @@ struct SqlFunctionCtx; ...@@ -87,8 +87,6 @@ struct SqlFunctionCtx;
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
void initResultRowInfo(SResultRowInfo* pResultRowInfo); void initResultRowInfo(SResultRowInfo* pResultRowInfo);
void initResultRow(SResultRow* pResultRow);
void closeResultRow(SResultRow* pResultRow); void closeResultRow(SResultRow* pResultRow);
struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset);
......
source/libs/executor/src/dataDeleter.c
浏览文件 @ 48593015
...@@ -168,7 +168,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE ...@@ -168,7 +168,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf));
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
*pLen = ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->dataLen;
SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData;
*pLen = pEntry->dataLen;
*pQueryEnd = pDeleter->queryEnd; *pQueryEnd = pDeleter->queryEnd;
qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows);
} }
......
source/libs/executor/src/dataDispatcher.c
浏览文件 @ 48593015
...@@ -93,6 +93,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn ...@@ -93,6 +93,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pBuf->useSize = sizeof(SDataCacheEntry); pBuf->useSize = sizeof(SDataCacheEntry);
blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed);
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
pBuf->useSize += pEntry->dataLen; pBuf->useSize += pEntry->dataLen;
...@@ -170,7 +172,13 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE ...@@ -170,7 +172,13 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf));
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
*pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen;
SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData;
*pLen = pEntry->dataLen;
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
*pQueryEnd = pDispatcher->queryEnd; *pQueryEnd = pDispatcher->queryEnd;
qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows); qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows);
} }
...@@ -191,6 +199,9 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { ...@@ -191,6 +199,9 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
pOutput->numOfCols = pEntry->numOfCols; pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed; pOutput->compressed = pEntry->compressed;
ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8));
ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4));
atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
......
source/libs/executor/src/scanoperator.c
浏览文件 @ 48593015
...@@ -695,6 +695,7 @@ static void destroyTableScanOperatorInfo(void* param) { ...@@ -695,6 +695,7 @@ static void destroyTableScanOperatorInfo(void* param) {
cleanupQueryTableDataCond(&pTableScanInfo->cond); cleanupQueryTableDataCond(&pTableScanInfo->cond);
tsdbReaderClose(pTableScanInfo->dataReader); tsdbReaderClose(pTableScanInfo->dataReader);
pTableScanInfo->dataReader = NULL;
if (pTableScanInfo->pColMatchInfo != NULL) { if (pTableScanInfo->pColMatchInfo != NULL) {
taosArrayDestroy(pTableScanInfo->pColMatchInfo); taosArrayDestroy(pTableScanInfo->pColMatchInfo);
......
source/libs/executor/src/timewindowoperator.c
浏览文件 @ 48593015
...@@ -622,7 +622,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num ...@@ -622,7 +622,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
if (pr->closed) { if (pr->closed) {
ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) &&
isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); isResultRowInterpolated(pr, RESULT_ROW_END_INTERP));
tdListPopHead(pResultRowInfo->openWindow); SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
taosMemoryFree(pNode);
continue; continue;
} }
...@@ -651,7 +652,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num ...@@ -651,7 +652,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
closeResultRow(pr); closeResultRow(pr);
tdListPopHead(pResultRowInfo->openWindow); SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow);
taosMemoryFree(pNode);
} else { // the remains are can not be closed yet. } else { // the remains are can not be closed yet.
break; break;
} }
...@@ -1731,6 +1733,10 @@ void destroyIntervalOperatorInfo(void* param) { ...@@ -1731,6 +1733,10 @@ void destroyIntervalOperatorInfo(void* param) {
SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param;
cleanupBasicInfo(&pInfo->binfo); cleanupBasicInfo(&pInfo->binfo);
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
cleanupExprSupp(&pInfo->scalarSupp);
tdListFree(pInfo->binfo.resultRowInfo.openWindow);
pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);
pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols);
taosArrayDestroyEx(pInfo->pPrevValues, freeItem); taosArrayDestroyEx(pInfo->pPrevValues, freeItem);
......
source/libs/executor/src/tsort.c
浏览文件 @ 48593015
...@@ -227,9 +227,9 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int ...@@ -227,9 +227,9 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
continue; continue;
} }
SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex);
void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); void* pPage = getBufPage(pHandle->pBuf, *pPgId);
code = blockDataFromBuf(pSource->src.pBlock, pPage); code = blockDataFromBuf(pSource->src.pBlock, pPage);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
...@@ -302,9 +302,9 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeT ...@@ -302,9 +302,9 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeT
pSource->pageIndex = -1; pSource->pageIndex = -1;
pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock);
} else { } else {
SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex);
void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); void* pPage = getBufPage(pHandle->pBuf, *pPgId);
int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage); int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
......
source/libs/monitor/src/monMsg.c
浏览文件 @ 48593015
...@@ -510,6 +510,7 @@ int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) ...@@ -510,6 +510,7 @@ int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo)
SVnodeLoad *pLoad = taosArrayGet(pInfo->pVloads, i); SVnodeLoad *pLoad = taosArrayGet(pInfo->pVloads, i);
if (tEncodeI32(&encoder, pLoad->vgId) < 0) return -1; if (tEncodeI32(&encoder, pLoad->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pLoad->syncState) < 0) return -1; if (tEncodeI32(&encoder, pLoad->syncState) < 0) return -1;
if (tEncodeI64(&encoder, pLoad->cacheUsage) < 0) return -1;
if (tEncodeI64(&encoder, pLoad->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pLoad->numOfTables) < 0) return -1;
if (tEncodeI64(&encoder, pLoad->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pLoad->numOfTimeSeries) < 0) return -1;
if (tEncodeI64(&encoder, pLoad->totalStorage) < 0) return -1; if (tEncodeI64(&encoder, pLoad->totalStorage) < 0) return -1;
...@@ -544,6 +545,7 @@ int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInf ...@@ -544,6 +545,7 @@ int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInf
SVnodeLoad load = {0}; SVnodeLoad load = {0};
if (tDecodeI32(&decoder, &load.vgId) < 0) return -1; if (tDecodeI32(&decoder, &load.vgId) < 0) return -1;
if (tDecodeI32(&decoder, &load.syncState) < 0) return -1; if (tDecodeI32(&decoder, &load.syncState) < 0) return -1;
if (tDecodeI64(&decoder, &load.cacheUsage) < 0) return -1;
if (tDecodeI64(&decoder, &load.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &load.numOfTables) < 0) return -1;
if (tDecodeI64(&decoder, &load.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &load.numOfTimeSeries) < 0) return -1;
if (tDecodeI64(&decoder, &load.totalStorage) < 0) return -1; if (tDecodeI64(&decoder, &load.totalStorage) < 0) return -1;
...@@ -594,7 +596,6 @@ int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInf ...@@ -594,7 +596,6 @@ int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInf
return 0; return 0;
} }
int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) {
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);
...@@ -639,5 +640,3 @@ int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { ...@@ -639,5 +640,3 @@ int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
return 0; return 0;
} }
source/util/src/trbtree.c
浏览文件 @ 48593015
...@@ -13,179 +13,297 @@ ...@@ -13,179 +13,297 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h" #include "trbtree.h"
typedef int32_t (*tRBTreeCmprFn)(void *, void *); static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *x) {
SRBTreeNode *y = x->right;
typedef struct SRBTree SRBTree; x->right = y->left;
typedef struct SRBTreeNode SRBTreeNode; if (y->left != pTree->NIL) {
typedef struct SRBTreeIter SRBTreeIter; y->left->parent = x;
struct SRBTreeNode {
enum { RED, BLACK } color;
SRBTreeNode *parent;
SRBTreeNode *left;
SRBTreeNode *right;
uint8_t payload[];
};
struct SRBTree {
tRBTreeCmprFn cmprFn;
SRBTreeNode *root;
};
struct SRBTreeIter {
SRBTree *pTree;
};
#define RBTREE_NODE_COLOR(N) ((N) ? (N)->color : BLACK)
// APIs ================================================
static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *pNode) {
SRBTreeNode *right = pNode->right;
pNode->right = right->left;
if (pNode->right) {
pNode->right->parent = pNode;
} }
y->parent = x->parent;
right->parent = pNode->parent; if (x->parent == pTree->NIL) {
if (pNode->parent == NULL) { pTree->root = y;
pTree->root = right; } else if (x == x->parent->left) {
} else if (pNode == pNode->parent->left) { x->parent->left = y;
pNode->parent->left = right;
} else { } else {
pNode->parent->right = right; x->parent->right = y;
} }
y->left = x;
right->left = pNode; x->parent = y;
pNode->parent = right;
} }
static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *pNode) { static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *x) {
SRBTreeNode *left = pNode->left; SRBTreeNode *y = x->left;
x->left = y->right;
pNode->left = left->right; if (y->right != pTree->NIL) {
if (pNode->left) { y->right->parent = x;
pNode->left->parent = pNode;
} }
y->parent = x->parent;
left->parent = pNode->parent; if (x->parent == pTree->NIL) {
if (pNode->parent == NULL) { pTree->root = y;
pTree->root = left; } else if (x == x->parent->right) {
} else if (pNode == pNode->parent->left) { x->parent->right = y;
pNode->parent->left = left;
} else { } else {
pNode->parent->right = left; x->parent->left = y;
} }
y->right = x;
left->right = pNode; x->parent = y;
pNode->parent = left;
} }
#define tRBTreeCreate(compare) \ static void tRBTreePutFix(SRBTree *pTree, SRBTreeNode *z) {
(SRBTree) { .cmprFn = (compare), .root = NULL } while (z->parent->color == RED) {
if (z->parent == z->parent->parent->left) { // z.parent is the left child
SRBTreeNode *y = z->parent->parent->right; // uncle of z
if (y->color == RED) { // case 1
z->parent->color = BLACK;
y->color = BLACK;
z->parent->parent->color = RED;
z = z->parent->parent;
} else { // case2 or case3
if (z == z->parent->right) { // case2
z = z->parent; // marked z.parent as new z
tRBTreeRotateLeft(pTree, z);
}
// case3
z->parent->color = BLACK; // made parent black
z->parent->parent->color = RED; // made parent red
tRBTreeRotateRight(pTree, z->parent->parent);
}
} else { // z.parent is the right child
SRBTreeNode *y = z->parent->parent->left; // uncle of z
if (y->color == RED) {
z->parent->color = BLACK;
y->color = BLACK;
z->parent->parent->color = RED;
z = z->parent->parent;
} else {
if (z == z->parent->left) {
z = z->parent; // marked z.parent as new z
tRBTreeRotateRight(pTree, z);
}
z->parent->color = BLACK; // made parent black
z->parent->parent->color = RED; // made parent red
tRBTreeRotateLeft(pTree, z->parent->parent);
}
}
}
pTree->root->color = BLACK;
}
SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) {
pNew->left = NULL; if (u->parent == pTree->NIL)
pNew->right = NULL; pTree->root = v;
pNew->color = RED; else if (u == u->parent->left)
u->parent->left = v;
else
u->parent->right = v;
v->parent = u->parent;
}
// insert static void tRBTreeDropFix(SRBTree *pTree, SRBTreeNode *x) {
if (pTree->root == NULL) { while (x != pTree->root && x->color == BLACK) {
pNew->parent = NULL; if (x == x->parent->left) {
pTree->root = pNew; SRBTreeNode *w = x->parent->right;
if (w->color == RED) {
w->color = BLACK;
x->parent->color = RED;
tRBTreeRotateLeft(pTree, x->parent);
w = x->parent->right;
}
if (w->left->color == BLACK && w->right->color == BLACK) {
w->color = RED;
x = x->parent;
} else { } else {
SRBTreeNode *pNode = pTree->root; if (w->right->color == BLACK) {
while (true) { w->left->color = BLACK;
ASSERT(pNode); w->color = RED;
tRBTreeRotateRight(pTree, w);
w = x->parent->right;
}
w->color = x->parent->color;
x->parent->color = BLACK;
w->right->color = BLACK;
tRBTreeRotateLeft(pTree, x->parent);
x = pTree->root;
}
} else {
SRBTreeNode *w = x->parent->left;
if (w->color == RED) {
w->color = BLACK;
x->parent->color = RED;
tRBTreeRotateRight(pTree, x->parent);
w = x->parent->left;
}
if (w->right->color == BLACK && w->left->color == BLACK) {
w->color = RED;
x = x->parent;
} else {
if (w->left->color == BLACK) {
w->right->color = BLACK;
w->color = RED;
tRBTreeRotateLeft(pTree, w);
w = x->parent->left;
}
w->color = x->parent->color;
x->parent->color = BLACK;
w->left->color = BLACK;
tRBTreeRotateRight(pTree, x->parent);
x = pTree->root;
}
}
}
x->color = BLACK;
}
int32_t c = pTree->cmprFn(pNew->payload, pNode->payload); static SRBTreeNode *tRBTreeSuccessor(SRBTree *pTree, SRBTreeNode *pNode) {
if (c < 0) { if (pNode->right != pTree->NIL) {
if (pNode->left) { pNode = pNode->right;
while (pNode->left != pTree->NIL) {
pNode = pNode->left; pNode = pNode->left;
}
} else { } else {
pNew->parent = pNode; while (true) {
pNode->left = pNew; if (pNode->parent == pTree->NIL || pNode == pNode->parent->left) {
pNode = pNode->parent;
break; break;
} else {
pNode = pNode->parent;
} }
} else if (c > 0) { }
if (pNode->right) { }
return pNode;
}
static SRBTreeNode *tRBTreePredecessor(SRBTree *pTree, SRBTreeNode *pNode) {
if (pNode->left != pTree->NIL) {
pNode = pNode->left;
while (pNode->right != pTree->NIL) {
pNode = pNode->right; pNode = pNode->right;
}
} else { } else {
pNew->parent = pNode; while (true) {
pNode->right = pNew; if (pNode->parent == pTree->NIL || pNode == pNode->parent->right) {
pNode = pNode->parent;
break; break;
}
} else { } else {
return NULL; pNode = pNode->parent;
} }
} }
} }
// fix return pNode;
SRBTreeNode *pNode = pNew; }
while (pNode->parent && pNode->parent->color == RED) {
SRBTreeNode *p = pNode->parent; void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn) {
SRBTreeNode *g = p->parent; pTree->cmprFn = cmprFn;
pTree->n = 0;
pTree->NIL = &pTree->NILNODE;
pTree->NIL->color = BLACK;
pTree->NIL->parent = NULL;
pTree->NIL->left = NULL;
pTree->NIL->right = NULL;
pTree->root = pTree->NIL;
pTree->min = pTree->NIL;
pTree->max = pTree->NIL;
}
if (p == g->left) { SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) {
SRBTreeNode *u = g->right; SRBTreeNode *y = pTree->NIL; // variable for the parent of the added node
SRBTreeNode *temp = pTree->root;
if (RBTREE_NODE_COLOR(u) == RED) { while (temp != pTree->NIL) {
p->color = BLACK; y = temp;
u->color = BLACK;
g->color = RED; int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp));
pNode = g; if (c < 0) {
temp = temp->left;
} else if (c > 0) {
temp = temp->right;
} else { } else {
if (pNode == p->right) { return NULL;
pNode = p;
tRBTreeRotateLeft(pTree, pNode);
} }
pNode->parent->color = BLACK;
pNode->parent->parent->color = RED;
tRBTreeRotateRight(pTree, pNode->parent->parent);
} }
} else { z->parent = y;
SRBTreeNode *u = g->left;
if (RBTREE_NODE_COLOR(u) == RED) { if (y == pTree->NIL) {
p->color = BLACK; pTree->root = z;
u->color = BLACK; } else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) {
g->color = RED; y->left = z;
} else { } else {
if (pNode == p->left) { y->right = z;
pNode = p;
tRBTreeRotateRight(pTree, pNode);
}
pNode->parent->color = BLACK;
pNode->parent->parent->color = RED;
tRBTreeRotateLeft(pTree, pNode->parent->parent);
} }
z->color = RED;
z->left = pTree->NIL;
z->right = pTree->NIL;
tRBTreePutFix(pTree, z);
// update min/max node
if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) {
pTree->min = z;
} }
if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) {
pTree->max = z;
} }
pTree->n++;
pTree->root->color = BLACK; return z;
return pNew;
} }
SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) { void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z) {
SRBTreeNode *pNode = pTree->root; SRBTreeNode *y = z;
SRBTreeNode *x;
ECOLOR y_orignal_color = y->color;
while (pNode) { // update min/max node
int32_t c = pTree->cmprFn(pKey, pNode->payload); if (pTree->min == z) {
pTree->min = tRBTreeSuccessor(pTree, pTree->min);
}
if (pTree->max == z) {
pTree->max = tRBTreePredecessor(pTree, pTree->max);
}
if (c < 0) { // drop impl
pNode = pNode->left; if (z->left == pTree->NIL) {
} else if (c > 0) { x = z->right;
pNode = pNode->right; tRBTreeTransplant(pTree, z, z->right);
} else if (z->right == pTree->NIL) {
x = z->left;
tRBTreeTransplant(pTree, z, z->left);
} else { } else {
break; y = tRBTreeSuccessor(pTree, z);
y_orignal_color = y->color;
x = y->right;
if (y->parent == z) {
x->parent = z;
} else {
tRBTreeTransplant(pTree, y, y->right);
y->right = z->right;
y->right->parent = y;
} }
tRBTreeTransplant(pTree, z, y);
y->left = z->left;
y->left->parent = y;
y->color = z->color;
}
// fix
if (y_orignal_color == BLACK) {
tRBTreeDropFix(pTree, x);
} }
pTree->n--;
}
SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) {
SRBTreeNode *pNode = tRBTreeGet(pTree, pKey);
if (pNode) { if (pNode) {
// TODO tRBTreeDrop(pTree, pNode);
} }
return pNode; return pNode;
...@@ -194,8 +312,8 @@ SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) { ...@@ -194,8 +312,8 @@ SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) {
SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) {
SRBTreeNode *pNode = pTree->root; SRBTreeNode *pNode = pTree->root;
while (pNode) { while (pNode != pTree->NIL) {
int32_t c = pTree->cmprFn(pKey, pNode->payload); int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode));
if (c < 0) { if (c < 0) {
pNode = pNode->left; pNode = pNode->left;
...@@ -206,5 +324,23 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { ...@@ -206,5 +324,23 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) {
} }
} }
return pNode; return (pNode == pTree->NIL) ? NULL : pNode;
}
// SRBTreeIter ================================================
SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) {
SRBTreeNode *pNode = pIter->pNode;
if (pIter->pNode != pIter->pTree->NIL) {
if (pIter->asc) {
// ascend
pIter->pNode = tRBTreeSuccessor(pIter->pTree, pIter->pNode);
} else {
// descend
pIter->pNode = tRBTreePredecessor(pIter->pTree, pIter->pNode);
}
}
_exit:
return (pNode == pIter->pTree->NIL) ? NULL : pNode;
} }
\ No newline at end of file
source/util/test/CMakeLists.txt
浏览文件 @ 48593015
...@@ -76,3 +76,11 @@ add_test( ...@@ -76,3 +76,11 @@ add_test(
NAME taosbsearchTest NAME taosbsearchTest
COMMAND taosbsearchTest COMMAND taosbsearchTest
) )
# trbtreeTest
add_executable(rbtreeTest "trbtreeTest.cpp")
target_link_libraries(rbtreeTest os util gtest_main)
add_test(
NAME rbtreeTest
COMMAND rbtreeTest
)
\ No newline at end of file
source/util/test/trbtreeTest.cpp 0 → 100644
浏览文件 @ 48593015
#include <gtest/gtest.h>
#include <stdio.h>
#include <stdlib.h>
#include "trbtree.h"
static int32_t tCmprInteger(const void *p1, const void *p2) {
if (*(int *)p1 < *(int *)p2) {
return -1;
} else if (*(int *)p1 > *(int *)p2) {
return 1;
}
return 0;
}
TEST(trbtreeTest, rbtree_test1) {
#if 0
SRBTree rt;
tRBTreeCreate(&rt, tCmprInteger);
int a[] = {1, 3, 4, 2, 7, 5, 8};
for (int i = 0; i < sizeof(a) / sizeof(a[0]); i++) {
SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryMalloc(sizeof(*pNode) + sizeof(int));
*(int *)pNode->payload = a[i];
tRBTreePut(&rt, pNode);
}
SRBTreeIter rti = tRBTreeIterCreate(&rt, 1);
SRBTreeNode *pNode = tRBTreeIterNext(&rti);
int la = 0;
while (pNode) {
GTEST_ASSERT_GT(*(int *)pNode->payload, la);
la = *(int *)pNode->payload;
// printf("%d\n", la);
pNode = tRBTreeIterNext(&rti);
}
#endif
}
\ No newline at end of file
tests/script/tsim/parser/slimit_alter_tags.sim
浏览文件 @ 48593015
...@@ -128,6 +128,7 @@ if $rows != 5 then ...@@ -128,6 +128,7 @@ if $rows != 5 then
return -1 return -1
endi endi
if $data00 != $rowNum then if $data00 != $rowNum then
print expect $rowNum , actual: $data00
return -1 return -1
endi endi
if $data10 != $rowNum then if $data10 != $rowNum then
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或