Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1193
Star 22018
Fork 4786
T
TDengine
提交 3b090271 编写于 作者: S Shengliang Guan
浏览文件
操作

Merge branch '3.0' of https://github.com/taosdata/TDengine into 3.0

上级 881c0e44 db1d8fcd
隐藏空白更改
内联 并排
Showing with 2259 addition and 1592 deletion
include/libs/function/functionMgt.h
浏览文件 @ 3b090271
...@@ -197,6 +197,7 @@ bool fmIsSystemInfoFunc(int32_t funcId); ...@@ -197,6 +197,7 @@ bool fmIsSystemInfoFunc(int32_t funcId);
bool fmIsImplicitTsFunc(int32_t funcId); bool fmIsImplicitTsFunc(int32_t funcId);
bool fmIsClientPseudoColumnFunc(int32_t funcId); bool fmIsClientPseudoColumnFunc(int32_t funcId);
bool fmIsMultiRowsFunc(int32_t funcId); bool fmIsMultiRowsFunc(int32_t funcId);
bool fmIsKeepOrderFunc(int32_t funcId);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc);
......
include/libs/nodes/plannodes.h
浏览文件 @ 3b090271
...@@ -29,9 +29,17 @@ extern "C" { ...@@ -29,9 +29,17 @@ extern "C" {
typedef enum EDataOrderLevel { typedef enum EDataOrderLevel {
DATA_ORDER_LEVEL_NONE = 1, DATA_ORDER_LEVEL_NONE = 1,
DATA_ORDER_LEVEL_IN_BLOCK, DATA_ORDER_LEVEL_IN_BLOCK,
DATA_ORDER_LEVEL_IN_GROUP DATA_ORDER_LEVEL_IN_GROUP,
DATA_ORDER_LEVEL_GLOBAL
} EDataOrderLevel; } EDataOrderLevel;
typedef enum EGroupAction {
GROUP_ACTION_NONE = 1,
GROUP_ACTION_SET,
GROUP_ACTION_KEEP,
GROUP_ACTION_CLEAR
} EGroupAction;
typedef struct SLogicNode { typedef struct SLogicNode {
ENodeType type; ENodeType type;
SNodeList* pTargets; // SColumnNode SNodeList* pTargets; // SColumnNode
...@@ -44,6 +52,7 @@ typedef struct SLogicNode { ...@@ -44,6 +52,7 @@ typedef struct SLogicNode {
SNode* pSlimit; SNode* pSlimit;
EDataOrderLevel requireDataOrder; // requirements for input data EDataOrderLevel requireDataOrder; // requirements for input data
EDataOrderLevel resultDataOrder; // properties of the output data EDataOrderLevel resultDataOrder; // properties of the output data
EGroupAction groupAction;
} SLogicNode; } SLogicNode;
typedef enum EScanType { typedef enum EScanType {
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ 3b090271
...@@ -40,12 +40,9 @@ typedef struct SDelIdx SDelIdx; ...@@ -40,12 +40,9 @@ typedef struct SDelIdx SDelIdx;
typedef struct STbData STbData; typedef struct STbData STbData;
typedef struct SMemTable SMemTable; typedef struct SMemTable SMemTable;
typedef struct STbDataIter STbDataIter; typedef struct STbDataIter STbDataIter;
typedef struct STable STable;
typedef struct SMapData SMapData; typedef struct SMapData SMapData;
typedef struct SBlockIdx SBlockIdx; typedef struct SBlockIdx SBlockIdx;
typedef struct SBlock SBlock; typedef struct SBlock SBlock;
typedef struct SBlockStatis SBlockStatis;
typedef struct SAggrBlkCol SAggrBlkCol;
typedef struct SColData SColData; typedef struct SColData SColData;
typedef struct SBlockDataHdr SBlockDataHdr; typedef struct SBlockDataHdr SBlockDataHdr;
typedef struct SBlockData SBlockData; typedef struct SBlockData SBlockData;
...@@ -62,8 +59,7 @@ typedef struct SDelFReader SDelFReader; ...@@ -62,8 +59,7 @@ typedef struct SDelFReader SDelFReader;
typedef struct SRowIter SRowIter; typedef struct SRowIter SRowIter;
typedef struct STsdbFS STsdbFS; typedef struct STsdbFS STsdbFS;
typedef struct SRowMerger SRowMerger; typedef struct SRowMerger SRowMerger;
typedef struct STsdbFSState STsdbFSState; typedef struct STsdbReadSnap STsdbReadSnap;
typedef struct STsdbSnapHdr STsdbSnapHdr;
#define TSDB_MAX_SUBBLOCKS 8 #define TSDB_MAX_SUBBLOCKS 8
#define TSDB_FHDR_SIZE 512 #define TSDB_FHDR_SIZE 512
...@@ -176,8 +172,6 @@ void tsdbMemTableDestroy(SMemTable *pMemTable); ...@@ -176,8 +172,6 @@ void tsdbMemTableDestroy(SMemTable *pMemTable);
void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData); void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
void tsdbRefMemTable(SMemTable *pMemTable); void tsdbRefMemTable(SMemTable *pMemTable);
void tsdbUnrefMemTable(SMemTable *pMemTable); void tsdbUnrefMemTable(SMemTable *pMemTable);
int32_t tsdbTakeMemSnapshot(STsdb *pTsdb, SMemTable **ppMem, SMemTable **ppIMem);
void tsdbUntakeMemSnapshot(STsdb *pTsdb, SMemTable *pMem, SMemTable *pIMem);
// STbDataIter // STbDataIter
int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
void *tsdbTbDataIterDestroy(STbDataIter *pIter); void *tsdbTbDataIterDestroy(STbDataIter *pIter);
...@@ -188,30 +182,39 @@ bool tsdbTbDataIterNext(STbDataIter *pIter); ...@@ -188,30 +182,39 @@ bool tsdbTbDataIterNext(STbDataIter *pIter);
int32_t tsdbGetNRowsInTbData(STbData *pTbData); int32_t tsdbGetNRowsInTbData(STbData *pTbData);
// tsdbFile.c ============================================================================================== // tsdbFile.c ==============================================================================================
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
void tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]);
bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype);
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile);
int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile);
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet); int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet);
int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet); int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet);
void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]);
void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]);
void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]);
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]);
// SDelFile // SDelFile
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]); void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]);
// tsdbFS.c ============================================================================================== // tsdbFS.c ==============================================================================================
int32_t tsdbFSOpen(STsdb *pTsdb, STsdbFS **ppFS); int32_t tsdbFSOpen(STsdb *pTsdb);
int32_t tsdbFSClose(STsdbFS *pFS); int32_t tsdbFSClose(STsdb *pTsdb);
int32_t tsdbFSBegin(STsdbFS *pFS); int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS);
int32_t tsdbFSCommit(STsdbFS *pFS); void tsdbFSDestroy(STsdbFS *pFS);
int32_t tDFileSetCmprFn(const void *p1, const void *p2);
int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFS);
int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFS);
int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS);
void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS);
int32_t tsdbFSRollback(STsdbFS *pFS); int32_t tsdbFSRollback(STsdbFS *pFS);
int32_t tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile); int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet);
int32_t tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet); int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile);
void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid);
SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState);
SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag);
// tsdbReaderWriter.c ============================================================================================== // tsdbReaderWriter.c ==============================================================================================
// SDataFWriter // SDataFWriter
int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet);
...@@ -222,8 +225,7 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBu ...@@ -222,8 +225,7 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBu
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2, int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2,
SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg); SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg);
SDFileSet *tsdbDataFWriterGetWSet(SDataFWriter *pWriter); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
// SDataFReader // SDataFReader
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet);
int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
...@@ -245,6 +247,9 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb ...@@ -245,6 +247,9 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
int32_t tsdbDelFReaderClose(SDelFReader **ppReader); int32_t tsdbDelFReaderClose(SDelFReader **ppReader);
int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf);
int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf);
// tsdbRead.c ==============================================================================================
int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap);
void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap);
#define TSDB_CACHE_NO(c) ((c).cacheLast == 0) #define TSDB_CACHE_NO(c) ((c).cacheLast == 0)
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
...@@ -276,6 +281,11 @@ typedef struct { ...@@ -276,6 +281,11 @@ typedef struct {
TSKEY minKey; TSKEY minKey;
} SRtn; } SRtn;
struct STsdbFS {
SDelFile *pDelFile;
SArray *aDFileSet; // SArray<SDFileSet>
};
struct STsdb { struct STsdb {
char *path; char *path;
SVnode *pVnode; SVnode *pVnode;
...@@ -283,7 +293,7 @@ struct STsdb { ...@@ -283,7 +293,7 @@ struct STsdb {
TdThreadRwlock rwLock; TdThreadRwlock rwLock;
SMemTable *mem; SMemTable *mem;
SMemTable *imem; SMemTable *imem;
STsdbFS *pFS; STsdbFS fs;
SLRUCache *lruCache; SLRUCache *lruCache;
}; };
...@@ -402,16 +412,6 @@ struct SBlock { ...@@ -402,16 +412,6 @@ struct SBlock {
SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS]; SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS];
}; };
struct SAggrBlkCol {
int16_t colId;
int16_t maxIndex;
int16_t minIndex;
int16_t numOfNull;
int64_t sum;
int64_t max;
int64_t min;
};
struct SColData { struct SColData {
int16_t cid; int16_t cid;
int8_t type; int8_t type;
...@@ -465,12 +465,6 @@ struct SDelIdx { ...@@ -465,12 +465,6 @@ struct SDelIdx {
int64_t size; int64_t size;
}; };
struct SDelFile {
int64_t commitID;
int64_t size;
int64_t offset;
};
#pragma pack(push, 1) #pragma pack(push, 1)
struct SBlockDataHdr { struct SBlockDataHdr {
uint32_t delimiter; uint32_t delimiter;
...@@ -479,34 +473,50 @@ struct SBlockDataHdr { ...@@ -479,34 +473,50 @@ struct SBlockDataHdr {
}; };
#pragma pack(pop) #pragma pack(pop)
struct SDelFile {
volatile int32_t nRef;
int64_t commitID;
int64_t size;
int64_t offset;
};
struct SHeadFile { struct SHeadFile {
volatile int32_t nRef;
int64_t commitID; int64_t commitID;
int64_t size; int64_t size;
int64_t offset; int64_t offset;
}; };
struct SDataFile { struct SDataFile {
volatile int32_t nRef;
int64_t commitID; int64_t commitID;
int64_t size; int64_t size;
}; };
struct SLastFile { struct SLastFile {
volatile int32_t nRef;
int64_t commitID; int64_t commitID;
int64_t size; int64_t size;
}; };
struct SSmaFile { struct SSmaFile {
volatile int32_t nRef;
int64_t commitID; int64_t commitID;
int64_t size; int64_t size;
}; };
struct SDFileSet { struct SDFileSet {
SDiskID diskId; SDiskID diskId;
int32_t fid; int32_t fid;
SHeadFile fHead; SHeadFile *pHeadF;
SDataFile fData; SDataFile *pDataF;
SLastFile fLast; SLastFile *pLastF;
SSmaFile fSma; SSmaFile *pSmaF;
}; };
struct SRowIter { struct SRowIter {
...@@ -521,26 +531,33 @@ struct SRowMerger { ...@@ -521,26 +531,33 @@ struct SRowMerger {
SArray *pArray; // SArray<SColVal> SArray *pArray; // SArray<SColVal>
}; };
struct STsdbFSState {
SDelFile *pDelFile;
SArray *aDFileSet; // SArray<aDFileSet>
SDelFile delFile;
};
struct STsdbFS {
STsdb *pTsdb;
TdThreadRwlock lock;
int8_t inTxn;
STsdbFSState *cState;
STsdbFSState *nState;
};
struct SDelFWriter { struct SDelFWriter {
STsdb *pTsdb; STsdb *pTsdb;
SDelFile fDel; SDelFile fDel;
TdFilePtr pWriteH; TdFilePtr pWriteH;
}; };
struct SDataFWriter {
STsdb *pTsdb;
SDFileSet wSet;
TdFilePtr pHeadFD;
TdFilePtr pDataFD;
TdFilePtr pLastFD;
TdFilePtr pSmaFD;
SHeadFile fHead;
SDataFile fData;
SLastFile fLast;
SSmaFile fSma;
};
struct STsdbReadSnap {
SMemTable *pMem;
SMemTable *pIMem;
STsdbFS fs;
};
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
source/dnode/vnode/src/sma/smaRollup.c
浏览文件 @ 3b090271
...@@ -582,7 +582,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche ...@@ -582,7 +582,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche
int32_t code = qExecTask(pItem->taskInfo, &output, &ts); int32_t code = qExecTask(pItem->taskInfo, &output, &ts);
if (code < 0) { if (code < 0) {
smaError("vgId:%d, qExecTask for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma), suid, smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid,
pItem->level, terrstr(code)); pItem->level, terrstr(code));
goto _err; goto _err;
} }
......
source/dnode/vnode/src/tsdb/tsdbCache.c
浏览文件 @ 3b090271
...@@ -464,7 +464,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { ...@@ -464,7 +464,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
switch (state->state) { switch (state->state) {
case SFSNEXTROW_FS: case SFSNEXTROW_FS:
state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet; // state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet;
state->nFileSet = taosArrayGetSize(state->aDFileSet); state->nFileSet = taosArrayGetSize(state->aDFileSet);
state->iFileSet = state->nFileSet; state->iFileSet = state->nFileSet;
...@@ -793,9 +793,10 @@ typedef struct { ...@@ -793,9 +793,10 @@ typedef struct {
TSDBROW memRow, imemRow, fsRow; TSDBROW memRow, imemRow, fsRow;
TsdbNextRowState input[3]; TsdbNextRowState input[3];
SMemTable *pMemTable; // SMemTable *pMemTable;
SMemTable *pIMemTable; // SMemTable *pIMemTable;
STsdb *pTsdb; STsdbReadSnap *pReadSnap;
STsdb *pTsdb;
} CacheNextRowIter; } CacheNextRowIter;
static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) { static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) {
...@@ -803,16 +804,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs ...@@ -803,16 +804,16 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
tb_uid_t suid = getTableSuidByUid(uid, pTsdb); tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
tsdbTakeMemSnapshot(pTsdb, &pIter->pMemTable, &pIter->pIMemTable); tsdbTakeReadSnap(pTsdb, &pIter->pReadSnap);
STbData *pMem = NULL; STbData *pMem = NULL;
if (pIter->pMemTable) { if (pIter->pReadSnap->pMem) {
tsdbGetTbDataFromMemTable(pIter->pMemTable, suid, uid, &pMem); tsdbGetTbDataFromMemTable(pIter->pReadSnap->pMem, suid, uid, &pMem);
} }
STbData *pIMem = NULL; STbData *pIMem = NULL;
if (pIter->pIMemTable) { if (pIter->pReadSnap->pIMem) {
tsdbGetTbDataFromMemTable(pIter->pIMemTable, suid, uid, &pIMem); tsdbGetTbDataFromMemTable(pIter->pReadSnap->pIMem, suid, uid, &pIMem);
} }
pIter->pTsdb = pTsdb; pIter->pTsdb = pTsdb;
...@@ -821,7 +822,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs ...@@ -821,7 +822,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
SDelIdx delIdx; SDelIdx delIdx;
SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState); SDelFile *pDelFile = pIter->pReadSnap->fs.pDelFile;
if (pDelFile) { if (pDelFile) {
SDelFReader *pDelFReader; SDelFReader *pDelFReader;
...@@ -846,6 +847,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs ...@@ -846,6 +847,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.state = SFSNEXTROW_FS;
pIter->fsState.pTsdb = pTsdb; pIter->fsState.pTsdb = pTsdb;
pIter->fsState.aDFileSet = pIter->pReadSnap->fs.aDFileSet;
pIter->fsState.pBlockIdxExp = &pIter->idx; pIter->fsState.pBlockIdxExp = &pIter->idx;
pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL}; pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL};
...@@ -885,7 +887,7 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) { ...@@ -885,7 +887,7 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) {
taosArrayDestroy(pIter->pSkyline); taosArrayDestroy(pIter->pSkyline);
} }
tsdbUntakeMemSnapshot(pIter->pTsdb, pIter->pMemTable, pIter->pIMemTable); tsdbUntakeReadSnap(pIter->pTsdb, pIter->pReadSnap);
return code; return code;
_err: _err:
...@@ -1172,480 +1174,480 @@ _err: ...@@ -1172,480 +1174,480 @@ _err:
return code; return code;
} }
static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRow) { // static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRow) {
int32_t code = 0; // int32_t code = 0;
SArray *pSkyline = NULL; // SArray *pSkyline = NULL;
STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1); // STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
int16_t nCol = pTSchema->numOfCols; // int16_t nCol = pTSchema->numOfCols;
SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal)); // SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal));
tb_uid_t suid = getTableSuidByUid(uid, pTsdb); // tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
STbData *pMem = NULL; // STbData *pMem = NULL;
if (pTsdb->mem) { // if (pTsdb->mem) {
tsdbGetTbDataFromMemTable(pTsdb->mem, suid, uid, &pMem); // tsdbGetTbDataFromMemTable(pTsdb->mem, suid, uid, &pMem);
} // }
STbData *pIMem = NULL; // STbData *pIMem = NULL;
if (pTsdb->imem) { // if (pTsdb->imem) {
tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem); // tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem);
} // }
*ppRow = NULL; // *ppRow = NULL;
pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); // pSkyline = taosArrayInit(32, sizeof(TSDBKEY));
SDelIdx delIdx; // SDelIdx delIdx;
SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState); // SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState);
if (pDelFile) { // if (pDelFile) {
SDelFReader *pDelFReader; // SDelFReader *pDelFReader;
code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL); // code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL);
if (code) goto _err; // if (code) goto _err;
code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); // code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
if (code) goto _err; // if (code) goto _err;
code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pSkyline); // code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pSkyline);
if (code) goto _err; // if (code) goto _err;
tsdbDelFReaderClose(&pDelFReader); // tsdbDelFReaderClose(&pDelFReader);
} else { // } else {
code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pSkyline); // code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pSkyline);
if (code) goto _err; // if (code) goto _err;
} // }
int64_t iSkyline = taosArrayGetSize(pSkyline) - 1; // int64_t iSkyline = taosArrayGetSize(pSkyline) - 1;
SBlockIdx idx = {.suid = suid, .uid = uid}; // SBlockIdx idx = {.suid = suid, .uid = uid};
SFSNextRowIter fsState = {0}; // SFSNextRowIter fsState = {0};
fsState.state = SFSNEXTROW_FS; // fsState.state = SFSNEXTROW_FS;
fsState.pTsdb = pTsdb; // fsState.pTsdb = pTsdb;
fsState.pBlockIdxExp = &idx; // fsState.pBlockIdxExp = &idx;
SMemNextRowIter memState = {0}; // SMemNextRowIter memState = {0};
SMemNextRowIter imemState = {0}; // SMemNextRowIter imemState = {0};
TSDBROW memRow, imemRow, fsRow; // TSDBROW memRow, imemRow, fsRow;
TsdbNextRowState input[3] = {{&memRow, true, false, &memState, getNextRowFromMem, NULL}, // TsdbNextRowState input[3] = {{&memRow, true, false, &memState, getNextRowFromMem, NULL},
{&imemRow, true, false, &imemState, getNextRowFromMem, NULL}, // {&imemRow, true, false, &imemState, getNextRowFromMem, NULL},
{&fsRow, false, true, &fsState, getNextRowFromFS, clearNextRowFromFS}}; // {&fsRow, false, true, &fsState, getNextRowFromFS, clearNextRowFromFS}};
if (pMem) { // if (pMem) {
memState.pMem = pMem; // memState.pMem = pMem;
memState.state = SMEMNEXTROW_ENTER; // memState.state = SMEMNEXTROW_ENTER;
input[0].stop = false; // input[0].stop = false;
input[0].next = true; // input[0].next = true;
} // }
if (pIMem) { // if (pIMem) {
imemState.pMem = pIMem; // imemState.pMem = pIMem;
imemState.state = SMEMNEXTROW_ENTER; // imemState.state = SMEMNEXTROW_ENTER;
input[1].stop = false; // input[1].stop = false;
input[1].next = true; // input[1].next = true;
} // }
int16_t nilColCount = nCol - 1; // count of null & none cols // int16_t nilColCount = nCol - 1; // count of null & none cols
int iCol = 0; // index of first nil col index from left to right // int iCol = 0; // index of first nil col index from left to right
bool setICol = false; // bool setICol = false;
do { // do {
for (int i = 0; i < 3; ++i) { // for (int i = 0; i < 3; ++i) {
if (input[i].next && !input[i].stop) { // if (input[i].next && !input[i].stop) {
if (input[i].pRow == NULL) { // if (input[i].pRow == NULL) {
code = input[i].nextRowFn(input[i].iter, &input[i].pRow); // code = input[i].nextRowFn(input[i].iter, &input[i].pRow);
if (code) goto _err; // if (code) goto _err;
if (input[i].pRow == NULL) { // if (input[i].pRow == NULL) {
input[i].stop = true; // input[i].stop = true;
input[i].next = false; // input[i].next = false;
} // }
} // }
} // }
} // }
if (input[0].stop && input[1].stop && input[2].stop) { // if (input[0].stop && input[1].stop && input[2].stop) {
break; // break;
} // }
// select maxpoint(s) from mem, imem, fs // // select maxpoint(s) from mem, imem, fs
TSDBROW *max[3] = {0}; // TSDBROW *max[3] = {0};
int iMax[3] = {-1, -1, -1}; // int iMax[3] = {-1, -1, -1};
int nMax = 0; // int nMax = 0;
TSKEY maxKey = TSKEY_MIN; // TSKEY maxKey = TSKEY_MIN;
for (int i = 0; i < 3; ++i) { // for (int i = 0; i < 3; ++i) {
if (!input[i].stop && input[i].pRow != NULL) { // if (!input[i].stop && input[i].pRow != NULL) {
TSDBKEY key = TSDBROW_KEY(input[i].pRow); // TSDBKEY key = TSDBROW_KEY(input[i].pRow);
// merging & deduplicating on client side // // merging & deduplicating on client side
if (maxKey <= key.ts) { // if (maxKey <= key.ts) {
if (maxKey < key.ts) { // if (maxKey < key.ts) {
nMax = 0; // nMax = 0;
maxKey = key.ts; // maxKey = key.ts;
} // }
iMax[nMax] = i; // iMax[nMax] = i;
max[nMax++] = input[i].pRow; // max[nMax++] = input[i].pRow;
} // }
} // }
} // }
// delete detection // // delete detection
TSDBROW *merge[3] = {0}; // TSDBROW *merge[3] = {0};
int iMerge[3] = {-1, -1, -1}; // int iMerge[3] = {-1, -1, -1};
int nMerge = 0; // int nMerge = 0;
for (int i = 0; i < nMax; ++i) { // for (int i = 0; i < nMax; ++i) {
TSDBKEY maxKey = TSDBROW_KEY(max[i]); // TSDBKEY maxKey = TSDBROW_KEY(max[i]);
bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline); // bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline);
if (!deleted) { // if (!deleted) {
iMerge[nMerge] = i; // iMerge[nMerge] = i;
merge[nMerge++] = max[i]; // merge[nMerge++] = max[i];
} // }
input[iMax[i]].next = deleted; // input[iMax[i]].next = deleted;
} // }
// merge if nMerge > 1 // // merge if nMerge > 1
if (nMerge > 0) { // if (nMerge > 0) {
*dup = false; // *dup = false;
if (nMerge == 1) { // if (nMerge == 1) {
code = tsRowFromTsdbRow(pTSchema, merge[nMerge - 1], ppRow); // code = tsRowFromTsdbRow(pTSchema, merge[nMerge - 1], ppRow);
if (code) goto _err; // if (code) goto _err;
} else { // } else {
// merge 2 or 3 rows // // merge 2 or 3 rows
SRowMerger merger = {0}; // SRowMerger merger = {0};
tRowMergerInit(&merger, merge[0], pTSchema); // tRowMergerInit(&merger, merge[0], pTSchema);
for (int i = 1; i < nMerge; ++i) { // for (int i = 1; i < nMerge; ++i) {
tRowMerge(&merger, merge[i]); // tRowMerge(&merger, merge[i]);
} // }
tRowMergerGetRow(&merger, ppRow); // tRowMergerGetRow(&merger, ppRow);
tRowMergerClear(&merger); // tRowMergerClear(&merger);
} // }
} // }
} while (1); // } while (1);
for (int i = 0; i < 3; ++i) { // for (int i = 0; i < 3; ++i) {
if (input[i].nextRowClearFn) { // if (input[i].nextRowClearFn) {
input[i].nextRowClearFn(input[i].iter); // input[i].nextRowClearFn(input[i].iter);
} // }
} // }
if (pSkyline) { // if (pSkyline) {
taosArrayDestroy(pSkyline); // taosArrayDestroy(pSkyline);
} // }
taosMemoryFreeClear(pTSchema); // taosMemoryFreeClear(pTSchema);
return code; // return code;
_err: // _err:
for (int i = 0; i < 3; ++i) { // for (int i = 0; i < 3; ++i) {
if (input[i].nextRowClearFn) { // if (input[i].nextRowClearFn) {
input[i].nextRowClearFn(input[i].iter); // input[i].nextRowClearFn(input[i].iter);
} // }
} // }
if (pSkyline) { // if (pSkyline) {
taosArrayDestroy(pSkyline); // taosArrayDestroy(pSkyline);
} // }
taosMemoryFreeClear(pTSchema); // taosMemoryFreeClear(pTSchema);
tsdbError("vgId:%d merge last_row failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); // tsdbError("vgId:%d merge last_row failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; // return code;
} // }
// static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) { // static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, STSRow **ppRow) {
static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { // static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
int32_t code = 0; // int32_t code = 0;
SArray *pSkyline = NULL; // SArray *pSkyline = NULL;
STSRow *pRow = NULL; // STSRow *pRow = NULL;
STSRow **ppRow = &pRow; // STSRow **ppRow = &pRow;
STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1); // STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1);
int16_t nCol = pTSchema->numOfCols; // int16_t nCol = pTSchema->numOfCols;
// SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal)); // // SArray *pColArray = taosArrayInit(nCol, sizeof(SColVal));
SArray *pColArray = taosArrayInit(nCol, sizeof(SLastCol)); // SArray *pColArray = taosArrayInit(nCol, sizeof(SLastCol));
tb_uid_t suid = getTableSuidByUid(uid, pTsdb); // tb_uid_t suid = getTableSuidByUid(uid, pTsdb);
STbData *pMem = NULL; // STbData *pMem = NULL;
if (pTsdb->mem) { // if (pTsdb->mem) {
tsdbGetTbDataFromMemTable(pTsdb->mem, suid, uid, &pMem); // tsdbGetTbDataFromMemTable(pTsdb->mem, suid, uid, &pMem);
} // }
STbData *pIMem = NULL; // STbData *pIMem = NULL;
if (pTsdb->imem) { // if (pTsdb->imem) {
tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem); // tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem);
} // }
*ppLastArray = NULL; // *ppLastArray = NULL;
pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); // pSkyline = taosArrayInit(32, sizeof(TSDBKEY));
SDelIdx delIdx; // SDelIdx delIdx;
SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState); // SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState);
if (pDelFile) { // if (pDelFile) {
SDelFReader *pDelFReader; // SDelFReader *pDelFReader;
code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL); // code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL);
if (code) goto _err; // if (code) goto _err;
code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); // code = getTableDelIdx(pDelFReader, suid, uid, &delIdx);
if (code) goto _err; // if (code) goto _err;
code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pSkyline); // code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pSkyline);
if (code) goto _err; // if (code) goto _err;
tsdbDelFReaderClose(&pDelFReader); // tsdbDelFReaderClose(&pDelFReader);
} else { // } else {
code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pSkyline); // code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pSkyline);
if (code) goto _err; // if (code) goto _err;
} // }
int64_t iSkyline = taosArrayGetSize(pSkyline) - 1; // int64_t iSkyline = taosArrayGetSize(pSkyline) - 1;
SBlockIdx idx = {.suid = suid, .uid = uid}; // SBlockIdx idx = {.suid = suid, .uid = uid};
SFSNextRowIter fsState = {0}; // SFSNextRowIter fsState = {0};
fsState.state = SFSNEXTROW_FS; // fsState.state = SFSNEXTROW_FS;
fsState.pTsdb = pTsdb; // fsState.pTsdb = pTsdb;
fsState.pBlockIdxExp = &idx; // fsState.pBlockIdxExp = &idx;
SMemNextRowIter memState = {0}; // SMemNextRowIter memState = {0};
SMemNextRowIter imemState = {0}; // SMemNextRowIter imemState = {0};
TSDBROW memRow, imemRow, fsRow; // TSDBROW memRow, imemRow, fsRow;
TsdbNextRowState input[3] = {{&memRow, true, false, &memState, getNextRowFromMem, NULL}, // TsdbNextRowState input[3] = {{&memRow, true, false, &memState, getNextRowFromMem, NULL},
{&imemRow, true, false, &imemState, getNextRowFromMem, NULL}, // {&imemRow, true, false, &imemState, getNextRowFromMem, NULL},
{&fsRow, false, true, &fsState, getNextRowFromFS, clearNextRowFromFS}}; // {&fsRow, false, true, &fsState, getNextRowFromFS, clearNextRowFromFS}};
if (pMem) { // if (pMem) {
memState.pMem = pMem; // memState.pMem = pMem;
memState.state = SMEMNEXTROW_ENTER; // memState.state = SMEMNEXTROW_ENTER;
input[0].stop = false; // input[0].stop = false;
input[0].next = true; // input[0].next = true;
} // }
if (pIMem) { // if (pIMem) {
imemState.pMem = pIMem; // imemState.pMem = pIMem;
imemState.state = SMEMNEXTROW_ENTER; // imemState.state = SMEMNEXTROW_ENTER;
input[1].stop = false; // input[1].stop = false;
input[1].next = true; // input[1].next = true;
} // }
int16_t nilColCount = nCol - 1; // count of null & none cols // int16_t nilColCount = nCol - 1; // count of null & none cols
int iCol = 0; // index of first nil col index from left to right // int iCol = 0; // index of first nil col index from left to right
bool setICol = false; // bool setICol = false;
do { // do {
for (int i = 0; i < 3; ++i) { // for (int i = 0; i < 3; ++i) {
if (input[i].next && !input[i].stop) { // if (input[i].next && !input[i].stop) {
code = input[i].nextRowFn(input[i].iter, &input[i].pRow); // code = input[i].nextRowFn(input[i].iter, &input[i].pRow);
if (code) goto _err; // if (code) goto _err;
if (input[i].pRow == NULL) { // if (input[i].pRow == NULL) {
input[i].stop = true; // input[i].stop = true;
input[i].next = false; // input[i].next = false;
} // }
} // }
} // }
if (input[0].stop && input[1].stop && input[2].stop) { // if (input[0].stop && input[1].stop && input[2].stop) {
break; // break;
} // }
// select maxpoint(s) from mem, imem, fs // // select maxpoint(s) from mem, imem, fs
TSDBROW *max[3] = {0}; // TSDBROW *max[3] = {0};
int iMax[3] = {-1, -1, -1}; // int iMax[3] = {-1, -1, -1};
int nMax = 0; // int nMax = 0;
TSKEY maxKey = TSKEY_MIN; // TSKEY maxKey = TSKEY_MIN;
for (int i = 0; i < 3; ++i) { // for (int i = 0; i < 3; ++i) {
if (!input[i].stop && input[i].pRow != NULL) { // if (!input[i].stop && input[i].pRow != NULL) {
TSDBKEY key = TSDBROW_KEY(input[i].pRow); // TSDBKEY key = TSDBROW_KEY(input[i].pRow);
// merging & deduplicating on client side // // merging & deduplicating on client side
if (maxKey <= key.ts) { // if (maxKey <= key.ts) {
if (maxKey < key.ts) { // if (maxKey < key.ts) {
nMax = 0; // nMax = 0;
maxKey = key.ts; // maxKey = key.ts;
} // }
iMax[nMax] = i; // iMax[nMax] = i;
max[nMax++] = input[i].pRow; // max[nMax++] = input[i].pRow;
} // }
} // }
} // }
// delete detection // // delete detection
TSDBROW *merge[3] = {0}; // TSDBROW *merge[3] = {0};
int iMerge[3] = {-1, -1, -1}; // int iMerge[3] = {-1, -1, -1};
int nMerge = 0; // int nMerge = 0;
for (int i = 0; i < nMax; ++i) { // for (int i = 0; i < nMax; ++i) {
TSDBKEY maxKey = TSDBROW_KEY(max[i]); // TSDBKEY maxKey = TSDBROW_KEY(max[i]);
bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline); // bool deleted = tsdbKeyDeleted(&maxKey, pSkyline, &iSkyline);
if (!deleted) { // if (!deleted) {
iMerge[nMerge] = iMax[i]; // iMerge[nMerge] = iMax[i];
merge[nMerge++] = max[i]; // merge[nMerge++] = max[i];
} // }
input[iMax[i]].next = deleted; // input[iMax[i]].next = deleted;
} // }
// merge if nMerge > 1 // // merge if nMerge > 1
if (nMerge > 0) { // if (nMerge > 0) {
if (nMerge == 1) { // if (nMerge == 1) {
code = tsRowFromTsdbRow(pTSchema, merge[nMerge - 1], ppRow); // code = tsRowFromTsdbRow(pTSchema, merge[nMerge - 1], ppRow);
if (code) goto _err; // if (code) goto _err;
} else { // } else {
// merge 2 or 3 rows // // merge 2 or 3 rows
SRowMerger merger = {0}; // SRowMerger merger = {0};
tRowMergerInit(&merger, merge[0], pTSchema); // tRowMergerInit(&merger, merge[0], pTSchema);
for (int i = 1; i < nMerge; ++i) { // for (int i = 1; i < nMerge; ++i) {
tRowMerge(&merger, merge[i]); // tRowMerge(&merger, merge[i]);
} // }
tRowMergerGetRow(&merger, ppRow); // tRowMergerGetRow(&merger, ppRow);
tRowMergerClear(&merger); // tRowMergerClear(&merger);
} // }
} else { // } else {
/* *ppRow = NULL; */ // /* *ppRow = NULL; */
/* return code; */ // /* return code; */
continue; // continue;
} // }
if (iCol == 0) { // if (iCol == 0) {
STColumn *pTColumn = &pTSchema->columns[0]; // STColumn *pTColumn = &pTSchema->columns[0];
SColVal *pColVal = &(SColVal){0}; // SColVal *pColVal = &(SColVal){0};
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = maxKey}); // *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = maxKey});
// if (taosArrayPush(pColArray, pColVal) == NULL) { // // if (taosArrayPush(pColArray, pColVal) == NULL) {
if (taosArrayPush(pColArray, &(SLastCol){.ts = maxKey, .colVal = *pColVal}) == NULL) { // if (taosArrayPush(pColArray, &(SLastCol){.ts = maxKey, .colVal = *pColVal}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; // code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; // goto _err;
} // }
++iCol; // ++iCol;
setICol = false; // setICol = false;
for (int16_t i = iCol; i < nCol; ++i) { // for (int16_t i = iCol; i < nCol; ++i) {
// tsdbRowGetColVal(*ppRow, pTSchema, i, pColVal); // // tsdbRowGetColVal(*ppRow, pTSchema, i, pColVal);
tTSRowGetVal(*ppRow, pTSchema, i, pColVal); // tTSRowGetVal(*ppRow, pTSchema, i, pColVal);
// if (taosArrayPush(pColArray, pColVal) == NULL) { // // if (taosArrayPush(pColArray, pColVal) == NULL) {
if (taosArrayPush(pColArray, &(SLastCol){.ts = maxKey, .colVal = *pColVal}) == NULL) { // if (taosArrayPush(pColArray, &(SLastCol){.ts = maxKey, .colVal = *pColVal}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; // code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; // goto _err;
} // }
if (pColVal->isNull || pColVal->isNone) { // if (pColVal->isNull || pColVal->isNone) {
for (int j = 0; j < nMerge; ++j) { // for (int j = 0; j < nMerge; ++j) {
SColVal jColVal = {0}; // SColVal jColVal = {0};
tsdbRowGetColVal(merge[j], pTSchema, i, &jColVal); // tsdbRowGetColVal(merge[j], pTSchema, i, &jColVal);
if (jColVal.isNull || jColVal.isNone) { // if (jColVal.isNull || jColVal.isNone) {
input[iMerge[j]].next = true; // input[iMerge[j]].next = true;
} // }
} // }
if (!setICol) { // if (!setICol) {
iCol = i; // iCol = i;
setICol = true; // setICol = true;
} // }
} else { // } else {
--nilColCount; // --nilColCount;
} // }
} // }
if (*ppRow) { // if (*ppRow) {
taosMemoryFreeClear(*ppRow); // taosMemoryFreeClear(*ppRow);
} // }
continue; // continue;
} // }
setICol = false; // setICol = false;
for (int16_t i = iCol; i < nCol; ++i) { // for (int16_t i = iCol; i < nCol; ++i) {
SColVal colVal = {0}; // SColVal colVal = {0};
tTSRowGetVal(*ppRow, pTSchema, i, &colVal); // tTSRowGetVal(*ppRow, pTSchema, i, &colVal);
TSKEY rowTs = (*ppRow)->ts; // TSKEY rowTs = (*ppRow)->ts;
// SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, i); // // SColVal *tColVal = (SColVal *)taosArrayGet(pColArray, i);
SLastCol *tTsVal = (SLastCol *)taosArrayGet(pColArray, i); // SLastCol *tTsVal = (SLastCol *)taosArrayGet(pColArray, i);
SColVal *tColVal = &tTsVal->colVal; // SColVal *tColVal = &tTsVal->colVal;
if (!colVal.isNone && !colVal.isNull) { // if (!colVal.isNone && !colVal.isNull) {
if (tColVal->isNull || tColVal->isNone) { // if (tColVal->isNull || tColVal->isNone) {
// taosArraySet(pColArray, i, &colVal); // // taosArraySet(pColArray, i, &colVal);
taosArraySet(pColArray, i, &(SLastCol){.ts = rowTs, .colVal = colVal}); // taosArraySet(pColArray, i, &(SLastCol){.ts = rowTs, .colVal = colVal});
--nilColCount; // --nilColCount;
} // }
} else { // } else {
if ((tColVal->isNull || tColVal->isNone) && !setICol) { // if ((tColVal->isNull || tColVal->isNone) && !setICol) {
iCol = i; // iCol = i;
setICol = true; // setICol = true;
for (int j = 0; j < nMerge; ++j) { // for (int j = 0; j < nMerge; ++j) {
SColVal jColVal = {0}; // SColVal jColVal = {0};
tsdbRowGetColVal(merge[j], pTSchema, i, &jColVal); // tsdbRowGetColVal(merge[j], pTSchema, i, &jColVal);
if (jColVal.isNull || jColVal.isNone) { // if (jColVal.isNull || jColVal.isNone) {
input[iMerge[j]].next = true; // input[iMerge[j]].next = true;
} // }
} // }
} // }
} // }
} // }
if (*ppRow) { // if (*ppRow) {
taosMemoryFreeClear(*ppRow); // taosMemoryFreeClear(*ppRow);
} // }
} while (nilColCount > 0); // } while (nilColCount > 0);
// if () new ts row from pColArray if non empty // // if () new ts row from pColArray if non empty
/* if (taosArrayGetSize(pColArray) == nCol) { */ // /* if (taosArrayGetSize(pColArray) == nCol) { */
/* code = tdSTSRowNew(pColArray, pTSchema, ppRow); */ // /* code = tdSTSRowNew(pColArray, pTSchema, ppRow); */
/* if (code) goto _err; */ // /* if (code) goto _err; */
/* } */ // /* } */
/* taosArrayDestroy(pColArray); */ // /* taosArrayDestroy(pColArray); */
if (taosArrayGetSize(pColArray) <= 0) { // if (taosArrayGetSize(pColArray) <= 0) {
*ppLastArray = NULL; // *ppLastArray = NULL;
taosArrayDestroy(pColArray); // taosArrayDestroy(pColArray);
} else { // } else {
*ppLastArray = pColArray; // *ppLastArray = pColArray;
} // }
if (*ppRow) { // if (*ppRow) {
taosMemoryFreeClear(*ppRow); // taosMemoryFreeClear(*ppRow);
} // }
for (int i = 0; i < 3; ++i) { // for (int i = 0; i < 3; ++i) {
if (input[i].nextRowClearFn) { // if (input[i].nextRowClearFn) {
input[i].nextRowClearFn(input[i].iter); // input[i].nextRowClearFn(input[i].iter);
} // }
} // }
if (pSkyline) { // if (pSkyline) {
taosArrayDestroy(pSkyline); // taosArrayDestroy(pSkyline);
} // }
taosMemoryFreeClear(pTSchema); // taosMemoryFreeClear(pTSchema);
return code; // return code;
_err: // _err:
taosArrayDestroy(pColArray); // taosArrayDestroy(pColArray);
if (*ppRow) { // if (*ppRow) {
taosMemoryFreeClear(*ppRow); // taosMemoryFreeClear(*ppRow);
} // }
for (int i = 0; i < 3; ++i) { // for (int i = 0; i < 3; ++i) {
if (input[i].nextRowClearFn) { // if (input[i].nextRowClearFn) {
input[i].nextRowClearFn(input[i].iter); // input[i].nextRowClearFn(input[i].iter);
} // }
} // }
if (pSkyline) { // if (pSkyline) {
taosArrayDestroy(pSkyline); // taosArrayDestroy(pSkyline);
} // }
taosMemoryFreeClear(pTSchema); // taosMemoryFreeClear(pTSchema);
tsdbError("vgId:%d merge last_row failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); // tsdbError("vgId:%d merge last_row failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; // return code;
} // }
int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **handle) { int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **handle) {
int32_t code = 0; int32_t code = 0;
......
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ 3b090271
...@@ -29,6 +29,7 @@ typedef struct { ...@@ -29,6 +29,7 @@ typedef struct {
int32_t minRow; int32_t minRow;
int32_t maxRow; int32_t maxRow;
int8_t cmprAlg; int8_t cmprAlg;
STsdbFS fs;
// -------------- // --------------
TSKEY nextKey; // reset by each table commit TSKEY nextKey; // reset by each table commit
int32_t commitFid; int32_t commitFid;
...@@ -119,9 +120,6 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -119,9 +120,6 @@ int32_t tsdbCommit(STsdb *pTsdb) {
code = tsdbCommitDel(&commith); code = tsdbCommitDel(&commith);
if (code) goto _err; if (code) goto _err;
code = tsdbCommitCache(&commith);
if (code) goto _err;
// end commit // end commit
code = tsdbEndCommit(&commith, 0); code = tsdbEndCommit(&commith, 0);
if (code) goto _err; if (code) goto _err;
...@@ -158,7 +156,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { ...@@ -158,7 +156,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) {
goto _err; goto _err;
} }
SDelFile *pDelFileR = pTsdb->pFS->nState->pDelFile; SDelFile *pDelFileR = pCommitter->fs.pDelFile;
if (pDelFileR) { if (pDelFileR) {
code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb, NULL); code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb, NULL);
if (code) goto _err; if (code) goto _err;
...@@ -247,7 +245,7 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { ...@@ -247,7 +245,7 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) {
code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter);
if (code) goto _err; if (code) goto _err;
code = tsdbFSStateUpsertDelFile(pTsdb->pFS->nState, &pCommitter->pDelFWriter->fDel); code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel);
if (code) goto _err; if (code) goto _err;
code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1); code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1);
...@@ -273,7 +271,6 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { ...@@ -273,7 +271,6 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SDFileSet *pRSet = NULL; SDFileSet *pRSet = NULL;
SDFileSet wSet;
// memory // memory
pCommitter->nextKey = TSKEY_MAX; pCommitter->nextKey = TSKEY_MAX;
...@@ -282,7 +279,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { ...@@ -282,7 +279,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
taosArrayClear(pCommitter->aBlockIdx); taosArrayClear(pCommitter->aBlockIdx);
tMapDataReset(&pCommitter->oBlockMap); tMapDataReset(&pCommitter->oBlockMap);
tBlockDataReset(&pCommitter->oBlockData); tBlockDataReset(&pCommitter->oBlockData);
pRSet = tsdbFSStateGetDFileSet(pTsdb->pFS->nState, pCommitter->commitFid, TD_EQ); pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &(SDFileSet){.fid = pCommitter->commitFid},
tDFileSetCmprFn, TD_EQ);
if (pRSet) { if (pRSet) {
code = tsdbDataFReaderOpen(&pCommitter->pReader, pTsdb, pRSet); code = tsdbDataFReaderOpen(&pCommitter->pReader, pTsdb, pRSet);
if (code) goto _err; if (code) goto _err;
...@@ -292,23 +290,29 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { ...@@ -292,23 +290,29 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
} }
// new // new
SHeadFile fHead;
SDataFile fData;
SLastFile fLast;
SSmaFile fSma;
SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma};
taosArrayClear(pCommitter->aBlockIdxN); taosArrayClear(pCommitter->aBlockIdxN);
tMapDataReset(&pCommitter->nBlockMap); tMapDataReset(&pCommitter->nBlockMap);
tBlockDataReset(&pCommitter->nBlockData); tBlockDataReset(&pCommitter->nBlockData);
if (pRSet) { if (pRSet) {
wSet = (SDFileSet){.diskId = pRSet->diskId, wSet.diskId = pRSet->diskId;
.fid = pCommitter->commitFid, wSet.fid = pCommitter->commitFid;
.fHead = {.commitID = pCommitter->commitID, .offset = 0, .size = 0}, fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0};
.fData = pRSet->fData, fData = *pRSet->pDataF;
.fLast = {.commitID = pCommitter->commitID, .size = 0}, fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0};
.fSma = pRSet->fSma}; fSma = *pRSet->pSmaF;
} else { } else {
wSet = (SDFileSet){.diskId = (SDiskID){.level = 0, .id = 0}, wSet.diskId = (SDiskID){.level = 0, .id = 0};
.fid = pCommitter->commitFid, wSet.fid = pCommitter->commitFid;
.fHead = {.commitID = pCommitter->commitID, .offset = 0, .size = 0}, fHead = (SHeadFile){.commitID = pCommitter->commitID, .offset = 0, .size = 0};
.fData = {.commitID = pCommitter->commitID, .size = 0}, fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0};
.fLast = {.commitID = pCommitter->commitID, .size = 0}, fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0};
.fSma = {.commitID = pCommitter->commitID, .size = 0}}; fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0};
} }
code = tsdbDataFWriterOpen(&pCommitter->pWriter, pTsdb, &wSet); code = tsdbDataFWriterOpen(&pCommitter->pWriter, pTsdb, &wSet);
if (code) goto _err; if (code) goto _err;
...@@ -855,7 +859,7 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { ...@@ -855,7 +859,7 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
if (code) goto _err; if (code) goto _err;
// upsert SDFileSet // upsert SDFileSet
code = tsdbFSStateUpsertDFileSet(pCommitter->pTsdb->pFS->nState, tsdbDataFWriterGetWSet(pCommitter->pWriter)); code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->pWriter->wSet);
if (code) goto _err; if (code) goto _err;
// close and sync // close and sync
...@@ -973,7 +977,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { ...@@ -973,7 +977,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
code = tsdbFSBegin(pTsdb->pFS); code = tsdbFSCopy(pTsdb, &pCommitter->fs);
if (code) goto _err; if (code) goto _err;
return code; return code;
...@@ -1142,28 +1146,33 @@ _err: ...@@ -1142,28 +1146,33 @@ _err:
return code; return code;
} }
static int32_t tsdbCommitCache(SCommitter *pCommitter) {
int32_t code = 0;
// TODO
return code;
}
static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem; SMemTable *pMemTable = pTsdb->imem;
if (eno == 0) { ASSERT(eno == 0);
code = tsdbFSCommit(pTsdb->pFS);
} else { code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
code = tsdbFSRollback(pTsdb->pFS); if (code) goto _err;
}
// lock
taosThreadRwlockWrlock(&pTsdb->rwLock); taosThreadRwlockWrlock(&pTsdb->rwLock);
// commit or rollback
code = tsdbFSCommit2(pTsdb, &pCommitter->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _err;
}
pTsdb->imem = NULL; pTsdb->imem = NULL;
// unlock
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&pTsdb->rwLock);
tsdbUnrefMemTable(pMemTable); tsdbUnrefMemTable(pMemTable);
tsdbFSDestroy(&pCommitter->fs);
tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode));
return code; return code;
......
source/dnode/vnode/src/tsdb/tsdbFS.c
浏览文件 @ 3b090271
...@@ -16,67 +16,41 @@ ...@@ -16,67 +16,41 @@
#include "tsdb.h" #include "tsdb.h"
// ================================================================================================= // =================================================================================================
static int32_t tPutFSState(uint8_t *p, STsdbFSState *pState) { static int32_t tsdbEncodeFS(uint8_t *p, STsdbFS *pFS) {
int32_t n = 0; int32_t n = 0;
int8_t hasDel = pState->pDelFile ? 1 : 0; int8_t hasDel = pFS->pDelFile ? 1 : 0;
uint32_t nDFileSet = taosArrayGetSize(pState->aDFileSet); uint32_t nSet = taosArrayGetSize(pFS->aDFileSet);
// SDelFile // SDelFile
n += tPutI8(p ? p + n : p, hasDel); n += tPutI8(p ? p + n : p, hasDel);
if (hasDel) { if (hasDel) {
n += tPutDelFile(p ? p + n : p, pState->pDelFile); n += tPutDelFile(p ? p + n : p, pFS->pDelFile);
} }
// SArray<SDFileSet> // SArray<SDFileSet>
n += tPutU32v(p ? p + n : p, nDFileSet); n += tPutU32v(p ? p + n : p, nSet);
for (uint32_t iDFileSet = 0; iDFileSet < nDFileSet; iDFileSet++) { for (uint32_t iSet = 0; iSet < nSet; iSet++) {
n += tPutDFileSet(p ? p + n : p, (SDFileSet *)taosArrayGet(pState->aDFileSet, iDFileSet)); n += tPutDFileSet(p ? p + n : p, (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet));
} }
return n; return n;
} }
static int32_t tGetFSState(uint8_t *p, STsdbFSState *pState) { static int32_t tsdbGnrtCurrent(STsdb *pTsdb, STsdbFS *pFS, char *fname) {
int32_t n = 0;
int8_t hasDel;
uint32_t nDFileSet;
SDFileSet *pSet = &(SDFileSet){0};
// SDelFile
n += tGetI8(p + n, &hasDel);
if (hasDel) {
pState->pDelFile = &pState->delFile;
n += tGetDelFile(p + n, pState->pDelFile);
} else {
pState->pDelFile = NULL;
}
// SArray<SDFileSet>
taosArrayClear(pState->aDFileSet);
n += tGetU32v(p + n, &nDFileSet);
for (uint32_t iDFileSet = 0; iDFileSet < nDFileSet; iDFileSet++) {
n += tGetDFileSet(p + n, pSet);
taosArrayPush(pState->aDFileSet, pSet);
}
return n;
}
static int32_t tsdbGnrtCurrent(const char *fname, STsdbFSState *pState) {
int32_t code = 0; int32_t code = 0;
int64_t n; int64_t n;
int64_t size; int64_t size;
uint8_t *pData; uint8_t *pData = NULL;
TdFilePtr pFD = NULL; TdFilePtr pFD = NULL;
// to binary // to binary
size = tPutFSState(NULL, pState) + sizeof(TSCKSUM); size = tsdbEncodeFS(NULL, pFS) + sizeof(TSCKSUM);
pData = taosMemoryMalloc(size); pData = taosMemoryMalloc(size);
if (pData == NULL) { if (pData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
n = tPutFSState(pData, pState); n = tsdbEncodeFS(pData, pFS);
ASSERT(n + sizeof(TSCKSUM) == size); ASSERT(n + sizeof(TSCKSUM) == size);
taosCalcChecksumAppend(0, pData, size); taosCalcChecksumAppend(0, pData, size);
...@@ -104,600 +78,1008 @@ static int32_t tsdbGnrtCurrent(const char *fname, STsdbFSState *pState) { ...@@ -104,600 +78,1008 @@ static int32_t tsdbGnrtCurrent(const char *fname, STsdbFSState *pState) {
return code; return code;
_err: _err:
tsdbError("tsdb gnrt current failed since %s", tstrerror(code)); tsdbError("vgId:%d tsdb gnrt current failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
if (pData) taosMemoryFree(pData); if (pData) taosMemoryFree(pData);
return code; return code;
} }
static int32_t tsdbLoadCurrentState(STsdbFS *pFS, STsdbFSState *pState) { // static int32_t tsdbApplyDFileSetChange(STsdbFS *pFS, SDFileSet *pFrom, SDFileSet *pTo) {
int32_t code = 0; // int32_t code = 0;
int64_t size; // char fname[TSDB_FILENAME_LEN];
int64_t n;
char fname[TSDB_FILENAME_LEN]; // if (pFrom && pTo) {
uint8_t *pData = NULL; // bool isSameDisk = (pFrom->diskId.level == pTo->diskId.level) && (pFrom->diskId.id == pTo->diskId.id);
TdFilePtr pFD;
// // head
// if (isSameDisk && pFrom->pHeadF->commitID == pTo->pHeadF->commitID) {
// ASSERT(pFrom->pHeadF->size == pTo->pHeadF->size);
// ASSERT(pFrom->pHeadF->offset == pTo->pHeadF->offset);
// } else {
// tsdbHeadFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pHeadF, fname);
// taosRemoveFile(fname);
// }
// // data
// if (isSameDisk && pFrom->pDataF->commitID == pTo->pDataF->commitID) {
// if (pFrom->pDataF->size > pTo->pDataF->size) {
// code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_DATA_FILE);
// if (code) goto _err;
// }
// } else {
// tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname);
// taosRemoveFile(fname);
// }
// // last
// if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) {
// if (pFrom->pLastF->size > pTo->pLastF->size) {
// code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE);
// if (code) goto _err;
// }
// } else {
// tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname);
// taosRemoveFile(fname);
// }
// // sma
// if (isSameDisk && pFrom->pSmaF->commitID == pTo->pSmaF->commitID) {
// if (pFrom->pSmaF->size > pTo->pSmaF->size) {
// code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_SMA_FILE);
// if (code) goto _err;
// }
// } else {
// tsdbSmaFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pSmaF, fname);
// taosRemoveFile(fname);
// }
// } else if (pFrom) {
// // head
// tsdbHeadFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pHeadF, fname);
// taosRemoveFile(fname);
// // data
// tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname);
// taosRemoveFile(fname);
// // last
// tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname);
// taosRemoveFile(fname);
// // fsm
// tsdbSmaFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pSmaF, fname);
// taosRemoveFile(fname);
// }
// return code;
// _err:
// tsdbError("vgId:%d tsdb apply disk file set change failed since %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code));
// return code;
// }
// static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *pTo) {
// int32_t code = 0;
// char fname[TSDB_FILENAME_LEN];
// if (pFrom && pTo) {
// if (!tsdbDelFileIsSame(pFrom, pTo)) {
// tsdbDelFileName(pFS->pTsdb, pFrom, fname);
// if (taosRemoveFile(fname) < 0) {
// code = TAOS_SYSTEM_ERROR(errno);
// goto _err;
// }
// }
// } else if (pFrom) {
// tsdbDelFileName(pFS->pTsdb, pFrom, fname);
// if (taosRemoveFile(fname) < 0) {
// code = TAOS_SYSTEM_ERROR(errno);
// goto _err;
// }
// } else {
// // do nothing
// }
// return code;
// _err:
// tsdbError("vgId:%d tsdb apply del file change failed since %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code));
// return code;
// }
// static int32_t tsdbFSApplyDiskChange(STsdbFS *pFS, STsdbFSState *pFrom, STsdbFSState *pTo) {
// int32_t code = 0;
// int32_t iFrom = 0;
// int32_t nFrom = taosArrayGetSize(pFrom->aDFileSet);
// int32_t iTo = 0;
// int32_t nTo = taosArrayGetSize(pTo->aDFileSet);
// SDFileSet *pDFileSetFrom;
// SDFileSet *pDFileSetTo;
// // SDelFile
// code = tsdbApplyDelFileChange(pFS, pFrom->pDelFile, pTo->pDelFile);
// if (code) goto _err;
// // SDFileSet
// while (iFrom < nFrom && iTo < nTo) {
// pDFileSetFrom = (SDFileSet *)taosArrayGet(pFrom->aDFileSet, iFrom);
// pDFileSetTo = (SDFileSet *)taosArrayGet(pTo->aDFileSet, iTo);
// if (pDFileSetFrom->fid == pDFileSetTo->fid) {
// code = tsdbApplyDFileSetChange(pFS, pDFileSetFrom, pDFileSetTo);
// if (code) goto _err;
// iFrom++;
// iTo++;
// } else if (pDFileSetFrom->fid < pDFileSetTo->fid) {
// code = tsdbApplyDFileSetChange(pFS, pDFileSetFrom, NULL);
// if (code) goto _err;
// iFrom++;
// } else {
// iTo++;
// }
// }
// while (iFrom < nFrom) {
// pDFileSetFrom = (SDFileSet *)taosArrayGet(pFrom->aDFileSet, iFrom);
// code = tsdbApplyDFileSetChange(pFS, pDFileSetFrom, NULL);
// if (code) goto _err;
// iFrom++;
// }
// #if 0
// // do noting
// while (iTo < nTo) {
// pDFileSetTo = (SDFileSet *)taosArrayGetP(pTo->aDFileSet, iTo);
// code = tsdbApplyDFileSetChange(pFS, NULL, pDFileSetTo);
// if (code) goto _err;
// iTo++;
// }
// #endif
// return code;
// _err:
// tsdbError("vgId:%d tsdb fs apply disk change failed sicne %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code));
// return code;
// }
void tsdbFSDestroy(STsdbFS *pFS) {
if (pFS->pDelFile) {
taosMemoryFree(pFS->pDelFile);
}
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT", tfsGetPrimaryPath(pFS->pTsdb->pVnode->pTfs), TD_DIRSEP, for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) {
pFS->pTsdb->path, TD_DIRSEP); SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet);
taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF);
taosMemoryFree(pSet->pLastF);
taosMemoryFree(pSet->pSmaF);
}
if (!taosCheckExistFile(fname)) { taosArrayDestroy(pFS->aDFileSet);
// create an empry CURRENT file if not exists }
code = tsdbGnrtCurrent(fname, pState);
if (code) goto _err; static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
} else { int32_t code = 0;
// open the file and load int64_t size;
pFD = taosOpenFile(fname, TD_FILE_READ); char fname[TSDB_FILENAME_LEN];
if (pFD == NULL) {
// SDelFile
if (pTsdb->fs.pDelFile) {
tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname);
if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (taosFStatFile(pFD, &size, NULL) < 0) { if (size != pTsdb->fs.pDelFile->size) {
code = TAOS_SYSTEM_ERROR(errno); code = TSDB_CODE_FILE_CORRUPTED;
goto _err; goto _err;
} }
}
pData = taosMemoryMalloc(size); // SArray<SDFileSet>
if (pData == NULL) { for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) {
code = TSDB_CODE_OUT_OF_MEMORY; SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
// head =========
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (size != pSet->pHeadF->size) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _err; goto _err;
} }
n = taosReadFile(pFD, pData, size); // data =========
if (n < 0) { tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (size < pSet->pDataF->size) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _err;
} else if (size > pSet->pDataF->size) {
code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE);
if (code) goto _err;
}
if (!taosCheckChecksumWhole(pData, size)) { // last ===========
tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname);
if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (size != pSet->pLastF->size) {
code = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
goto _err; goto _err;
} }
taosCloseFile(&pFD); // sma =============
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (size < pSet->pSmaF->size) {
code = TSDB_CODE_FILE_CORRUPTED;
goto _err;
} else if (size > pSet->pSmaF->size) {
code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE);
if (code) goto _err;
}
}
// decode {
tGetFSState(pData, pState); // remove those invalid files (todo)
} }
if (pData) taosMemoryFree(pData);
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb load current state failed since %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb scan and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
if (pData) taosMemoryFree(pData);
return code; return code;
} }
static int32_t tsdbApplyDFileSetChange(STsdbFS *pFS, SDFileSet *pFrom, SDFileSet *pTo) { int32_t tDFileSetCmprFn(const void *p1, const void *p2) {
int32_t code = 0; if (((SDFileSet *)p1)->fid < ((SDFileSet *)p2)->fid) {
char fname[TSDB_FILENAME_LEN]; return -1;
} else if (((SDFileSet *)p1)->fid > ((SDFileSet *)p2)->fid) {
return 1;
}
return 0;
}
static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) {
int32_t code = 0;
int8_t hasDel;
uint32_t nSet;
int32_t n;
// SDelFile
n = 0;
n += tGetI8(pData + n, &hasDel);
if (hasDel) {
pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));
if (pTsdb->fs.pDelFile == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pTsdb->fs.pDelFile->nRef = 1;
n += tGetDelFile(pData + n, pTsdb->fs.pDelFile);
} else {
pTsdb->fs.pDelFile = NULL;
}
// SArray<SDFileSet>
taosArrayClear(pTsdb->fs.aDFileSet);
n += tGetU32v(pData + n, &nSet);
for (uint32_t iSet = 0; iSet < nSet; iSet++) {
SDFileSet fSet;
if (pFrom && pTo) {
// head // head
if (tsdbFileIsSame(pFrom, pTo, TSDB_HEAD_FILE)) { fSet.pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile));
ASSERT(pFrom->fHead.size == pTo->fHead.size); if (fSet.pHeadF == NULL) {
ASSERT(pFrom->fHead.offset == pTo->fHead.offset); code = TSDB_CODE_OUT_OF_MEMORY;
} else { goto _err;
tsdbDataFileName(pFS->pTsdb, pFrom, TSDB_HEAD_FILE, fname);
taosRemoveFile(fname);
} }
fSet.pHeadF->nRef = 1;
// data // data
if (tsdbFileIsSame(pFrom, pTo, TSDB_DATA_FILE)) { fSet.pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile));
if (pFrom->fData.size > pTo->fData.size) { if (fSet.pDataF == NULL) {
code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_DATA_FILE); code = TSDB_CODE_OUT_OF_MEMORY;
if (code) goto _err; goto _err;
}
} else {
tsdbDataFileName(pFS->pTsdb, pFrom, TSDB_DATA_FILE, fname);
taosRemoveFile(fname);
} }
fSet.pDataF->nRef = 1;
// last // last
if (tsdbFileIsSame(pFrom, pTo, TSDB_LAST_FILE)) { fSet.pLastF = (SLastFile *)taosMemoryCalloc(1, sizeof(SLastFile));
if (pFrom->fLast.size > pTo->fLast.size) { if (fSet.pLastF == NULL) {
code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE); code = TSDB_CODE_OUT_OF_MEMORY;
if (code) goto _err; goto _err;
}
} else {
tsdbDataFileName(pFS->pTsdb, pFrom, TSDB_LAST_FILE, fname);
taosRemoveFile(fname);
} }
fSet.pLastF->nRef = 1;
// sma // sma
if (tsdbFileIsSame(pFrom, pTo, TSDB_SMA_FILE)) { fSet.pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile));
if (pFrom->fSma.size > pTo->fSma.size) { if (fSet.pSmaF == NULL) {
code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_SMA_FILE); code = TSDB_CODE_OUT_OF_MEMORY;
if (code) goto _err; goto _err;
}
} else {
tsdbDataFileName(pFS->pTsdb, pFrom, TSDB_SMA_FILE, fname);
taosRemoveFile(fname);
} }
} else if (pFrom) { fSet.pSmaF->nRef = 1;
// head
tsdbDataFileName(pFS->pTsdb, pFrom, TSDB_HEAD_FILE, fname);
taosRemoveFile(fname);
// data
tsdbDataFileName(pFS->pTsdb, pFrom, TSDB_DATA_FILE, fname);
taosRemoveFile(fname);
// last n += tGetDFileSet(pData + n, &fSet);
tsdbDataFileName(pFS->pTsdb, pFrom, TSDB_LAST_FILE, fname);
taosRemoveFile(fname);
// fsm if (taosArrayPush(pTsdb->fs.aDFileSet, &fSet) == NULL) {
tsdbDataFileName(pFS->pTsdb, pFrom, TSDB_SMA_FILE, fname); code = TSDB_CODE_OUT_OF_MEMORY;
taosRemoveFile(fname); goto _err;
}
} }
ASSERT(n + sizeof(TSCKSUM) == nData);
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb apply disk file set change failed since %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *pTo) { // EXPOSED APIS ====================================================================================
int32_t tsdbFSOpen(STsdb *pTsdb) {
int32_t code = 0; int32_t code = 0;
char fname[TSDB_FILENAME_LEN];
if (pFrom && pTo) { // open handle
if (!tsdbDelFileIsSame(pFrom, pTo)) { pTsdb->fs.pDelFile = NULL;
tsdbDelFileName(pFS->pTsdb, pFrom, fname); pTsdb->fs.aDFileSet = taosArrayInit(0, sizeof(SDFileSet));
if (taosRemoveFile(fname) < 0) { if (pTsdb->fs.aDFileSet == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
}
}
} else if (pFrom) {
tsdbDelFileName(pFS->pTsdb, pFrom, fname);
if (taosRemoveFile(fname) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
} else {
// do nothing
} }
return code; // load fs or keep empty
char fname[TSDB_FILENAME_LEN];
_err: snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP,
tsdbError("vgId:%d tsdb apply del file change failed since %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code)); pTsdb->path, TD_DIRSEP);
return code;
}
static int32_t tsdbFSApplyDiskChange(STsdbFS *pFS, STsdbFSState *pFrom, STsdbFSState *pTo) {
int32_t code = 0;
int32_t iFrom = 0;
int32_t nFrom = taosArrayGetSize(pFrom->aDFileSet);
int32_t iTo = 0;
int32_t nTo = taosArrayGetSize(pTo->aDFileSet);
SDFileSet *pDFileSetFrom;
SDFileSet *pDFileSetTo;
// SDelFile
code = tsdbApplyDelFileChange(pFS, pFrom->pDelFile, pTo->pDelFile);
if (code) goto _err;
// SDFileSet if (!taosCheckExistFile(fname)) {
while (iFrom < nFrom && iTo < nTo) { // empty one
pDFileSetFrom = (SDFileSet *)taosArrayGet(pFrom->aDFileSet, iFrom); code = tsdbGnrtCurrent(pTsdb, &pTsdb->fs, fname);
pDFileSetTo = (SDFileSet *)taosArrayGet(pTo->aDFileSet, iTo); if (code) goto _err;
} else {
// read
TdFilePtr pFD = taosOpenFile(fname, TD_FILE_READ);
if (pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (pDFileSetFrom->fid == pDFileSetTo->fid) { int64_t size;
code = tsdbApplyDFileSetChange(pFS, pDFileSetFrom, pDFileSetTo); if (taosFStatFile(pFD, &size, NULL) < 0) {
if (code) goto _err; code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFD);
goto _err;
}
iFrom++; uint8_t *pData = taosMemoryMalloc(size);
iTo++; if (pData == NULL) {
} else if (pDFileSetFrom->fid < pDFileSetTo->fid) { code = TSDB_CODE_OUT_OF_MEMORY;
code = tsdbApplyDFileSetChange(pFS, pDFileSetFrom, NULL); taosCloseFile(&pFD);
if (code) goto _err; goto _err;
}
iFrom++; int64_t n = taosReadFile(pFD, pData, size);
} else { if (n < 0) {
iTo++; code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pData);
taosCloseFile(&pFD);
goto _err;
} }
}
while (iFrom < nFrom) { if (!taosCheckChecksumWhole(pData, size)) {
pDFileSetFrom = (SDFileSet *)taosArrayGet(pFrom->aDFileSet, iFrom); code = TSDB_CODE_FILE_CORRUPTED;
code = tsdbApplyDFileSetChange(pFS, pDFileSetFrom, NULL); taosMemoryFree(pData);
if (code) goto _err; taosCloseFile(&pFD);
goto _err;
}
iFrom++; taosCloseFile(&pFD);
}
#if 0 // recover fs
// do noting code = tsdbRecoverFS(pTsdb, pData, size);
while (iTo < nTo) { if (code) {
pDFileSetTo = (SDFileSet *)taosArrayGetP(pTo->aDFileSet, iTo); taosMemoryFree(pData);
code = tsdbApplyDFileSetChange(pFS, NULL, pDFileSetTo); goto _err;
if (code) goto _err; }
iTo++; taosMemoryFree(pData);
} }
#endif
// scan and fix FS
code = tsdbScanAndTryFixFS(pTsdb);
if (code) goto _err;
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb fs apply disk change failed sicne %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb fs open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static void tsdbFSDestroy(STsdbFS *pFS) { int32_t tsdbFSClose(STsdb *pTsdb) {
if (pFS) { int32_t code = 0;
if (pFS->nState) {
taosArrayDestroy(pFS->nState->aDFileSet);
taosMemoryFree(pFS->nState);
}
if (pFS->cState) {
taosArrayDestroy(pFS->cState->aDFileSet);
taosMemoryFree(pFS->cState);
}
taosThreadRwlockDestroy(&pFS->lock); if (pTsdb->fs.pDelFile) {
taosMemoryFree(pFS); ASSERT(pTsdb->fs.pDelFile->nRef == 1);
taosMemoryFree(pTsdb->fs.pDelFile);
} }
// TODO
}
static int32_t tsdbFSCreate(STsdb *pTsdb, STsdbFS **ppFS) { for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) {
int32_t code = 0; SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
STsdbFS *pFS = NULL;
pFS = (STsdbFS *)taosMemoryCalloc(1, sizeof(*pFS)); // head
if (pFS == NULL) { ASSERT(pSet->pHeadF->nRef == 1);
code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pSet->pHeadF);
goto _err;
}
pFS->pTsdb = pTsdb;
code = taosThreadRwlockInit(&pFS->lock, NULL);
if (code) {
taosMemoryFree(pFS);
code = TAOS_SYSTEM_ERROR(code);
goto _err;
}
pFS->inTxn = 0; // data
ASSERT(pSet->pDataF->nRef == 1);
taosMemoryFree(pSet->pDataF);
pFS->cState = (STsdbFSState *)taosMemoryCalloc(1, sizeof(STsdbFSState)); // last
if (pFS->cState == NULL) { ASSERT(pSet->pLastF->nRef == 1);
code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pSet->pLastF);
goto _err;
}
pFS->cState->aDFileSet = taosArrayInit(0, sizeof(SDFileSet));
if (pFS->cState->aDFileSet == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pFS->nState = (STsdbFSState *)taosMemoryCalloc(1, sizeof(STsdbFSState)); // sma
if (pFS->nState == NULL) { ASSERT(pSet->pSmaF->nRef == 1);
code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pSet->pSmaF);
goto _err;
}
pFS->nState->aDFileSet = taosArrayInit(0, sizeof(SDFileSet));
if (pFS->nState->aDFileSet == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
} }
*ppFS = pFS; taosArrayDestroy(pTsdb->fs.aDFileSet);
return code;
_err:
tsdbError("vgId:%d tsdb fs create failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
tsdbFSDestroy(pFS);
*ppFS = NULL;
return code; return code;
} }
static int32_t tsdbScanAndTryFixFS(STsdbFS *pFS, int8_t deepScan) { int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
int32_t code = 0; int32_t code = 0;
STsdb *pTsdb = pFS->pTsdb;
STfs *pTfs = pTsdb->pVnode->pTfs;
int64_t size;
char fname[TSDB_FILENAME_LEN];
char pHdr[TSDB_FHDR_SIZE];
TdFilePtr pFD;
// SDelFile pFS->pDelFile = NULL;
if (pFS->cState->pDelFile) { pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet));
tsdbDelFileName(pTsdb, pFS->cState->pDelFile, fname); if (pFS->aDFileSet == NULL) {
if (taosStatFile(fname, &size, NULL)) { code = TSDB_CODE_OUT_OF_MEMORY;
code = TAOS_SYSTEM_ERROR(errno); goto _exit;
goto _err; }
}
if (size != pFS->cState->pDelFile->size) { if (pTsdb->fs.pDelFile) {
code = TSDB_CODE_FILE_CORRUPTED; pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));
goto _err; if (pFS->pDelFile == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
if (deepScan) { *pFS->pDelFile = *pTsdb->fs.pDelFile;
// TODO
}
} }
// SArray<SDFileSet> for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) {
for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->cState->aDFileSet); iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pFS->cState->aDFileSet, iSet); SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid};
// head ========= // head
tsdbDataFileName(pTsdb, pDFileSet, TSDB_HEAD_FILE, fname); fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
if (taosStatFile(fname, &size, NULL)) { if (fSet.pHeadF == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _exit;
} }
fSet.pHeadF->nRef = 0;
fSet.pHeadF->commitID = pSet->pHeadF->commitID;
fSet.pHeadF->size = pSet->pHeadF->size;
fSet.pHeadF->offset = pSet->pHeadF->offset;
if (deepScan) { // data
// TODO fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile));
if (fSet.pDataF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
fSet.pDataF->nRef = 0;
fSet.pDataF->commitID = pSet->pDataF->commitID;
fSet.pDataF->size = pSet->pDataF->size;
// data ========= // data
tsdbDataFileName(pTsdb, pDFileSet, TSDB_DATA_FILE, fname); fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
if (taosStatFile(fname, &size, NULL)) { if (fSet.pLastF == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _exit;
} }
fSet.pLastF->nRef = 0;
fSet.pLastF->commitID = pSet->pLastF->commitID;
fSet.pLastF->size = pSet->pLastF->size;
if (size < pDFileSet->fData.size) { // last
code = TSDB_CODE_FILE_CORRUPTED; fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
goto _err; if (fSet.pSmaF == NULL) {
} else if (size > pDFileSet->fData.size) { code = TSDB_CODE_OUT_OF_MEMORY;
ASSERT(0); goto _exit;
// need to rollback the file
} }
fSet.pSmaF->nRef = 0;
fSet.pSmaF->commitID = pSet->pSmaF->commitID;
fSet.pSmaF->size = pSet->pSmaF->size;
if (deepScan) { if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
// TODO code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
}
// last =========== _exit:
tsdbDataFileName(pTsdb, pDFileSet, TSDB_LAST_FILE, fname); return code;
if (taosStatFile(fname, &size, NULL)) { }
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (size < pDFileSet->fLast.size) { int32_t tsdbFSRollback(STsdbFS *pFS) {
code = TSDB_CODE_FILE_CORRUPTED; int32_t code = 0;
goto _err;
} else if (size > pDFileSet->fLast.size) {
ASSERT(0);
// need to rollback the file
}
if (deepScan) { ASSERT(0);
// TODO
}
// sma ============= return code;
tsdbDataFileName(pTsdb, pDFileSet, TSDB_SMA_FILE, fname);
if (taosStatFile(fname, &size, NULL)) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (size < pDFileSet->fSma.size) { _err:
code = TSDB_CODE_FILE_CORRUPTED; return code;
goto _err; }
} else if (size > pDFileSet->fSma.size) {
ASSERT(0);
// need to rollback the file
}
if (deepScan) { int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) {
// TODO int32_t code = 0;
if (pFS->pDelFile == NULL) {
pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));
if (pFS->pDelFile == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
} }
*pFS->pDelFile = *pDelFile;
// remove those invalid files (todo) _exit:
#if 0 return code;
STfsDir *tdir; }
const STfsFile *pf;
tdir = tfsOpendir(pTfs, pTsdb->path); int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
if (tdir == NULL) { int32_t code = 0;
code = TAOS_SYSTEM_ERROR(errno); int32_t idx = taosArraySearchIdx(pFS->aDFileSet, pSet, tDFileSetCmprFn, TD_GE);
goto _err;
if (idx < 0) {
idx = taosArrayGetSize(pFS->aDFileSet);
} else {
SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, idx);
int32_t c = tDFileSetCmprFn(pSet, pDFileSet);
if (c == 0) {
*pDFileSet->pHeadF = *pSet->pHeadF;
*pDFileSet->pDataF = *pSet->pDataF;
*pDFileSet->pLastF = *pSet->pLastF;
*pDFileSet->pSmaF = *pSet->pSmaF;
goto _exit;
}
} }
while ((pf = tfsReaddir(tdir))) { SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid};
tfsBasename(pf, fname);
// head
fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
if (fSet.pHeadF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
} }
*fSet.pHeadF = *pSet->pHeadF;
tfsClosedir(tdir); // data
#endif fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile));
if (fSet.pDataF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
*fSet.pDataF = *pSet->pDataF;
return code; // data
fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
if (fSet.pLastF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
*fSet.pLastF = *pSet->pLastF;
_err: // last
tsdbError("vgId:%d tsdb scan and try fix fs failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
return code; if (fSet.pSmaF == NULL) {
} code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
*fSet.pSmaF = *pSet->pSmaF;
static int32_t tDFileSetCmprFn(const void *p1, const void *p2) { if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) {
if (((SDFileSet *)p1)->fid < ((SDFileSet *)p2)->fid) { code = TSDB_CODE_OUT_OF_MEMORY;
return -1; goto _exit;
} else if (((SDFileSet *)p1)->fid > ((SDFileSet *)p2)->fid) {
return 1;
} }
return 0; _exit:
return code;
} }
// EXPOSED APIS ==================================================================================== int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFSNew) {
int32_t tsdbFSOpen(STsdb *pTsdb, STsdbFS **ppFS) {
int32_t code = 0; int32_t code = 0;
char tfname[TSDB_FILENAME_LEN];
char fname[TSDB_FILENAME_LEN];
// create handle snprintf(tfname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT.t", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP,
code = tsdbFSCreate(pTsdb, ppFS); pTsdb->path, TD_DIRSEP);
if (code) goto _err; snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT", tfsGetPrimaryPath(pTsdb->pVnode->pTfs), TD_DIRSEP,
pTsdb->path, TD_DIRSEP);
// load current state // gnrt CURRENT.t
code = tsdbLoadCurrentState(*ppFS, (*ppFS)->cState); code = tsdbGnrtCurrent(pTsdb, pFSNew, tfname);
if (code) { if (code) goto _err;
tsdbFSDestroy(*ppFS);
goto _err;
}
// scan and fix FS // rename
code = tsdbScanAndTryFixFS(*ppFS, 0); code = taosRenameFile(tfname, fname);
if (code) { if (code) {
tsdbFSDestroy(*ppFS); code = TAOS_SYSTEM_ERROR(code);
goto _err; goto _err;
} }
return code; return code;
_err: _err:
*ppFS = NULL; tsdbError("vgId:%d tsdb fs commit phase 1 failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
tsdbError("vgId:%d tsdb fs open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
int32_t tsdbFSClose(STsdbFS *pFS) { int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
int32_t code = 0; int32_t code = 0;
tsdbFSDestroy(pFS); int32_t nRef;
return code; char fname[TSDB_FILENAME_LEN];
}
int32_t tsdbFSBegin(STsdbFS *pFS) { // del
int32_t code = 0; if (pFSNew->pDelFile) {
SDelFile *pDelFile = pTsdb->fs.pDelFile;
ASSERT(!pFS->inTxn); if (pDelFile == NULL || (pDelFile->commitID != pFSNew->pDelFile->commitID)) {
pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));
if (pTsdb->fs.pDelFile == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
// SDelFile *pTsdb->fs.pDelFile = *pFSNew->pDelFile;
pFS->nState->pDelFile = NULL; pTsdb->fs.pDelFile->nRef = 1;
if (pFS->cState->pDelFile) {
pFS->nState->delFile = pFS->cState->delFile; if (pDelFile) {
pFS->nState->pDelFile = &pFS->nState->delFile; nRef = atomic_sub_fetch_32(&pDelFile->nRef, 1);
if (nRef == 0) {
tsdbDelFileName(pTsdb, pDelFile, fname);
taosRemoveFile(fname);
taosMemoryFree(pDelFile);
}
}
}
} else {
ASSERT(pTsdb->fs.pDelFile == NULL);
} }
// SArray<aDFileSet> // data
taosArrayClear(pFS->nState->aDFileSet); int32_t iOld = 0;
for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->cState->aDFileSet); iSet++) { int32_t iNew = 0;
SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pFS->cState->aDFileSet, iSet); while (true) {
int32_t nOld = taosArrayGetSize(pTsdb->fs.aDFileSet);
int32_t nNew = taosArrayGetSize(pFSNew->aDFileSet);
SDFileSet fSet;
int8_t sameDisk;
if (iOld >= nOld && iNew >= nNew) break;
SDFileSet *pSetOld = (iOld < nOld) ? taosArrayGet(pTsdb->fs.aDFileSet, iOld) : NULL;
SDFileSet *pSetNew = (iNew < nNew) ? taosArrayGet(pFSNew->aDFileSet, iNew) : NULL;
if (pSetOld && pSetNew) {
if (pSetOld->fid == pSetNew->fid) {
goto _merge_old_and_new;
} else if (pSetOld->fid < pSetNew->fid) {
goto _remove_old;
} else {
goto _add_new;
}
} else if (pSetOld) {
goto _remove_old;
} else {
goto _add_new;
}
if (taosArrayPush(pFS->nState->aDFileSet, pDFileSet) == NULL) { _merge_old_and_new:
code = TSDB_CODE_OUT_OF_MEMORY; sameDisk = ((pSetOld->diskId.level == pSetNew->diskId.level) && (pSetOld->diskId.id == pSetNew->diskId.id));
goto _err;
// head
fSet.pHeadF = pSetOld->pHeadF;
if ((!sameDisk) || (pSetOld->pHeadF->commitID != pSetNew->pHeadF->commitID)) {
pSetOld->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
if (pSetOld->pHeadF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->pHeadF = *pSetNew->pHeadF;
pSetOld->pHeadF->nRef = 1;
nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname);
taosRemoveFile(fname);
taosMemoryFree(fSet.pHeadF);
}
} else {
ASSERT(fSet.pHeadF->size == pSetNew->pHeadF->size);
ASSERT(fSet.pHeadF->offset == pSetNew->pHeadF->offset);
} }
}
pFS->inTxn = 1; // data
return code; fSet.pDataF = pSetOld->pDataF;
if ((!sameDisk) || (pSetOld->pDataF->commitID != pSetNew->pDataF->commitID)) {
pSetOld->pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile));
if (pSetOld->pDataF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->pDataF = *pSetNew->pDataF;
pSetOld->pDataF->nRef = 1;
nRef = atomic_sub_fetch_32(&fSet.pDataF->nRef, 1);
if (nRef == 0) {
tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pDataF, fname);
taosRemoveFile(fname);
taosMemoryFree(fSet.pDataF);
}
} else {
ASSERT(pSetOld->pDataF->size <= pSetNew->pDataF->size);
pSetOld->pDataF->size = pSetNew->pDataF->size;
}
_err: // last
tsdbError("vgId:%d tsdb fs begin failed since %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code)); fSet.pLastF = pSetOld->pLastF;
return code; if ((!sameDisk) || (pSetOld->pLastF->commitID != pSetNew->pLastF->commitID)) {
} pSetOld->pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
if (pSetOld->pLastF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->pLastF = *pSetNew->pLastF;
pSetOld->pLastF->nRef = 1;
nRef = atomic_sub_fetch_32(&fSet.pLastF->nRef, 1);
if (nRef == 0) {
tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pLastF, fname);
taosRemoveFile(fname);
taosMemoryFree(fSet.pLastF);
}
} else {
ASSERT(pSetOld->pLastF->size == pSetNew->pLastF->size);
}
int32_t tsdbFSCommit(STsdbFS *pFS) { // sma
int32_t code = 0; fSet.pSmaF = pSetOld->pSmaF;
STsdbFSState *pState = pFS->nState; if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) {
char tfname[TSDB_FILENAME_LEN]; pSetOld->pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
char fname[TSDB_FILENAME_LEN]; if (pSetOld->pSmaF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*pSetOld->pSmaF = *pSetNew->pSmaF;
pSetOld->pSmaF->nRef = 1;
nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1);
if (nRef == 0) {
tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname);
taosRemoveFile(fname);
taosMemoryFree(fSet.pSmaF);
}
} else {
ASSERT(pSetOld->pSmaF->size <= pSetNew->pSmaF->size);
pSetOld->pSmaF->size = pSetNew->pSmaF->size;
}
// need lock (todo) if (!sameDisk) {
pFS->nState = pFS->cState; pSetOld->diskId = pSetNew->diskId;
pFS->cState = pState; }
snprintf(tfname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT.t", tfsGetPrimaryPath(pFS->pTsdb->pVnode->pTfs), TD_DIRSEP, iOld++;
pFS->pTsdb->path, TD_DIRSEP); iNew++;
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sCURRENT", tfsGetPrimaryPath(pFS->pTsdb->pVnode->pTfs), TD_DIRSEP, continue;
pFS->pTsdb->path, TD_DIRSEP);
// gnrt CURRENT.t _remove_old:
code = tsdbGnrtCurrent(tfname, pFS->cState); nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1);
if (code) goto _err; if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname);
taosRemoveFile(fname);
taosMemoryFree(pSetOld->pHeadF);
}
// rename nRef = atomic_sub_fetch_32(&pSetOld->pDataF->nRef, 1);
code = taosRenameFile(tfname, fname); if (nRef == 0) {
if (code) { tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pDataF, fname);
code = TAOS_SYSTEM_ERROR(code); taosRemoveFile(fname);
goto _err; taosMemoryFree(pSetOld->pDataF);
} }
// apply commit on disk nRef = atomic_sub_fetch_32(&pSetOld->pLastF->nRef, 1);
code = tsdbFSApplyDiskChange(pFS, pFS->nState, pFS->cState); if (nRef == 0) {
if (code) goto _err; tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pLastF, fname);
taosRemoveFile(fname);
taosMemoryFree(pSetOld->pLastF);
}
pFS->inTxn = 0; nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1);
if (nRef == 0) {
tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname);
taosRemoveFile(fname);
taosMemoryFree(pSetOld->pSmaF);
}
return code; taosArrayRemove(pTsdb->fs.aDFileSet, iOld);
continue;
_err: _add_new:
tsdbError("vgId:%d tsdb fs commit failed since %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code)); fSet.diskId = pSetNew->diskId;
return code; fSet.fid = pSetNew->fid;
}
int32_t tsdbFSRollback(STsdbFS *pFS) { // head
int32_t code = 0; fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
if (fSet.pHeadF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*fSet.pHeadF = *pSetNew->pHeadF;
fSet.pHeadF->nRef = 1;
code = tsdbFSApplyDiskChange(pFS, pFS->nState, pFS->cState); // data
if (code) goto _err; fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile));
if (fSet.pDataF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*fSet.pDataF = *pSetNew->pDataF;
fSet.pDataF->nRef = 1;
// last
fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile));
if (fSet.pLastF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*fSet.pLastF = *pSetNew->pLastF;
fSet.pLastF->nRef = 1;
pFS->inTxn = 0; // sma
fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile));
if (fSet.pSmaF == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
*fSet.pSmaF = *pSetNew->pSmaF;
fSet.pSmaF->nRef = 1;
if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
iOld++;
iNew++;
continue;
}
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb fs rollback failed since %s", TD_VID(pFS->pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb fs commit phase 2 failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
int32_t tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile) { int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) {
int32_t code = 0; int32_t code = 0;
pState->delFile = *pDelFile; int32_t nRef;
pState->pDelFile = &pState->delFile;
return code;
}
int32_t tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet) { pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet));
int32_t code = 0; if (pFS->aDFileSet == NULL) {
int32_t idx = taosArraySearchIdx(pState->aDFileSet, pSet, tDFileSetCmprFn, TD_GE); code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
if (idx < 0) { pFS->pDelFile = pTsdb->fs.pDelFile;
if (taosArrayPush(pState->aDFileSet, pSet) == NULL) { if (pFS->pDelFile) {
nRef = atomic_fetch_add_32(&pFS->pDelFile->nRef, 1);
ASSERT(nRef > 0);
}
SDFileSet fSet;
for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
fSet = *pSet;
nRef = atomic_fetch_add_32(&pSet->pHeadF->nRef, 1);
ASSERT(nRef > 0);
nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1);
ASSERT(nRef > 0);
nRef = atomic_fetch_add_32(&pSet->pLastF->nRef, 1);
ASSERT(nRef > 0);
nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1);
ASSERT(nRef > 0);
if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
} else {
SDFileSet *tDFileSet = (SDFileSet *)taosArrayGet(pState->aDFileSet, idx);
int32_t c = tDFileSetCmprFn(pSet, tDFileSet);
if (c == 0) {
taosArraySet(pState->aDFileSet, idx, pSet);
} else {
if (taosArrayInsert(pState->aDFileSet, idx, pSet) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
} }
_exit: _exit:
return code; return code;
} }
void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid) { void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
int32_t idx; int32_t nRef;
char fname[TSDB_FILENAME_LEN];
idx = taosArraySearchIdx(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); if (pFS->pDelFile) {
ASSERT(idx >= 0); nRef = atomic_sub_fetch_32(&pFS->pDelFile->nRef, 1);
taosArrayRemove(pState->aDFileSet, idx); ASSERT(nRef >= 0);
} if (nRef == 0) {
tsdbDelFileName(pTsdb, pFS->pDelFile, fname);
taosRemoveFile(fname);
taosMemoryFree(pFS->pDelFile);
}
}
SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState) { return pState->pDelFile; } for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) {
SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet);
SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag) { // head
return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, flag); nRef = atomic_sub_fetch_32(&pSet->pHeadF->nRef, 1);
} ASSERT(nRef >= 0);
if (nRef == 0) {
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
taosRemoveFile(fname);
taosMemoryFree(pSet->pHeadF);
}
// data
nRef = atomic_sub_fetch_32(&pSet->pDataF->nRef, 1);
ASSERT(nRef >= 0);
if (nRef == 0) {
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
taosRemoveFile(fname);
taosMemoryFree(pSet->pDataF);
}
// last
nRef = atomic_sub_fetch_32(&pSet->pLastF->nRef, 1);
ASSERT(nRef >= 0);
if (nRef == 0) {
tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname);
taosRemoveFile(fname);
taosMemoryFree(pSet->pLastF);
}
// sma
nRef = atomic_sub_fetch_32(&pSet->pSmaF->nRef, 1);
ASSERT(nRef >= 0);
if (nRef == 0) {
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
taosRemoveFile(fname);
taosMemoryFree(pSet->pSmaF);
}
}
taosArrayDestroy(pFS->aDFileSet);
}
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbFile.c
浏览文件 @ 3b090271
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
#include "tsdb.h" #include "tsdb.h"
static int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pHeadFile->commitID); n += tPutI64v(p ? p + n : p, pHeadFile->commitID);
...@@ -35,7 +35,7 @@ static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) { ...@@ -35,7 +35,7 @@ static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) {
return n; return n;
} }
static int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile) { int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pDataFile->commitID); n += tPutI64v(p ? p + n : p, pDataFile->commitID);
...@@ -53,7 +53,7 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) { ...@@ -53,7 +53,7 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) {
return n; return n;
} }
static int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) { int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pLastFile->commitID); n += tPutI64v(p ? p + n : p, pLastFile->commitID);
...@@ -71,7 +71,7 @@ static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) { ...@@ -71,7 +71,7 @@ static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) {
return n; return n;
} }
static int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile) { int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pSmaFile->commitID); n += tPutI64v(p ? p + n : p, pSmaFile->commitID);
...@@ -90,123 +90,79 @@ static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) { ...@@ -90,123 +90,79 @@ static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) {
} }
// EXPOSED APIS ================================================== // EXPOSED APIS ==================================================
void tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]) { void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]) {
STfs *pTfs = pTsdb->pVnode->pTfs; snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pHeadF->commitID, ".head");
}
switch (ftype) { void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]) {
case TSDB_HEAD_FILE: snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did),
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTfs, pDFileSet->diskId), TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data");
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pDFileSet->fid, pDFileSet->fHead.commitID,
".head");
break;
case TSDB_DATA_FILE:
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTfs, pDFileSet->diskId),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pDFileSet->fid, pDFileSet->fData.commitID,
".data");
break;
case TSDB_LAST_FILE:
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTfs, pDFileSet->diskId),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pDFileSet->fid, pDFileSet->fLast.commitID,
".last");
break;
case TSDB_SMA_FILE:
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTfs, pDFileSet->diskId),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pDFileSet->fid, pDFileSet->fSma.commitID,
".sma");
break;
default:
ASSERT(0);
break;
}
} }
bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype) { void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]) {
if (pDFileSet1->diskId.level != pDFileSet2->diskId.level || pDFileSet1->diskId.id != pDFileSet2->diskId.id) { snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did),
return false; TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pLastF->commitID, ".last");
} }
switch (ftype) { void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) {
case TSDB_HEAD_FILE: snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did),
return pDFileSet1->fHead.commitID == pDFileSet2->fHead.commitID; TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSmaF->commitID, ".sma");
case TSDB_DATA_FILE:
return pDFileSet1->fData.commitID == pDFileSet2->fData.commitID;
case TSDB_LAST_FILE:
return pDFileSet1->fLast.commitID == pDFileSet2->fLast.commitID;
case TSDB_SMA_FILE:
return pDFileSet1->fSma.commitID == pDFileSet2->fSma.commitID;
default:
ASSERT(0);
break;
}
} }
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; } bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; }
int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) {
int32_t code = 0;
int64_t n;
char hdr[TSDB_FHDR_SIZE];
memset(hdr, 0, TSDB_FHDR_SIZE);
tPutDataFileHdr(hdr, pSet, ftype);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pFD, 0, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
n = taosWriteFile(pFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _exit;
}
_exit:
return code;
}
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
int32_t code = 0; int32_t code = 0;
int64_t size; int64_t size;
int64_t n;
TdFilePtr pFD; TdFilePtr pFD;
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
char hdr[TSDB_FHDR_SIZE] = {0};
tsdbDataFileName(pTsdb, pSet, ftype, fname);
// open
pFD = taosOpenFile(fname, TD_FILE_WRITE);
if (pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// truncate // truncate
switch (ftype) { switch (ftype) {
case TSDB_HEAD_FILE:
size = pSet->fHead.size;
break;
case TSDB_DATA_FILE: case TSDB_DATA_FILE:
size = pSet->fData.size; size = pSet->pDataF->size;
break; tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
case TSDB_LAST_FILE: tPutDataFile(hdr, pSet->pDataF);
size = pSet->fLast.size;
break; break;
case TSDB_SMA_FILE: case TSDB_SMA_FILE:
size = pSet->fSma.size; size = pSet->pSmaF->size;
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
tPutSmaFile(hdr, pSet->pSmaF);
break; break;
default: default:
ASSERT(0); ASSERT(0);
} }
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
// open
pFD = taosOpenFile(fname, TD_FILE_WRITE);
if (pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// ftruncate
if (taosFtruncateFile(pFD, size) < 0) { if (taosFtruncateFile(pFD, size) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
// update header // update header
code = tsdbUpdateDFileHdr(pFD, pSet, ftype); n = taosLSeekFile(pFD, 0, SEEK_SET);
if (code) goto _err; if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
n = taosWriteFile(pFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// sync // sync
if (taosFsyncFile(pFD) < 0) { if (taosFsyncFile(pFD) < 0) {
...@@ -220,42 +176,20 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { ...@@ -220,42 +176,20 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb rollback file failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype) {
int32_t n = 0;
switch (ftype) {
case TSDB_HEAD_FILE:
n += tPutHeadFile(p ? p + n : p, &pSet->fHead);
break;
case TSDB_DATA_FILE:
n += tPutDataFile(p ? p + n : p, &pSet->fData);
break;
case TSDB_LAST_FILE:
n += tPutLastFile(p ? p + n : p, &pSet->fLast);
break;
case TSDB_SMA_FILE:
n += tPutSmaFile(p ? p + n : p, &pSet->fSma);
break;
default:
ASSERT(0);
}
return n;
}
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
int32_t n = 0; int32_t n = 0;
n += tPutI32v(p ? p + n : p, pSet->diskId.level); n += tPutI32v(p ? p + n : p, pSet->diskId.level);
n += tPutI32v(p ? p + n : p, pSet->diskId.id); n += tPutI32v(p ? p + n : p, pSet->diskId.id);
n += tPutI32v(p ? p + n : p, pSet->fid); n += tPutI32v(p ? p + n : p, pSet->fid);
n += tPutHeadFile(p ? p + n : p, &pSet->fHead); n += tPutHeadFile(p ? p + n : p, pSet->pHeadF);
n += tPutDataFile(p ? p + n : p, &pSet->fData); n += tPutDataFile(p ? p + n : p, pSet->pDataF);
n += tPutLastFile(p ? p + n : p, &pSet->fLast); n += tPutLastFile(p ? p + n : p, pSet->pLastF);
n += tPutSmaFile(p ? p + n : p, &pSet->fSma); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF);
return n; return n;
} }
...@@ -266,20 +200,18 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { ...@@ -266,20 +200,18 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
n += tGetI32v(p + n, &pSet->diskId.level); n += tGetI32v(p + n, &pSet->diskId.level);
n += tGetI32v(p + n, &pSet->diskId.id); n += tGetI32v(p + n, &pSet->diskId.id);
n += tGetI32v(p + n, &pSet->fid); n += tGetI32v(p + n, &pSet->fid);
n += tGetHeadFile(p + n, &pSet->fHead); n += tGetHeadFile(p + n, pSet->pHeadF);
n += tGetDataFile(p + n, &pSet->fData); n += tGetDataFile(p + n, pSet->pDataF);
n += tGetLastFile(p + n, &pSet->fLast); n += tGetLastFile(p + n, pSet->pLastF);
n += tGetSmaFile(p + n, &pSet->fSma); n += tGetSmaFile(p + n, pSet->pSmaF);
return n; return n;
} }
// SDelFile =============================================== // SDelFile ===============================================
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]) { void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]) {
STfs *pTfs = pTsdb->pVnode->pTfs; snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%dver%" PRId64 "%s", tfsGetPrimaryPath(pTsdb->pVnode->pTfs),
TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), pFile->commitID, ".del");
snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%dver%" PRId64 "%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, pTsdb->path,
TD_DIRSEP, TD_VID(pTsdb->pVnode), pFile->commitID, ".del");
} }
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) { int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) {
......
source/dnode/vnode/src/tsdb/tsdbMemTable.c
浏览文件 @ 3b090271
...@@ -93,7 +93,11 @@ static int32_t tbDataPCmprFn(const void *p1, const void *p2) { ...@@ -93,7 +93,11 @@ static int32_t tbDataPCmprFn(const void *p1, const void *p2) {
} }
void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) { void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
STbData *pTbData = &(STbData){.suid = suid, .uid = uid}; STbData *pTbData = &(STbData){.suid = suid, .uid = uid};
void *p = taosArraySearch(pMemTable->aTbData, &pTbData, tbDataPCmprFn, TD_EQ);
taosRLockLatch(&pMemTable->latch);
void *p = taosArraySearch(pMemTable->aTbData, &pTbData, tbDataPCmprFn, TD_EQ);
taosRUnLockLatch(&pMemTable->latch);
*ppTbData = p ? *(STbData **)p : NULL; *ppTbData = p ? *(STbData **)p : NULL;
} }
...@@ -363,10 +367,13 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid ...@@ -363,10 +367,13 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
void *p; void *p;
if (idx < 0) { if (idx < 0) {
p = taosArrayPush(pMemTable->aTbData, &pTbData); idx = taosArrayGetSize(pMemTable->aTbData);
} else {
p = taosArrayInsert(pMemTable->aTbData, idx, &pTbData);
} }
taosWLockLatch(&pMemTable->latch);
p = taosArrayInsert(pMemTable->aTbData, idx, &pTbData);
taosWUnLockLatch(&pMemTable->latch);
if (p == NULL) { if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
...@@ -605,46 +612,3 @@ void tsdbUnrefMemTable(SMemTable *pMemTable) { ...@@ -605,46 +612,3 @@ void tsdbUnrefMemTable(SMemTable *pMemTable) {
tsdbMemTableDestroy(pMemTable); tsdbMemTableDestroy(pMemTable);
} }
} }
int32_t tsdbTakeMemSnapshot(STsdb *pTsdb, SMemTable **ppMem, SMemTable **ppIMem) {
int32_t code = 0;
// lock
code = taosThreadRwlockRdlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
goto _exit;
}
// take snapshot
*ppMem = pTsdb->mem;
*ppIMem = pTsdb->imem;
if (*ppMem) {
tsdbRefMemTable(*ppMem);
}
if (*ppIMem) {
tsdbRefMemTable(*ppIMem);
}
// unlock
code = taosThreadRwlockUnlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
goto _exit;
}
_exit:
return code;
}
void tsdbUntakeMemSnapshot(STsdb *pTsdb, SMemTable *pMem, SMemTable *pIMem) {
if (pMem) {
tsdbUnrefMemTable(pMem);
}
if (pIMem) {
tsdbUnrefMemTable(pIMem);
}
}
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbOpen.c
浏览文件 @ 3b090271
...@@ -66,7 +66,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee ...@@ -66,7 +66,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
tfsMkdir(pVnode->pTfs, pTsdb->path); tfsMkdir(pVnode->pTfs, pTsdb->path);
// open tsdb // open tsdb
if (tsdbFSOpen(pTsdb, &pTsdb->pFS) < 0) { if (tsdbFSOpen(pTsdb) < 0) {
goto _err; goto _err;
} }
...@@ -88,7 +88,7 @@ _err: ...@@ -88,7 +88,7 @@ _err:
int tsdbClose(STsdb **pTsdb) { int tsdbClose(STsdb **pTsdb) {
if (*pTsdb) { if (*pTsdb) {
taosThreadRwlockDestroy(&(*pTsdb)->rwLock); taosThreadRwlockDestroy(&(*pTsdb)->rwLock);
tsdbFSClose((*pTsdb)->pFS); tsdbFSClose(*pTsdb);
tsdbCloseCache((*pTsdb)->lruCache); tsdbCloseCache((*pTsdb)->lruCache);
taosMemoryFreeClear(*pTsdb); taosMemoryFreeClear(*pTsdb);
} }
......
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ 3b090271
...@@ -118,8 +118,7 @@ struct STsdbReader { ...@@ -118,8 +118,7 @@ struct STsdbReader {
char* idStr; // query info handle, for debug purpose char* idStr; // query info handle, for debug purpose
int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows
SBlockLoadSuppInfo suppInfo; SBlockLoadSuppInfo suppInfo;
SMemTable* pMem; STsdbReadSnap* pReadSnap;
SMemTable* pIMem;
SIOCostSummary cost; SIOCostSummary cost;
STSchema* pSchema; STSchema* pSchema;
...@@ -275,20 +274,18 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap ...@@ -275,20 +274,18 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap
} }
// init file iterator // init file iterator
static int32_t initFilesetIterator(SFilesetIter* pIter, const STsdbFSState* pFState, int32_t order, const char* idstr) { static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, int32_t order, const char* idstr) {
size_t numOfFileset = taosArrayGetSize(pFState->aDFileSet); size_t numOfFileset = taosArrayGetSize(aDFileSet);
pIter->index = ASCENDING_TRAVERSE(order) ? -1 : numOfFileset; pIter->index = ASCENDING_TRAVERSE(order) ? -1 : numOfFileset;
pIter->order = order; pIter->order = order;
pIter->pFileList = taosArrayDup(pFState->aDFileSet); pIter->pFileList = aDFileSet;
pIter->numOfFiles = numOfFileset; pIter->numOfFiles = numOfFileset;
tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, idstr); tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, idstr);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void cleanupFilesetIterator(SFilesetIter* pIter) { taosArrayDestroy(pIter->pFileList); }
static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
bool asc = ASCENDING_TRAVERSE(pIter->order); bool asc = ASCENDING_TRAVERSE(pIter->order);
int32_t step = asc ? 1 : -1; int32_t step = asc ? 1 : -1;
...@@ -1881,8 +1878,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea ...@@ -1881,8 +1878,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); int32_t backward = (!ASCENDING_TRAVERSE(pReader->order));
STbData* d = NULL; STbData* d = NULL;
if (pReader->pMem != NULL) { if (pReader->pReadSnap->pMem != NULL) {
tsdbGetTbDataFromMemTable(pReader->pMem, pReader->suid, pBlockScanInfo->uid, &d); tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid, &d);
if (d != NULL) { if (d != NULL) {
code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
...@@ -1902,8 +1899,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea ...@@ -1902,8 +1899,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea
} }
STbData* di = NULL; STbData* di = NULL;
if (pReader->pIMem != NULL) { if (pReader->pReadSnap->pIMem != NULL) {
tsdbGetTbDataFromMemTable(pReader->pIMem, pReader->suid, pBlockScanInfo->uid, &di); tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid, &di);
if (di != NULL) { if (di != NULL) {
code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
...@@ -1939,7 +1936,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* ...@@ -1939,7 +1936,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader*
SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); SArray* pDelData = taosArrayInit(4, sizeof(SDelData));
SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState); SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile;
if (pDelFile) { if (pDelFile) {
SDelFReader* pDelFReader = NULL; SDelFReader* pDelFReader = NULL;
code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL); code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL);
...@@ -2836,8 +2833,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl ...@@ -2836,8 +2833,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
SDataBlockIter* pBlockIter = &pReader->status.blockIter; SDataBlockIter* pBlockIter = &pReader->status.blockIter;
STsdbFSState* pFState = pReader->pTsdb->pFS->cState; code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap);
initFilesetIterator(&pReader->status.fileIter, pFState, pReader->order, pReader->idStr); if (code) goto _err;
initFilesetIterator(&pReader->status.fileIter, (*ppReader)->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr);
resetDataBlockIterator(&pReader->status.blockIter, pReader->order); resetDataBlockIterator(&pReader->status.blockIter, pReader->order);
// no data in files, let's try buffer in memory // no data in files, let's try buffer in memory
...@@ -2850,8 +2849,6 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl ...@@ -2850,8 +2849,6 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
} }
} }
tsdbTakeMemSnapshot(pReader->pTsdb, &pReader->pMem, &pReader->pIMem);
tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
return code; return code;
...@@ -2867,7 +2864,7 @@ void tsdbReaderClose(STsdbReader* pReader) { ...@@ -2867,7 +2864,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
tsdbUntakeMemSnapshot(pReader->pTsdb, pReader->pMem, pReader->pIMem); tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap);
taosMemoryFreeClear(pSupInfo->plist); taosMemoryFreeClear(pSupInfo->plist);
taosMemoryFree(pSupInfo->colIds); taosMemoryFree(pSupInfo->colIds);
...@@ -2880,7 +2877,6 @@ void tsdbReaderClose(STsdbReader* pReader) { ...@@ -2880,7 +2877,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
} }
taosMemoryFree(pSupInfo->buildBuf); taosMemoryFree(pSupInfo->buildBuf);
cleanupFilesetIterator(&pReader->status.fileIter);
cleanupDataBlockIterator(&pReader->status.blockIter); cleanupDataBlockIterator(&pReader->status.blockIter);
destroyBlockScanInfo(pReader->status.pTableMap); destroyBlockScanInfo(pReader->status.pTableMap);
blockDataDestroy(pReader->pResBlock); blockDataDestroy(pReader->pResBlock);
...@@ -3087,8 +3083,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { ...@@ -3087,8 +3083,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
tsdbDataFReaderClose(&pReader->pFileReader); tsdbDataFReaderClose(&pReader->pFileReader);
STsdbFSState* pFState = pReader->pTsdb->pFS->cState; initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr);
initFilesetIterator(&pReader->status.fileIter, pFState, pReader->order, pReader->idStr);
resetDataBlockIterator(&pReader->status.blockIter, pReader->order); resetDataBlockIterator(&pReader->status.blockIter, pReader->order);
resetDataBlockScanInfo(pReader->status.pTableMap); resetDataBlockScanInfo(pReader->status.pTableMap);
...@@ -3250,3 +3245,68 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 ...@@ -3250,3 +3245,68 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap) {
int32_t code = 0;
// alloc
*ppSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap));
if (*ppSnap == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
// lock
code = taosThreadRwlockRdlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
goto _exit;
}
// take snapshot
(*ppSnap)->pMem = pTsdb->mem;
(*ppSnap)->pIMem = pTsdb->imem;
if ((*ppSnap)->pMem) {
tsdbRefMemTable((*ppSnap)->pMem);
}
if ((*ppSnap)->pIMem) {
tsdbRefMemTable((*ppSnap)->pIMem);
}
// fs
code = tsdbFSRef(pTsdb, &(*ppSnap)->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _exit;
}
// unlock
code = taosThreadRwlockUnlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
goto _exit;
}
tsdbTrace("vgId:%d take read snapshot", TD_VID(pTsdb->pVnode));
_exit:
return code;
}
void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap) {
if (pSnap) {
if (pSnap->pMem) {
tsdbUnrefMemTable(pSnap->pMem);
}
if (pSnap->pIMem) {
tsdbUnrefMemTable(pSnap->pIMem);
}
tsdbFSUnref(pTsdb, &pSnap->fs);
taosMemoryFree(pSnap);
}
tsdbTrace("vgId:%d untake read snapshot", TD_VID(pTsdb->pVnode));
}
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
浏览文件 @ 3b090271
...@@ -459,7 +459,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS ...@@ -459,7 +459,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
// open impl // open impl
// head // head
tsdbDataFileName(pTsdb, pSet, TSDB_HEAD_FILE, fname); tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
pReader->pHeadFD = taosOpenFile(fname, TD_FILE_READ); pReader->pHeadFD = taosOpenFile(fname, TD_FILE_READ);
if (pReader->pHeadFD == NULL) { if (pReader->pHeadFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
...@@ -467,7 +467,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS ...@@ -467,7 +467,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
} }
// data // data
tsdbDataFileName(pTsdb, pSet, TSDB_DATA_FILE, fname); tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
pReader->pDataFD = taosOpenFile(fname, TD_FILE_READ); pReader->pDataFD = taosOpenFile(fname, TD_FILE_READ);
if (pReader->pDataFD == NULL) { if (pReader->pDataFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
...@@ -475,7 +475,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS ...@@ -475,7 +475,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
} }
// last // last
tsdbDataFileName(pTsdb, pSet, TSDB_LAST_FILE, fname); tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname);
pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ); pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ);
if (pReader->pLastFD == NULL) { if (pReader->pLastFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
...@@ -483,7 +483,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS ...@@ -483,7 +483,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
} }
// sma // sma
tsdbDataFileName(pTsdb, pSet, TSDB_SMA_FILE, fname); tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ); pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ);
if (pReader->pSmaFD == NULL) { if (pReader->pSmaFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
...@@ -536,8 +536,8 @@ _err: ...@@ -536,8 +536,8 @@ _err:
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf) { int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf) {
int32_t code = 0; int32_t code = 0;
int64_t offset = pReader->pSet->fHead.offset; int64_t offset = pReader->pSet->pHeadF->offset;
int64_t size = pReader->pSet->fHead.size - offset; int64_t size = pReader->pSet->pHeadF->size - offset;
uint8_t *pBuf = NULL; uint8_t *pBuf = NULL;
int64_t n; int64_t n;
uint32_t delimiter; uint32_t delimiter;
...@@ -1211,17 +1211,6 @@ _err: ...@@ -1211,17 +1211,6 @@ _err:
} }
// SDataFWriter ==================================================== // SDataFWriter ====================================================
struct SDataFWriter {
STsdb *pTsdb;
SDFileSet wSet;
TdFilePtr pHeadFD;
TdFilePtr pDataFD;
TdFilePtr pLastFD;
TdFilePtr pSmaFD;
};
SDFileSet *tsdbDataFWriterGetWSet(SDataFWriter *pWriter) { return &pWriter->wSet; }
int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) {
int32_t code = 0; int32_t code = 0;
int32_t flag; int32_t flag;
...@@ -1237,12 +1226,20 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ...@@ -1237,12 +1226,20 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
goto _err; goto _err;
} }
pWriter->pTsdb = pTsdb; pWriter->pTsdb = pTsdb;
pWriter->wSet = *pSet; pWriter->wSet = (SDFileSet){.diskId = pSet->diskId,
pSet = &pWriter->wSet; .fid = pSet->fid,
.pHeadF = &pWriter->fHead,
.pDataF = &pWriter->fData,
.pLastF = &pWriter->fLast,
.pSmaF = &pWriter->fSma};
pWriter->fHead = *pSet->pHeadF;
pWriter->fData = *pSet->pDataF;
pWriter->fLast = *pSet->pLastF;
pWriter->fSma = *pSet->pSmaF;
// head // head
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
tsdbDataFileName(pTsdb, pSet, TSDB_HEAD_FILE, fname); tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname);
pWriter->pHeadFD = taosOpenFile(fname, flag); pWriter->pHeadFD = taosOpenFile(fname, flag);
if (pWriter->pHeadFD == NULL) { if (pWriter->pHeadFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
...@@ -1257,28 +1254,28 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ...@@ -1257,28 +1254,28 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
ASSERT(n == TSDB_FHDR_SIZE); ASSERT(n == TSDB_FHDR_SIZE);
pSet->fHead.size += TSDB_FHDR_SIZE; pWriter->fHead.size += TSDB_FHDR_SIZE;
// data // data
if (pSet->fData.size == 0) { if (pWriter->fData.size == 0) {
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
} else { } else {
flag = TD_FILE_WRITE; flag = TD_FILE_WRITE;
} }
tsdbDataFileName(pTsdb, pSet, TSDB_DATA_FILE, fname); tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname);
pWriter->pDataFD = taosOpenFile(fname, flag); pWriter->pDataFD = taosOpenFile(fname, flag);
if (pWriter->pDataFD == NULL) { if (pWriter->pDataFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (pSet->fData.size == 0) { if (pWriter->fData.size == 0) {
n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
pSet->fData.size += TSDB_FHDR_SIZE; pWriter->fData.size += TSDB_FHDR_SIZE;
} else { } else {
n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END); n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END);
if (n < 0) { if (n < 0) {
...@@ -1286,29 +1283,29 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ...@@ -1286,29 +1283,29 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
goto _err; goto _err;
} }
ASSERT(n == pSet->fData.size); ASSERT(n == pWriter->fData.size);
} }
// last // last
if (pSet->fLast.size == 0) { if (pWriter->fLast.size == 0) {
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
} else { } else {
flag = TD_FILE_WRITE; flag = TD_FILE_WRITE;
} }
tsdbDataFileName(pTsdb, pSet, TSDB_LAST_FILE, fname); tsdbLastFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fLast, fname);
pWriter->pLastFD = taosOpenFile(fname, flag); pWriter->pLastFD = taosOpenFile(fname, flag);
if (pWriter->pLastFD == NULL) { if (pWriter->pLastFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (pSet->fLast.size == 0) { if (pWriter->fLast.size == 0) {
n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
pSet->fLast.size += TSDB_FHDR_SIZE; pWriter->fLast.size += TSDB_FHDR_SIZE;
} else { } else {
n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_END); n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_END);
if (n < 0) { if (n < 0) {
...@@ -1316,29 +1313,29 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ...@@ -1316,29 +1313,29 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
goto _err; goto _err;
} }
ASSERT(n == pSet->fLast.size); ASSERT(n == pWriter->fLast.size);
} }
// sma // sma
if (pSet->fSma.size == 0) { if (pWriter->fSma.size == 0) {
flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
} else { } else {
flag = TD_FILE_WRITE; flag = TD_FILE_WRITE;
} }
tsdbDataFileName(pTsdb, pSet, TSDB_SMA_FILE, fname); tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname);
pWriter->pSmaFD = taosOpenFile(fname, flag); pWriter->pSmaFD = taosOpenFile(fname, flag);
if (pWriter->pSmaFD == NULL) { if (pWriter->pSmaFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
if (pSet->fSma.size == 0) { if (pWriter->fSma.size == 0) {
n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
} }
pSet->fSma.size += TSDB_FHDR_SIZE; pWriter->fSma.size += TSDB_FHDR_SIZE;
} else { } else {
n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END); n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END);
if (n < 0) { if (n < 0) {
...@@ -1346,7 +1343,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS ...@@ -1346,7 +1343,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
goto _err; goto _err;
} }
ASSERT(n == pSet->fSma.size); ASSERT(n == pWriter->fSma.size);
} }
*ppWriter = pWriter; *ppWriter = pWriter;
...@@ -1418,22 +1415,76 @@ _err: ...@@ -1418,22 +1415,76 @@ _err:
int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) {
int32_t code = 0; int32_t code = 0;
int64_t n;
char hdr[TSDB_FHDR_SIZE];
// head ============== // head ==============
code = tsdbUpdateDFileHdr(pWriter->pHeadFD, &pWriter->wSet, TSDB_HEAD_FILE); memset(hdr, 0, TSDB_FHDR_SIZE);
if (code) goto _err; tPutHeadFile(hdr, &pWriter->fHead);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pWriter->pHeadFD, 0, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// data ============== // data ==============
code = tsdbUpdateDFileHdr(pWriter->pHeadFD, &pWriter->wSet, TSDB_DATA_FILE); memset(hdr, 0, TSDB_FHDR_SIZE);
if (code) goto _err; tPutDataFile(hdr, &pWriter->fData);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// last ============== // last ==============
code = tsdbUpdateDFileHdr(pWriter->pHeadFD, &pWriter->wSet, TSDB_LAST_FILE); memset(hdr, 0, TSDB_FHDR_SIZE);
if (code) goto _err; tPutLastFile(hdr, &pWriter->fLast);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// sma ============== // sma ==============
code = tsdbUpdateDFileHdr(pWriter->pHeadFD, &pWriter->wSet, TSDB_SMA_FILE); memset(hdr, 0, TSDB_FHDR_SIZE);
if (code) goto _err; tPutSmaFile(hdr, &pWriter->fSma);
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
return code; return code;
...@@ -1444,7 +1495,7 @@ _err: ...@@ -1444,7 +1495,7 @@ _err:
int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf) { int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf) {
int32_t code = 0; int32_t code = 0;
SHeadFile *pHeadFile = &pWriter->wSet.fHead; SHeadFile *pHeadFile = &pWriter->fHead;
uint8_t *pBuf = NULL; uint8_t *pBuf = NULL;
int64_t size; int64_t size;
int64_t n; int64_t n;
...@@ -1494,7 +1545,7 @@ _err: ...@@ -1494,7 +1545,7 @@ _err:
int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, SBlockIdx *pBlockIdx) { int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, SBlockIdx *pBlockIdx) {
int32_t code = 0; int32_t code = 0;
SHeadFile *pHeadFile = &pWriter->wSet.fHead; SHeadFile *pHeadFile = &pWriter->fHead;
SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
uint8_t *pBuf = NULL; uint8_t *pBuf = NULL;
int64_t size; int64_t size;
...@@ -1831,9 +1882,9 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_ ...@@ -1831,9 +1882,9 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_
pSubBlock->nRow = pBlockData->nRow; pSubBlock->nRow = pBlockData->nRow;
pSubBlock->cmprAlg = cmprAlg; pSubBlock->cmprAlg = cmprAlg;
if (pBlock->last) { if (pBlock->last) {
pSubBlock->offset = pWriter->wSet.fLast.size; pSubBlock->offset = pWriter->fLast.size;
} else { } else {
pSubBlock->offset = pWriter->wSet.fData.size; pSubBlock->offset = pWriter->fData.size;
} }
// ======================= BLOCK DATA ======================= // ======================= BLOCK DATA =======================
...@@ -1881,9 +1932,9 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_ ...@@ -1881,9 +1932,9 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_
pSubBlock->szBlock = pSubBlock->szBlockCol + sizeof(TSCKSUM) + nData; pSubBlock->szBlock = pSubBlock->szBlockCol + sizeof(TSCKSUM) + nData;
if (pBlock->last) { if (pBlock->last) {
pWriter->wSet.fLast.size += pSubBlock->szBlock; pWriter->fLast.size += pSubBlock->szBlock;
} else { } else {
pWriter->wSet.fData.size += pSubBlock->szBlock; pWriter->fData.size += pSubBlock->szBlock;
} }
// ======================= BLOCK SMA ======================= // ======================= BLOCK SMA =======================
...@@ -1896,8 +1947,8 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_ ...@@ -1896,8 +1947,8 @@ int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_
if (code) goto _err; if (code) goto _err;
if (pSubBlock->nSma > 0) { if (pSubBlock->nSma > 0) {
pSubBlock->sOffset = pWriter->wSet.fSma.size; pSubBlock->sOffset = pWriter->fSma.size;
pWriter->wSet.fSma.size += (sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM)); pWriter->fSma.size += (sizeof(SColumnDataAgg) * pSubBlock->nSma + sizeof(TSCKSUM));
} }
_exit: _exit:
...@@ -1924,8 +1975,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1924,8 +1975,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
char fNameTo[TSDB_FILENAME_LEN]; char fNameTo[TSDB_FILENAME_LEN];
// head // head
tsdbDataFileName(pTsdb, pSetFrom, TSDB_HEAD_FILE, fNameFrom); tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom);
tsdbDataFileName(pTsdb, pSetTo, TSDB_HEAD_FILE, fNameTo); tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
...@@ -1939,7 +1990,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1939,7 +1990,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->fHead.size); n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
...@@ -1948,8 +1999,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1948,8 +1999,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
taosCloseFile(&PInFD); taosCloseFile(&PInFD);
// data // data
tsdbDataFileName(pTsdb, pSetFrom, TSDB_DATA_FILE, fNameFrom); tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom);
tsdbDataFileName(pTsdb, pSetTo, TSDB_DATA_FILE, fNameTo); tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
...@@ -1963,7 +2014,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1963,7 +2014,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->fData.size); n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
...@@ -1972,8 +2023,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1972,8 +2023,9 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
taosCloseFile(&PInFD); taosCloseFile(&PInFD);
// last // last
tsdbDataFileName(pTsdb, pSetFrom, TSDB_LAST_FILE, fNameFrom); tsdbLastFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pLastF, fNameFrom);
tsdbDataFileName(pTsdb, pSetTo, TSDB_LAST_FILE, fNameTo); tsdbLastFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pLastF, fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
...@@ -1986,7 +2038,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1986,7 +2038,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->fLast.size); n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pLastF->size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
...@@ -1995,8 +2047,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -1995,8 +2047,8 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
taosCloseFile(&PInFD); taosCloseFile(&PInFD);
// sma // sma
tsdbDataFileName(pTsdb, pSetFrom, TSDB_SMA_FILE, fNameFrom); tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom);
tsdbDataFileName(pTsdb, pSetTo, TSDB_SMA_FILE, fNameTo); tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo);
pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (pOutFD == NULL) { if (pOutFD == NULL) {
...@@ -2010,7 +2062,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { ...@@ -2010,7 +2062,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
goto _err; goto _err;
} }
n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->fSma.size); n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
......
source/dnode/vnode/src/tsdb/tsdbRetention.c
浏览文件 @ 3b090271
...@@ -15,90 +15,99 @@ ...@@ -15,90 +15,99 @@
#include "tsdb.h" #include "tsdb.h"
static int32_t tsdbDoRetentionImpl(STsdb *pTsdb, int64_t now, int8_t try, int8_t *canDo) { static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) {
int32_t code = 0; for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) {
STsdbFSState *pState; SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
if (try) {
pState = pTsdb->pFS->cState;
*canDo = 0;
} else {
pState = pTsdb->pFS->nState;
}
for (int32_t iSet = 0; iSet < taosArrayGetSize(pState->aDFileSet); iSet++) {
SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pState->aDFileSet, iSet);
int32_t expLevel = tsdbFidLevel(pDFileSet->fid, &pTsdb->keepCfg, now);
SDiskID did; SDiskID did;
// check if (expLevel == pSet->diskId.level) continue;
if (expLevel == pDFileSet->diskId.id) continue;
// delete or move
if (expLevel < 0) { if (expLevel < 0) {
if (try) { return true;
*canDo = 1;
} else {
tsdbFSStateDeleteDFileSet(pState, pDFileSet->fid);
iSet--;
}
} else { } else {
// alloc
if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) {
code = terrno; return false;
goto _exit;
} }
if (did.level == pDFileSet->diskId.level) continue; if (did.level == pSet->diskId.level) continue;
if (try) { return true;
*canDo = 1;
} else {
// copy the file to new disk
SDFileSet nDFileSet = *pDFileSet;
nDFileSet.diskId = did;
tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did);
code = tsdbDFileSetCopy(pTsdb, pDFileSet, &nDFileSet);
if (code) goto _exit;
code = tsdbFSStateUpsertDFileSet(pState, &nDFileSet);
if (code) goto _exit;
}
} }
} }
_exit: return false;
return code;
} }
int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
int32_t code = 0; int32_t code = 0;
int8_t canDo;
// try
tsdbDoRetentionImpl(pTsdb, now, 1, &canDo);
if (!canDo) goto _exit;
// begin if (!tsdbShouldDoRetention(pTsdb, now)) {
code = tsdbFSBegin(pTsdb->pFS); return code;
if (code) goto _err; }
// do retention // do retention
code = tsdbDoRetentionImpl(pTsdb, now, 0, NULL); STsdbFS fs;
code = tsdbFSCopy(pTsdb, &fs);
if (code) goto _err; if (code) goto _err;
// commit for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) {
code = tsdbFSCommit(pTsdb->pFS); SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
SDiskID did;
if (expLevel < 0) {
taosMemoryFree(pSet->pHeadF);
taosMemoryFree(pSet->pDataF);
taosMemoryFree(pSet->pLastF);
taosMemoryFree(pSet->pSmaF);
taosArrayRemove(fs.aDFileSet, iSet);
iSet--;
} else {
if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) {
code = terrno;
goto _exit;
}
if (did.level == pSet->diskId.level) continue;
// copy file to new disk (todo)
SDFileSet fSet = *pSet;
fSet.diskId = did;
code = tsdbDFileSetCopy(pTsdb, pSet, &fSet);
if (code) goto _err;
code = tsdbFSUpsertFSet(&fs, &fSet);
if (code) goto _err;
}
/* code */
}
// do change fs
code = tsdbFSCommit1(pTsdb, &fs);
if (code) goto _err; if (code) goto _err;
taosThreadRwlockWrlock(&pTsdb->rwLock);
code = tsdbFSCommit2(pTsdb, &fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _err;
}
taosThreadRwlockUnlock(&pTsdb->rwLock);
tsdbFSDestroy(&fs);
_exit: _exit:
return code; return code;
_err: _err:
tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
tsdbFSRollback(pTsdb->pFS); ASSERT(0);
// tsdbFSRollback(pTsdb->pFS);
return code; return code;
} }
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
浏览文件 @ 3b090271
...@@ -20,6 +20,7 @@ struct STsdbSnapReader { ...@@ -20,6 +20,7 @@ struct STsdbSnapReader {
STsdb* pTsdb; STsdb* pTsdb;
int64_t sver; int64_t sver;
int64_t ever; int64_t ever;
STsdbFS fs;
// for data file // for data file
int8_t dataDone; int8_t dataDone;
int32_t fid; int32_t fid;
...@@ -45,7 +46,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { ...@@ -45,7 +46,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
while (true) { while (true) {
if (pReader->pDataFReader == NULL) { if (pReader->pDataFReader == NULL) {
SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->pFS->cState, pReader->fid, TD_GT); SDFileSet* pSet =
taosArraySearch(pReader->fs.aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSetCmprFn, TD_GT);
if (pSet == NULL) goto _exit; if (pSet == NULL) goto _exit;
...@@ -159,7 +161,7 @@ _err: ...@@ -159,7 +161,7 @@ _err:
static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
int32_t code = 0; int32_t code = 0;
STsdb* pTsdb = pReader->pTsdb; STsdb* pTsdb = pReader->pTsdb;
SDelFile* pDelFile = pTsdb->pFS->cState->pDelFile; SDelFile* pDelFile = pReader->fs.pDelFile;
if (pReader->pDelFReader == NULL) { if (pReader->pDelFReader == NULL) {
if (pDelFile == NULL) { if (pDelFile == NULL) {
...@@ -254,6 +256,24 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe ...@@ -254,6 +256,24 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe
pReader->sver = sver; pReader->sver = sver;
pReader->ever = ever; pReader->ever = ever;
code = taosThreadRwlockRdlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
goto _err;
}
code = tsdbFSRef(pTsdb, &pReader->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _err;
}
code = taosThreadRwlockUnlock(&pTsdb->rwLock);
if (code) {
code = TAOS_SYSTEM_ERROR(code);
goto _err;
}
pReader->fid = INT32_MIN; pReader->fid = INT32_MIN;
pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pReader->aBlockIdx == NULL) { if (pReader->aBlockIdx == NULL) {
...@@ -305,6 +325,8 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { ...@@ -305,6 +325,8 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
taosArrayDestroy(pReader->aDelIdx); taosArrayDestroy(pReader->aDelIdx);
taosArrayDestroy(pReader->aDelData); taosArrayDestroy(pReader->aDelData);
tsdbFSUnref(pReader->pTsdb, &pReader->fs);
tsdbInfo("vgId:%d vnode snapshot tsdb reader closed", TD_VID(pReader->pTsdb->pVnode)); tsdbInfo("vgId:%d vnode snapshot tsdb reader closed", TD_VID(pReader->pTsdb->pVnode));
taosMemoryFree(pReader); taosMemoryFree(pReader);
...@@ -358,6 +380,7 @@ struct STsdbSnapWriter { ...@@ -358,6 +380,7 @@ struct STsdbSnapWriter {
STsdb* pTsdb; STsdb* pTsdb;
int64_t sver; int64_t sver;
int64_t ever; int64_t ever;
STsdbFS fs;
// config // config
int32_t minutes; int32_t minutes;
...@@ -798,7 +821,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { ...@@ -798,7 +821,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL); code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL);
if (code) goto _err; if (code) goto _err;
code = tsdbFSStateUpsertDFileSet(pTsdb->pFS->nState, tsdbDataFWriterGetWSet(pWriter->pDataFWriter)); code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet);
if (code) goto _err; if (code) goto _err;
code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1); code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1);
...@@ -843,7 +866,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 ...@@ -843,7 +866,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
pWriter->fid = fid; pWriter->fid = fid;
// read // read
SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->pFS->nState, fid, TD_EQ); SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ);
if (pSet) { if (pSet) {
code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
if (code) goto _err; if (code) goto _err;
...@@ -863,22 +886,26 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 ...@@ -863,22 +886,26 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
tBlockDataReset(&pWriter->bDataR); tBlockDataReset(&pWriter->bDataR);
// write // write
SDFileSet wSet; SHeadFile fHead;
SDataFile fData;
SLastFile fLast;
SSmaFile fSma;
SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma};
if (pSet) { if (pSet) {
wSet = (SDFileSet){.diskId = pSet->diskId, wSet.diskId = pSet->diskId;
.fid = fid, wSet.fid = fid;
.fHead = {.commitID = pWriter->commitID, .offset = 0, .size = 0}, fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
.fData = pSet->fData, fData = *pSet->pDataF;
.fLast = {.commitID = pWriter->commitID, .size = 0}, fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0};
.fSma = pSet->fSma}; fSma = *pSet->pSmaF;
} else { } else {
wSet = (SDFileSet){.diskId = (SDiskID){.level = 0, .id = 0}, wSet.diskId = (SDiskID){.level = 0, .id = 0};
.fid = fid, wSet.fid = fid;
.fHead = {.commitID = pWriter->commitID, .offset = 0, .size = 0}, fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
.fData = {.commitID = pWriter->commitID, .size = 0}, fData = (SDataFile){.commitID = pWriter->commitID, .size = 0};
.fLast = {.commitID = pWriter->commitID, .size = 0}, fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0};
.fSma = {.commitID = pWriter->commitID, .size = 0}}; fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0};
} }
code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet); code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
...@@ -907,7 +934,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 ...@@ -907,7 +934,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
STsdb* pTsdb = pWriter->pTsdb; STsdb* pTsdb = pWriter->pTsdb;
if (pWriter->pDelFWriter == NULL) { if (pWriter->pDelFWriter == NULL) {
SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->nState); SDelFile* pDelFile = pWriter->fs.pDelFile;
// reader // reader
if (pDelFile) { if (pDelFile) {
...@@ -1017,7 +1044,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { ...@@ -1017,7 +1044,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter); code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter);
if (code) goto _err; if (code) goto _err;
code = tsdbFSStateUpsertDelFile(pTsdb->pFS->nState, &pWriter->pDelFWriter->fDel); code = tsdbFSUpsertDelFile(&pWriter->fs, &pWriter->pDelFWriter->fDel);
if (code) goto _err; if (code) goto _err;
code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1); code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1);
...@@ -1051,6 +1078,9 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr ...@@ -1051,6 +1078,9 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
pWriter->sver = sver; pWriter->sver = sver;
pWriter->ever = ever; pWriter->ever = ever;
code = tsdbFSCopy(pTsdb, &pWriter->fs);
if (code) goto _err;
// config // config
pWriter->minutes = pTsdb->keepCfg.days; pWriter->minutes = pTsdb->keepCfg.days;
pWriter->precision = pTsdb->keepCfg.precision; pWriter->precision = pTsdb->keepCfg.precision;
...@@ -1096,9 +1126,6 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr ...@@ -1096,9 +1126,6 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
goto _err; goto _err;
} }
code = tsdbFSBegin(pTsdb->pFS);
if (code) goto _err;
*ppWriter = pWriter; *ppWriter = pWriter;
return code; return code;
...@@ -1113,8 +1140,9 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { ...@@ -1113,8 +1140,9 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
STsdbSnapWriter* pWriter = *ppWriter; STsdbSnapWriter* pWriter = *ppWriter;
if (rollback) { if (rollback) {
code = tsdbFSRollback(pWriter->pTsdb->pFS); ASSERT(0);
if (code) goto _err; // code = tsdbFSRollback(pWriter->pTsdb->pFS);
// if (code) goto _err;
} else { } else {
code = tsdbSnapWriteDataEnd(pWriter); code = tsdbSnapWriteDataEnd(pWriter);
if (code) goto _err; if (code) goto _err;
...@@ -1122,7 +1150,10 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { ...@@ -1122,7 +1150,10 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
code = tsdbSnapWriteDelEnd(pWriter); code = tsdbSnapWriteDelEnd(pWriter);
if (code) goto _err; if (code) goto _err;
code = tsdbFSCommit(pWriter->pTsdb->pFS); code = tsdbFSCommit1(pWriter->pTsdb, &pWriter->fs);
if (code) goto _err;
code = tsdbFSCommit2(pWriter->pTsdb, &pWriter->fs);
if (code) goto _err; if (code) goto _err;
} }
......
source/dnode/vnode/src/vnd/vnodeSvr.c
浏览文件 @ 3b090271
...@@ -773,6 +773,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq ...@@ -773,6 +773,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
pRsp->code = 0; pRsp->code = 0;
pSubmitReq->version = version;
#ifdef TD_DEBUG_PRINT_ROW #ifdef TD_DEBUG_PRINT_ROW
vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__);
...@@ -791,7 +792,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq ...@@ -791,7 +792,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
submitRsp.pArray = taosArrayInit(msgIter.numOfBlocks, sizeof(SSubmitBlkRsp)); submitRsp.pArray = taosArrayInit(msgIter.numOfBlocks, sizeof(SSubmitBlkRsp));
newTbUids = taosArrayInit(msgIter.numOfBlocks, sizeof(int64_t)); newTbUids = taosArrayInit(msgIter.numOfBlocks, sizeof(int64_t));
if (!submitRsp.pArray) { if (!submitRsp.pArray || !newTbUids) {
pRsp->code = TSDB_CODE_OUT_OF_MEMORY; pRsp->code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
......
source/libs/executor/src/executorMain.c
浏览文件 @ 3b090271
...@@ -315,6 +315,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { ...@@ -315,6 +315,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamScanInfo* pInfo = pOperator->info; SStreamScanInfo* pInfo = pOperator->info;
if (pOffset->type == TMQ_OFFSET__LOG) { if (pOffset->type == TMQ_OFFSET__LOG) {
STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTSInfo->dataReader);
pTSInfo->dataReader = NULL;
#if 0 #if 0
if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) && if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) &&
pInfo->tqReader->pWalReader->curVersion != pOffset->version) { pInfo->tqReader->pWalReader->curVersion != pOffset->version) {
...@@ -349,8 +352,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { ...@@ -349,8 +352,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
#ifndef NDEBUG #ifndef NDEBUG
qDebug("switch to next table %ld (cursor %d), %ld rows returned", uid, qDebug("switch to next table %ld (cursor %d), %ld rows returned", uid, pTableScanInfo->currentTable,
pTableScanInfo->currentTable, pInfo->pTableScanOp->resultInfo.totalRows); pInfo->pTableScanOp->resultInfo.totalRows);
pInfo->pTableScanOp->resultInfo.totalRows = 0; pInfo->pTableScanOp->resultInfo.totalRows = 0;
#endif #endif
...@@ -367,6 +370,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { ...@@ -367,6 +370,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
// TODO after dropping table, table may be not found // TODO after dropping table, table may be not found
ASSERT(found); ASSERT(found);
if (pTableScanInfo->dataReader == NULL) {
if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond,
pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 ||
pTableScanInfo->dataReader == NULL) {
ASSERT(0);
}
}
tsdbSetTableId(pTableScanInfo->dataReader, uid); tsdbSetTableId(pTableScanInfo->dataReader, uid);
int64_t oldSkey = pTableScanInfo->cond.twindows.skey; int64_t oldSkey = pTableScanInfo->cond.twindows.skey;
pTableScanInfo->cond.twindows.skey = ts + 1; pTableScanInfo->cond.twindows.skey = ts + 1;
......
source/libs/executor/src/executorimpl.c
浏览文件 @ 3b090271
...@@ -2868,7 +2868,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan ...@@ -2868,7 +2868,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
*order = TSDB_ORDER_ASC; *order = TSDB_ORDER_ASC;
*scanFlag = MAIN_SCAN; *scanFlag = MAIN_SCAN;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) {
STableScanInfo* pTableScanInfo = pOperator->info; STableScanInfo* pTableScanInfo = pOperator->info;
*order = pTableScanInfo->cond.order; *order = pTableScanInfo->cond.order;
*scanFlag = pTableScanInfo->scanFlag; *scanFlag = pTableScanInfo->scanFlag;
......
source/libs/executor/src/scanoperator.c
浏览文件 @ 3b090271
...@@ -740,7 +740,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { ...@@ -740,7 +740,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
static void destroyBlockDistScanOperatorInfo(void* param, int32_t numOfOutput) { static void destroyBlockDistScanOperatorInfo(void* param, int32_t numOfOutput) {
SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param;
blockDataDestroy(pDistInfo->pResBlock); blockDataDestroy(pDistInfo->pResBlock);
tsdbReaderClose(pDistInfo->pHandle);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
...@@ -982,6 +982,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 ...@@ -982,6 +982,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
if (!pResult) { if (!pResult) {
blockDataCleanup(pSDB); blockDataCleanup(pSDB);
*pRowIndex = 0; *pRowIndex = 0;
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTableScanInfo->dataReader);
pTableScanInfo->dataReader = NULL;
return NULL; return NULL;
} }
...@@ -1003,6 +1006,9 @@ static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_ ...@@ -1003,6 +1006,9 @@ static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_
} }
if (!pResult) { if (!pResult) {
pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
tsdbReaderClose(pTableScanInfo->dataReader);
pTableScanInfo->dataReader = NULL;
return NULL; return NULL;
} }
...@@ -2047,8 +2053,8 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { ...@@ -2047,8 +2053,8 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
int32_t code = metaGetTableEntryByUid(&mr, suid); int32_t code = metaGetTableEntryByUid(&mr, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name,
pInfo->pCur->mr.me.name, suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
metaReaderClear(&mr); metaReaderClear(&mr);
metaCloseTbCursor(pInfo->pCur); metaCloseTbCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
...@@ -2154,7 +2160,6 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { ...@@ -2154,7 +2160,6 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
} }
} }
static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SSysTableScanInfo* pInfo = pOperator->info; SSysTableScanInfo* pInfo = pOperator->info;
...@@ -2180,12 +2185,13 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { ...@@ -2180,12 +2185,13 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
getDBNameFromCondition(pInfo->pCondition, dbName); getDBNameFromCondition(pInfo->pCondition, dbName);
sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
} }
if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
return sysTableScanUserTables(pOperator); return sysTableScanUserTables(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) {
return sysTableScanUserTags(pOperator); return sysTableScanUserTags(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && IS_SYS_DBNAME(pInfo->req.db)) { } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, TSDB_TABLE_FNAME_LEN) == 0 &&
IS_SYS_DBNAME(pInfo->req.db)) {
return sysTableScanUserSTables(pOperator); return sysTableScanUserSTables(pOperator);
} else { // load the meta from mnode of the given epset } else { // load the meta from mnode of the given epset
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
......
source/libs/function/inc/functionMgtInt.h
浏览文件 @ 3b090271
...@@ -47,6 +47,7 @@ extern "C" { ...@@ -47,6 +47,7 @@ extern "C" {
#define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18) #define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18)
#define FUNC_MGT_CLIENT_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19) #define FUNC_MGT_CLIENT_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19)
#define FUNC_MGT_MULTI_ROWS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20) #define FUNC_MGT_MULTI_ROWS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20)
#define FUNC_MGT_KEEP_ORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(21)
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
......
source/libs/function/src/builtins.c
浏览文件 @ 3b090271
...@@ -2097,7 +2097,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -2097,7 +2097,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{ {
.name = "top", .name = "top",
.type = FUNCTION_TYPE_TOP, .type = FUNCTION_TYPE_TOP,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
.translateFunc = translateTopBot, .translateFunc = translateTopBot,
.getEnvFunc = getTopBotFuncEnv, .getEnvFunc = getTopBotFuncEnv,
.initFunc = topBotFunctionSetup, .initFunc = topBotFunctionSetup,
...@@ -2112,7 +2112,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -2112,7 +2112,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{ {
.name = "bottom", .name = "bottom",
.type = FUNCTION_TYPE_BOTTOM, .type = FUNCTION_TYPE_BOTTOM,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
.translateFunc = translateTopBot, .translateFunc = translateTopBot,
.getEnvFunc = getTopBotFuncEnv, .getEnvFunc = getTopBotFuncEnv,
.initFunc = topBotFunctionSetup, .initFunc = topBotFunctionSetup,
...@@ -2480,7 +2480,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -2480,7 +2480,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{ {
.name = "sample", .name = "sample",
.type = FUNCTION_TYPE_SAMPLE, .type = FUNCTION_TYPE_SAMPLE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
.translateFunc = translateSample, .translateFunc = translateSample,
.getEnvFunc = getSampleFuncEnv, .getEnvFunc = getSampleFuncEnv,
.initFunc = sampleFunctionSetup, .initFunc = sampleFunctionSetup,
...@@ -2906,7 +2906,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { ...@@ -2906,7 +2906,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{ {
.name = "_select_value", .name = "_select_value",
.type = FUNCTION_TYPE_SELECT_VALUE, .type = FUNCTION_TYPE_SELECT_VALUE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateSelectValue, .translateFunc = translateSelectValue,
.getEnvFunc = getSelectivityFuncEnv, // todo remove this function later. .getEnvFunc = getSelectivityFuncEnv, // todo remove this function later.
.initFunc = functionSetup, .initFunc = functionSetup,
......
source/libs/function/src/functionMgt.c
浏览文件 @ 3b090271
...@@ -183,6 +183,8 @@ bool fmIsClientPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc( ...@@ -183,6 +183,8 @@ bool fmIsClientPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(
bool fmIsMultiRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_ROWS_FUNC); } bool fmIsMultiRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_ROWS_FUNC); }
bool fmIsKeepOrderFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_KEEP_ORDER_FUNC); }
bool fmIsInterpFunc(int32_t funcId) { bool fmIsInterpFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false; return false;
......
source/libs/nodes/src/nodesCloneFuncs.c
浏览文件 @ 3b090271
...@@ -332,6 +332,9 @@ static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { ...@@ -332,6 +332,9 @@ static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(precision);
CLONE_NODE_FIELD(pLimit); CLONE_NODE_FIELD(pLimit);
CLONE_NODE_FIELD(pSlimit); CLONE_NODE_FIELD(pSlimit);
COPY_SCALAR_FIELD(requireDataOrder);
COPY_SCALAR_FIELD(resultDataOrder);
COPY_SCALAR_FIELD(groupAction);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
......
source/libs/nodes/src/nodesCodeFuncs.c
浏览文件 @ 3b090271
...@@ -504,6 +504,9 @@ static const char* jkLogicPlanConditions = "Conditions"; ...@@ -504,6 +504,9 @@ static const char* jkLogicPlanConditions = "Conditions";
static const char* jkLogicPlanChildren = "Children"; static const char* jkLogicPlanChildren = "Children";
static const char* jkLogicPlanLimit = "Limit"; static const char* jkLogicPlanLimit = "Limit";
static const char* jkLogicPlanSlimit = "SLimit"; static const char* jkLogicPlanSlimit = "SLimit";
static const char* jkLogicPlanRequireDataOrder = "RequireDataOrder";
static const char* jkLogicPlanResultDataOrder = "ResultDataOrder";
static const char* jkLogicPlanGroupAction = "GroupAction";
static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
const SLogicNode* pNode = (const SLogicNode*)pObj; const SLogicNode* pNode = (const SLogicNode*)pObj;
...@@ -521,6 +524,15 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { ...@@ -521,6 +524,15 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkLogicPlanSlimit, nodeToJson, pNode->pSlimit); code = tjsonAddObject(pJson, jkLogicPlanSlimit, nodeToJson, pNode->pSlimit);
} }
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkLogicPlanRequireDataOrder, pNode->requireDataOrder);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkLogicPlanResultDataOrder, pNode->resultDataOrder);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkLogicPlanGroupAction, pNode->groupAction);
}
return code; return code;
} }
...@@ -541,6 +553,15 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { ...@@ -541,6 +553,15 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkLogicPlanSlimit, &pNode->pSlimit); code = jsonToNodeObject(pJson, jkLogicPlanSlimit, &pNode->pSlimit);
} }
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkLogicPlanRequireDataOrder, pNode->requireDataOrder, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkLogicPlanResultDataOrder, pNode->resultDataOrder, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkLogicPlanGroupAction, pNode->groupAction, code);
}
return code; return code;
} }
......
source/libs/parser/src/parInsert.c
浏览文件 @ 3b090271
...@@ -434,8 +434,12 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, ...@@ -434,8 +434,12 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf,
} }
static bool isNullStr(SToken* pToken) { static bool isNullStr(SToken* pToken) {
return (pToken->type == TK_NULL) || ((pToken->type == TK_NK_STRING) && (pToken->n != 0) && return ((pToken->type == TK_NK_STRING) && (pToken->n != 0) &&
(strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0));
}
static bool isNullValue(int8_t dataType, SToken* pToken) {
return TK_NULL == pToken->type || (!IS_STR_DATA_TYPE(dataType) && isNullStr(pToken));
} }
static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) {
...@@ -461,7 +465,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int ...@@ -461,7 +465,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
return code; return code;
} }
if (isNullStr(pToken)) { if (isNullValue(pSchema->type, pToken)) {
if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z);
} }
...@@ -735,11 +739,12 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* ...@@ -735,11 +739,12 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, SArray* tagName) { static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname,
SArray* tagName) {
pTbReq->type = TD_CHILD_TABLE; pTbReq->type = TD_CHILD_TABLE;
pTbReq->name = strdup(tname); pTbReq->name = strdup(tname);
pTbReq->ctb.suid = suid; pTbReq->ctb.suid = suid;
if(sname) pTbReq->ctb.name = strdup(sname); if (sname) pTbReq->ctb.name = strdup(sname);
pTbReq->ctb.pTag = (uint8_t*)pTag; pTbReq->ctb.pTag = (uint8_t*)pTag;
pTbReq->ctb.tagName = taosArrayDup(tagName); pTbReq->ctb.tagName = taosArrayDup(tagName);
pTbReq->commentLen = -1; pTbReq->commentLen = -1;
...@@ -753,7 +758,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 ...@@ -753,7 +758,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16
uint64_t uv; uint64_t uv;
char* endptr = NULL; char* endptr = NULL;
if (isNullStr(pToken)) { if (isNullValue(pSchema->type, pToken)) {
if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z);
} }
...@@ -761,7 +766,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 ...@@ -761,7 +766,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// strcpy(val->colName, pSchema->name); // strcpy(val->colName, pSchema->name);
val->cid = pSchema->colId; val->cid = pSchema->colId;
val->type = pSchema->type; val->type = pSchema->type;
...@@ -971,7 +976,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint ...@@ -971,7 +976,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
goto end; goto end;
} }
if (!isNullStr(&sToken)) { if (!isNullValue(pTagSchema->type, &sToken)) {
taosArrayPush(tagName, pTagSchema->name); taosArrayPush(tagName, pTagSchema->name);
} }
if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
...@@ -980,7 +985,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint ...@@ -980,7 +985,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
taosMemoryFree(tmpTokenBuf); taosMemoryFree(tmpTokenBuf);
goto end; goto end;
} }
if (isNullStr(&sToken)) { if (isNullValue(pTagSchema->type, &sToken)) {
code = tTagNew(pTagVals, 1, true, &pTag); code = tTagNew(pTagVals, 1, true, &pTag);
} else { } else {
code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg);
...@@ -1321,7 +1326,11 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB ...@@ -1321,7 +1326,11 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB
static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STableDataBlocks* dataBuf) { static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STableDataBlocks* dataBuf) {
char filePathStr[TSDB_FILENAME_LEN] = {0}; char filePathStr[TSDB_FILENAME_LEN] = {0};
strncpy(filePathStr, filePath.z, filePath.n); if (TK_NK_STRING == filePath.type) {
trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr));
} else {
strncpy(filePathStr, filePath.z, filePath.n);
}
TdFilePtr fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM); TdFilePtr fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM);
if (NULL == fp) { if (NULL == fp) {
return TAOS_SYSTEM_ERROR(errno); return TAOS_SYSTEM_ERROR(errno);
...@@ -1556,7 +1565,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache ...@@ -1556,7 +1565,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache
} else { } else {
nodesDestroyNode((*pQuery)->pRoot); nodesDestroyNode((*pQuery)->pRoot);
} }
(*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE;
(*pQuery)->haveResultSet = false; (*pQuery)->haveResultSet = false;
(*pQuery)->msgType = TDMT_VND_SUBMIT; (*pQuery)->msgType = TDMT_VND_SUBMIT;
...@@ -1804,8 +1813,8 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash ...@@ -1804,8 +1813,8 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, TAOS_MULTI_BIND* bind, int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
char* msgBuf, int32_t msgBufLen) { TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock;
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags;
...@@ -1856,7 +1865,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch ...@@ -1856,7 +1865,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
} }
} else { } else {
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
// strcpy(val.colName, pTagSchema->name); // strcpy(val.colName, pTagSchema->name);
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) { if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) {
val.pData = (uint8_t*)bind[c].buffer; val.pData = (uint8_t*)bind[c].buffer;
val.nData = colLen; val.nData = colLen;
...@@ -2247,7 +2256,8 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS ...@@ -2247,7 +2256,8 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS
* @param msg * @param msg
* @return int32_t * @return int32_t
*/ */
static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, SMsgBuf* msg) { static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName,
SMsgBuf* msg) {
SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));
if (!pTagArray) { if (!pTagArray) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
...@@ -2264,7 +2274,7 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p ...@@ -2264,7 +2274,7 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p
taosArrayPush(*tagName, pTagSchema->name); taosArrayPush(*tagName, pTagSchema->name);
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
// strcpy(val.colName, pTagSchema->name); // strcpy(val.colName, pTagSchema->name);
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) { if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) {
val.pData = (uint8_t*)kv->value; val.pData = (uint8_t*)kv->value;
val.nData = kv->length; val.nData = kv->length;
...@@ -2320,7 +2330,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols ...@@ -2320,7 +2330,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
buildInvalidOperationMsg(&pBuf, "bound tags error"); buildInvalidOperationMsg(&pBuf, "bound tags error");
return ret; return ret;
} }
STag* pTag = NULL; STag* pTag = NULL;
SArray* tagName = NULL; SArray* tagName = NULL;
ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &tagName, &pBuf); ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &tagName, &pBuf);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
...@@ -2404,9 +2414,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols ...@@ -2404,9 +2414,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
} else { } else {
int32_t colLen = kv->length; int32_t colLen = kv->length;
if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
// uError("SML:data before:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision); // uError("SML:data before:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision);
kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
// uError("SML:data after:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision); // uError("SML:data after:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision);
} }
if (IS_VAR_DATA_TYPE(kv->type)) { if (IS_VAR_DATA_TYPE(kv->type)) {
......
source/libs/parser/src/parTranslater.c
浏览文件 @ 3b090271
...@@ -1089,7 +1089,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN ...@@ -1089,7 +1089,7 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (0 == LIST_LENGTH(pFunc->pParameterList)) { if (0 == LIST_LENGTH(pFunc->pParameterList)) {
if (!isSelectStmt(pCxt->pCurrStmt) || if (!isSelectStmt(pCxt->pCurrStmt) || NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable ||
QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME);
} }
......
source/libs/parser/src/parser.c
浏览文件 @ 3b090271
...@@ -36,7 +36,7 @@ bool qIsInsertValuesSql(const char* pStr, size_t length) { ...@@ -36,7 +36,7 @@ bool qIsInsertValuesSql(const char* pStr, size_t length) {
pStr += index; pStr += index;
index = 0; index = 0;
t = tStrGetToken((char*)pStr, &index, false); t = tStrGetToken((char*)pStr, &index, false);
if (TK_USING == t.type || TK_VALUES == t.type) { if (TK_USING == t.type || TK_VALUES == t.type || TK_FILE == t.type) {
return true; return true;
} else if (TK_SELECT == t.type) { } else if (TK_SELECT == t.type) {
return false; return false;
......
source/libs/parser/test/parSelectTest.cpp
浏览文件 @ 3b090271
...@@ -444,4 +444,11 @@ TEST_F(ParserSelectTest, withoutFrom) { ...@@ -444,4 +444,11 @@ TEST_F(ParserSelectTest, withoutFrom) {
run("SELECT USER()"); run("SELECT USER()");
} }
TEST_F(ParserSelectTest, withoutFromSemanticCheck) {
useDb("root", "test");
run("SELECT c1", TSDB_CODE_PAR_INVALID_COLUMN);
run("SELECT TBNAME", TSDB_CODE_PAR_INVALID_TBNAME);
}
} // namespace ParserTest } // namespace ParserTest
source/libs/planner/inc/planInt.h
浏览文件 @ 3b090271
...@@ -35,6 +35,7 @@ int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); ...@@ -35,6 +35,7 @@ int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...);
int32_t createColumnByRewriteExprs(SNodeList* pExprs, SNodeList** pList); int32_t createColumnByRewriteExprs(SNodeList* pExprs, SNodeList** pList);
int32_t createColumnByRewriteExpr(SNode* pExpr, SNodeList** pList); int32_t createColumnByRewriteExpr(SNode* pExpr, SNodeList** pList);
int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew); int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew);
int32_t adjustLogicNodeDataRequirement(SLogicNode* pNode, EDataOrderLevel requirement);
int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan); int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan);
int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan);
......
source/libs/planner/src/planLogicCreater.c
浏览文件 @ 3b090271
...@@ -250,6 +250,9 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect ...@@ -250,6 +250,9 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
SScanLogicNode* pScan = NULL; SScanLogicNode* pScan = NULL;
int32_t code = makeScanLogicNode(pCxt, pRealTable, pSelect->hasRepeatScanFuncs, (SLogicNode**)&pScan); int32_t code = makeScanLogicNode(pCxt, pRealTable, pSelect->hasRepeatScanFuncs, (SLogicNode**)&pScan);
pScan->node.groupAction = GROUP_ACTION_NONE;
pScan->node.resultDataOrder = DATA_ORDER_LEVEL_IN_BLOCK;
// set columns to scan // set columns to scan
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL, code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL,
...@@ -336,6 +339,9 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect ...@@ -336,6 +339,9 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
pJoin->joinType = pJoinTable->joinType; pJoin->joinType = pJoinTable->joinType;
pJoin->isSingleTableJoin = pJoinTable->table.singleTable; pJoin->isSingleTableJoin = pJoinTable->table.singleTable;
pJoin->node.groupAction = GROUP_ACTION_CLEAR;
pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL;
pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
...@@ -472,6 +478,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, ...@@ -472,6 +478,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
} }
pAgg->hasLastRow = pSelect->hasLastRowFunc; pAgg->hasLastRow = pSelect->hasLastRowFunc;
pAgg->node.groupAction = GROUP_ACTION_SET;
pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
...@@ -540,6 +549,10 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt ...@@ -540,6 +549,10 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc; pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc;
pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc; pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc;
pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc; pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc;
pIdfRowsFunc->node.groupAction = GROUP_ACTION_KEEP;
pIdfRowsFunc->node.requireDataOrder =
pIdfRowsFunc->isTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE;
pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder;
// indefinite rows functions and _select_values functions // indefinite rows functions and _select_values functions
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs); int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs);
...@@ -571,6 +584,10 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p ...@@ -571,6 +584,10 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pInterpFunc->node.groupAction = GROUP_ACTION_KEEP;
pInterpFunc->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP;
pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder;
int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsInterpFunc, &pInterpFunc->pFuncs); int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsInterpFunc, &pInterpFunc->pFuncs);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT);
...@@ -642,10 +659,12 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo ...@@ -642,10 +659,12 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo
} }
pWindow->winType = WINDOW_TYPE_STATE; pWindow->winType = WINDOW_TYPE_STATE;
pWindow->node.groupAction = GROUP_ACTION_KEEP;
pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP;
pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP;
pWindow->pStateExpr = nodesCloneNode(pState->pExpr); pWindow->pStateExpr = nodesCloneNode(pState->pExpr);
pWindow->pTspk = nodesCloneNode(pState->pCol); pWindow->pTspk = nodesCloneNode(pState->pCol);
if (NULL == pWindow->pTspk) { if (NULL == pWindow->pStateExpr || NULL == pWindow->pTspk) {
nodesDestroyNode((SNode*)pWindow); nodesDestroyNode((SNode*)pWindow);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
...@@ -663,6 +682,9 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW ...@@ -663,6 +682,9 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW
pWindow->winType = WINDOW_TYPE_SESSION; pWindow->winType = WINDOW_TYPE_SESSION;
pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i;
pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE;
pWindow->node.groupAction = GROUP_ACTION_KEEP;
pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP;
pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP;
pWindow->pTspk = nodesCloneNode((SNode*)pSession->pCol); pWindow->pTspk = nodesCloneNode((SNode*)pSession->pCol);
if (NULL == pWindow->pTspk) { if (NULL == pWindow->pTspk) {
...@@ -689,6 +711,9 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva ...@@ -689,6 +711,9 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva
pWindow->slidingUnit = pWindow->slidingUnit =
(NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit);
pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH;
pWindow->node.groupAction = GROUP_ACTION_KEEP;
pWindow->node.requireDataOrder = DATA_ORDER_LEVEL_IN_BLOCK;
pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP;
pWindow->pTspk = nodesCloneNode(pInterval->pCol); pWindow->pTspk = nodesCloneNode(pInterval->pCol);
if (NULL == pWindow->pTspk) { if (NULL == pWindow->pTspk) {
...@@ -734,6 +759,10 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect ...@@ -734,6 +759,10 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pFill->node.groupAction = GROUP_ACTION_KEEP;
pFill->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP;
pFill->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP;
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets);
if (TSDB_CODE_SUCCESS == code && NULL == pFill->node.pTargets) { if (TSDB_CODE_SUCCESS == code && NULL == pFill->node.pTargets) {
code = nodesListMakeStrictAppend(&pFill->node.pTargets, code = nodesListMakeStrictAppend(&pFill->node.pTargets,
...@@ -768,6 +797,9 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect ...@@ -768,6 +797,9 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
} }
pSort->groupSort = pSelect->groupSort; pSort->groupSort = pSelect->groupSort;
pSort->node.groupAction = pSort->groupSort ? GROUP_ACTION_KEEP : GROUP_ACTION_CLEAR;
pSort->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pSort->node.resultDataOrder = pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL;
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets);
if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) {
...@@ -818,6 +850,9 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel ...@@ -818,6 +850,9 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel
TSWAP(pProject->node.pLimit, pSelect->pLimit); TSWAP(pProject->node.pLimit, pSelect->pLimit);
TSWAP(pProject->node.pSlimit, pSelect->pSlimit); TSWAP(pProject->node.pSlimit, pSelect->pSlimit);
pProject->node.groupAction = GROUP_ACTION_CLEAR;
pProject->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pProject->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
...@@ -850,6 +885,10 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS ...@@ -850,6 +885,10 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pPartition->node.groupAction = GROUP_ACTION_SET;
pPartition->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pPartition->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
int32_t code = int32_t code =
nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, COLLECT_COL_TYPE_ALL, &pPartition->node.pTargets); nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, COLLECT_COL_TYPE_ALL, &pPartition->node.pTargets);
if (TSDB_CODE_SUCCESS == code && NULL == pPartition->node.pTargets) { if (TSDB_CODE_SUCCESS == code && NULL == pPartition->node.pTargets) {
...@@ -882,6 +921,10 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe ...@@ -882,6 +921,10 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pAgg->node.groupAction = GROUP_ACTION_SET;
pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
// set grouyp keys, agg funcs and having conditions // set grouyp keys, agg funcs and having conditions
SNodeList* pGroupKeys = NULL; SNodeList* pGroupKeys = NULL;
...@@ -1369,6 +1412,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { ...@@ -1369,6 +1412,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
setLogicNodeParent(pSubplan->pNode); setLogicNodeParent(pSubplan->pNode);
setLogicSubplanType(cxt.hasScan, pSubplan); setLogicSubplanType(cxt.hasScan, pSubplan);
code = adjustLogicNodeDataRequirement(pSubplan->pNode, DATA_ORDER_LEVEL_NONE);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
......
source/libs/planner/src/planOptimizer.c
浏览文件 @ 3b090271
...@@ -1579,6 +1579,34 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { ...@@ -1579,6 +1579,34 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) {
return eliminateProjOptCheckProjColumnNames(pProjectNode); return eliminateProjOptCheckProjColumnNames(pProjectNode);
} }
typedef struct CheckNewChildTargetsCxt {
SNodeList* pNewChildTargets;
bool canUse;
} CheckNewChildTargetsCxt;
static EDealRes eliminateProjOptCanUseNewChildTargetsImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
CheckNewChildTargetsCxt* pCxt = pContext;
SNode* pTarget = NULL;
FOREACH(pTarget, pCxt->pNewChildTargets) {
if (!nodesEqualNode(pTarget, pNode)) {
pCxt->canUse = false;
return DEAL_RES_END;
}
}
}
return DEAL_RES_CONTINUE;
}
static bool eliminateProjOptCanUseNewChildTargets(SLogicNode* pChild, SNodeList* pNewChildTargets) {
if (NULL == pChild->pConditions) {
return true;
}
CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = true};
nodesWalkExpr(pChild->pConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt);
return cxt.canUse;
}
static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan,
SProjectLogicNode* pProjectNode) { SProjectLogicNode* pProjectNode) {
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0); SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0);
...@@ -1594,8 +1622,13 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* ...@@ -1594,8 +1622,13 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
} }
} }
} }
nodesDestroyList(pChild->pTargets); if (eliminateProjOptCanUseNewChildTargets(pChild, pNewChildTargets)) {
pChild->pTargets = pNewChildTargets; nodesDestroyList(pChild->pTargets);
pChild->pTargets = pNewChildTargets;
} else {
nodesDestroyList(pNewChildTargets);
return TSDB_CODE_SUCCESS;
}
int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pProjectNode, pChild); int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pProjectNode, pChild);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
...@@ -1873,6 +1906,8 @@ static int32_t rewriteUniqueOptCreateAgg(SIndefRowsFuncLogicNode* pIndef, SLogic ...@@ -1873,6 +1906,8 @@ static int32_t rewriteUniqueOptCreateAgg(SIndefRowsFuncLogicNode* pIndef, SLogic
TSWAP(pAgg->node.pChildren, pIndef->node.pChildren); TSWAP(pAgg->node.pChildren, pIndef->node.pChildren);
optResetParent((SLogicNode*)pAgg); optResetParent((SLogicNode*)pAgg);
pAgg->node.precision = pIndef->node.precision; pAgg->node.precision = pIndef->node.precision;
pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_IN_BLOCK; // first function requirement
pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
bool hasSelectPrimaryKey = false; bool hasSelectPrimaryKey = false;
...@@ -1945,6 +1980,8 @@ static int32_t rewriteUniqueOptCreateProject(SIndefRowsFuncLogicNode* pIndef, SL ...@@ -1945,6 +1980,8 @@ static int32_t rewriteUniqueOptCreateProject(SIndefRowsFuncLogicNode* pIndef, SL
TSWAP(pProject->node.pTargets, pIndef->node.pTargets); TSWAP(pProject->node.pTargets, pIndef->node.pTargets);
pProject->node.precision = pIndef->node.precision; pProject->node.precision = pIndef->node.precision;
pProject->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
pProject->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL; SNode* pNode = NULL;
...@@ -1973,11 +2010,16 @@ static int32_t rewriteUniqueOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* ...@@ -1973,11 +2010,16 @@ static int32_t rewriteUniqueOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeAppend(&pProject->pChildren, (SNode*)pAgg); code = nodesListMakeAppend(&pProject->pChildren, (SNode*)pAgg);
}
if (TSDB_CODE_SUCCESS == code) {
pAgg->pParent = pProject; pAgg->pParent = pProject;
pAgg = NULL; pAgg = NULL;
code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pIndef, pProject);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pIndef, pProject); code = adjustLogicNodeDataRequirement(
pProject, NULL == pProject->pParent ? DATA_ORDER_LEVEL_NONE : pProject->pParent->requireDataOrder);
pProject = NULL;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
nodesDestroyNode((SNode*)pIndef); nodesDestroyNode((SNode*)pIndef);
...@@ -2145,8 +2187,8 @@ static bool tagScanMayBeOptimized(SLogicNode* pNode) { ...@@ -2145,8 +2187,8 @@ static bool tagScanMayBeOptimized(SLogicNode* pNode) {
} }
SAggLogicNode* pAgg = (SAggLogicNode*)(pNode->pParent); SAggLogicNode* pAgg = (SAggLogicNode*)(pNode->pParent);
if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || planOptNodeListHasCol(pAgg->pGroupKeys) ||
planOptNodeListHasCol(pAgg->pGroupKeys) || !planOptNodeListHasTbname(pAgg->pGroupKeys)) { !planOptNodeListHasTbname(pAgg->pGroupKeys)) {
return false; return false;
} }
...@@ -2171,11 +2213,12 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp ...@@ -2171,11 +2213,12 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp
pScanNode->scanType = SCAN_TYPE_TAG; pScanNode->scanType = SCAN_TYPE_TAG;
SNode* pTarget = NULL; SNode* pTarget = NULL;
FOREACH(pTarget, pScanNode->node.pTargets) { FOREACH(pTarget, pScanNode->node.pTargets) {
if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)(pTarget))->colId) { if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)(pTarget))->colId) {
ERASE_NODE(pScanNode->node.pTargets); ERASE_NODE(pScanNode->node.pTargets);
break; break;
} }
} }
NODES_DESTORY_LIST(pScanNode->pScanCols); NODES_DESTORY_LIST(pScanNode->pScanCols);
SLogicNode* pAgg = pScanNode->node.pParent; SLogicNode* pAgg = pScanNode->node.pParent;
...@@ -2185,8 +2228,8 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp ...@@ -2185,8 +2228,8 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp
SNode* pAggTarget = NULL; SNode* pAggTarget = NULL;
FOREACH(pAggTarget, pAgg->pTargets) { FOREACH(pAggTarget, pAgg->pTargets) {
SNode* pScanTarget = NULL; SNode* pScanTarget = NULL;
FOREACH(pScanTarget, pScanNode->node.pTargets) { FOREACH(pScanTarget, pScanNode->node.pTargets) {
if (0 == strcmp( ((SColumnNode*)pAggTarget)->colName, ((SColumnNode*)pAggTarget)->colName )) { if (0 == strcmp(((SColumnNode*)pAggTarget)->colName, ((SColumnNode*)pAggTarget)->colName)) {
nodesListAppend(pScanTargets, nodesCloneNode(pScanTarget)); nodesListAppend(pScanTargets, nodesCloneNode(pScanTarget));
break; break;
} }
......
source/libs/planner/src/planPhysiCreater.c
浏览文件 @ 3b090271
...@@ -974,6 +974,17 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh ...@@ -974,6 +974,17 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh
return code; return code;
} }
static bool projectCanMergeDataBlock(SProjectLogicNode* pProject) {
if (DATA_ORDER_LEVEL_NONE == pProject->node.resultDataOrder) {
return true;
}
if (1 != LIST_LENGTH(pProject->node.pChildren)) {
return false;
}
SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProject->node.pChildren, 0);
return DATA_ORDER_LEVEL_GLOBAL == pChild->resultDataOrder ? true : false;
}
static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) {
SProjectPhysiNode* pProject = SProjectPhysiNode* pProject =
...@@ -982,6 +993,8 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild ...@@ -982,6 +993,8 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pProject->mergeDataBlock = projectCanMergeDataBlock(pProjectLogicNode);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (0 == LIST_LENGTH(pChildren)) { if (0 == LIST_LENGTH(pChildren)) {
pProject->pProjections = nodesCloneList(pProjectLogicNode->pProjections); pProject->pProjections = nodesCloneList(pProjectLogicNode->pProjections);
......
source/libs/planner/src/planSpliter.c
浏览文件 @ 3b090271
...@@ -657,6 +657,9 @@ static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitIn ...@@ -657,6 +657,9 @@ static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitIn
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_FILL == nodeType(pInfo->pSplitNode->pParent)) {
pInfo->pSplitNode = pInfo->pSplitNode->pParent;
}
SExchangeLogicNode* pExchange = NULL; SExchangeLogicNode* pExchange = NULL;
int32_t code = splCreateExchangeNode(pCxt, pInfo->pSplitNode, &pExchange); int32_t code = splCreateExchangeNode(pCxt, pInfo->pSplitNode, &pExchange);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
......
source/libs/planner/src/planUtil.c
浏览文件 @ 3b090271
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "functionMgt.h"
#include "planInt.h" #include "planInt.h"
static char* getUsageErrFormat(int32_t errCode) { static char* getUsageErrFormat(int32_t errCode) {
...@@ -121,3 +122,185 @@ int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* ...@@ -121,3 +122,185 @@ int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode*
} }
return TSDB_CODE_PLAN_INTERNAL_ERROR; return TSDB_CODE_PLAN_INTERNAL_ERROR;
} }
static int32_t adjustScanDataRequirement(SScanLogicNode* pScan, EDataOrderLevel requirement) {
if (SCAN_TYPE_TABLE != pScan->scanType || SCAN_TYPE_TABLE_MERGE != pScan->scanType) {
return TSDB_CODE_SUCCESS;
}
// The lowest sort level of scan output data is DATA_ORDER_LEVEL_IN_BLOCK
if (requirement < DATA_ORDER_LEVEL_IN_BLOCK) {
requirement = DATA_ORDER_LEVEL_IN_BLOCK;
}
if (DATA_ORDER_LEVEL_IN_BLOCK == requirement) {
pScan->scanType = SCAN_TYPE_TABLE;
} else if (TSDB_SUPER_TABLE == pScan->tableType) {
pScan->scanType = SCAN_TYPE_TABLE_MERGE;
}
pScan->node.resultDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustJoinDataRequirement(SJoinLogicNode* pJoin, EDataOrderLevel requirement) {
// The lowest sort level of join input and output data is DATA_ORDER_LEVEL_GLOBAL
return TSDB_CODE_SUCCESS;
}
static bool isKeepOrderAggFunc(SNodeList* pFuncs) {
SNode* pFunc = NULL;
FOREACH(pFunc, pFuncs) {
if (!fmIsKeepOrderFunc(((SFunctionNode*)pFunc)->funcId)) {
return false;
}
}
return true;
}
static int32_t adjustAggDataRequirement(SAggLogicNode* pAgg, EDataOrderLevel requirement) {
// The sort level of agg with group by output data can only be DATA_ORDER_LEVEL_NONE
if (requirement > DATA_ORDER_LEVEL_NONE && (NULL != pAgg->pGroupKeys || !isKeepOrderAggFunc(pAgg->pAggFuncs))) {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
pAgg->node.resultDataOrder = requirement;
pAgg->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustProjectDataRequirement(SProjectLogicNode* pProject, EDataOrderLevel requirement) {
pProject->node.resultDataOrder = requirement;
pProject->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustIntervalDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) {
// The lowest sort level of interval output data is DATA_ORDER_LEVEL_IN_GROUP
if (requirement < DATA_ORDER_LEVEL_IN_GROUP) {
requirement = DATA_ORDER_LEVEL_IN_GROUP;
}
// The sort level of interval input data is always DATA_ORDER_LEVEL_IN_BLOCK
pWindow->node.resultDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustSessionDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) {
if (requirement <= pWindow->node.resultDataOrder) {
return TSDB_CODE_SUCCESS;
}
pWindow->node.resultDataOrder = requirement;
pWindow->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustStateDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) {
if (requirement <= pWindow->node.resultDataOrder) {
return TSDB_CODE_SUCCESS;
}
pWindow->node.resultDataOrder = requirement;
pWindow->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) {
switch (pWindow->winType) {
case WINDOW_TYPE_INTERVAL:
return adjustIntervalDataRequirement(pWindow, requirement);
case WINDOW_TYPE_SESSION:
return adjustSessionDataRequirement(pWindow, requirement);
case WINDOW_TYPE_STATE:
return adjustStateDataRequirement(pWindow, requirement);
default:
break;
}
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
static int32_t adjustFillDataRequirement(SFillLogicNode* pFill, EDataOrderLevel requirement) {
if (requirement <= pFill->node.requireDataOrder) {
return TSDB_CODE_SUCCESS;
}
pFill->node.resultDataOrder = requirement;
pFill->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustSortDataRequirement(SSortLogicNode* pSort, EDataOrderLevel requirement) {
return TSDB_CODE_SUCCESS;
}
static int32_t adjustPartitionDataRequirement(SPartitionLogicNode* pPart, EDataOrderLevel requirement) {
if (DATA_ORDER_LEVEL_GLOBAL == requirement) {
return TSDB_CODE_PLAN_INTERNAL_ERROR;
}
pPart->node.resultDataOrder = requirement;
pPart->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustIndefRowsDataRequirement(SIndefRowsFuncLogicNode* pIndef, EDataOrderLevel requirement) {
if (requirement <= pIndef->node.resultDataOrder) {
return TSDB_CODE_SUCCESS;
}
pIndef->node.resultDataOrder = requirement;
pIndef->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
static int32_t adjustInterpDataRequirement(SInterpFuncLogicNode* pInterp, EDataOrderLevel requirement) {
if (requirement <= pInterp->node.requireDataOrder) {
return TSDB_CODE_SUCCESS;
}
pInterp->node.resultDataOrder = requirement;
pInterp->node.requireDataOrder = requirement;
return TSDB_CODE_SUCCESS;
}
int32_t adjustLogicNodeDataRequirement(SLogicNode* pNode, EDataOrderLevel requirement) {
int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) {
case QUERY_NODE_LOGIC_PLAN_SCAN:
code = adjustScanDataRequirement((SScanLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_JOIN:
code = adjustJoinDataRequirement((SJoinLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_AGG:
code = adjustAggDataRequirement((SAggLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_PROJECT:
code = adjustProjectDataRequirement((SProjectLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY:
case QUERY_NODE_LOGIC_PLAN_EXCHANGE:
case QUERY_NODE_LOGIC_PLAN_MERGE:
break;
case QUERY_NODE_LOGIC_PLAN_WINDOW:
code = adjustWindowDataRequirement((SWindowLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_FILL:
code = adjustFillDataRequirement((SFillLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_SORT:
code = adjustSortDataRequirement((SSortLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_PARTITION:
code = adjustPartitionDataRequirement((SPartitionLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
code = adjustIndefRowsDataRequirement((SIndefRowsFuncLogicNode*)pNode, requirement);
break;
case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC:
code = adjustInterpDataRequirement((SInterpFuncLogicNode*)pNode, requirement);
break;
default:
break;
}
if (TSDB_CODE_SUCCESS == code) {
SNode* pChild = NULL;
FOREACH(pChild, pNode->pChildren) {
code = adjustLogicNodeDataRequirement((SLogicNode*)pChild, pNode->requireDataOrder);
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
}
return code;
}
source/libs/planner/test/planIntervalTest.cpp
浏览文件 @ 3b090271
...@@ -38,9 +38,15 @@ TEST_F(PlanIntervalTest, fill) { ...@@ -38,9 +38,15 @@ TEST_F(PlanIntervalTest, fill) {
run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' "
"INTERVAL(10s) FILL(LINEAR)"); "INTERVAL(10s) FILL(LINEAR)");
run("SELECT COUNT(*) FROM st1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' "
"INTERVAL(10s) FILL(LINEAR)");
run("SELECT COUNT(*), SUM(c1) FROM t1 " run("SELECT COUNT(*), SUM(c1) FROM t1 "
"WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' "
"INTERVAL(10s) FILL(VALUE, 10, 20)"); "INTERVAL(10s) FILL(VALUE, 10, 20)");
run("SELECT COUNT(*) FROM st1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' "
"PARTITION BY TBNAME interval(10s) fill(prev)");
} }
TEST_F(PlanIntervalTest, selectFunc) { TEST_F(PlanIntervalTest, selectFunc) {
......
source/libs/planner/test/planSubqueryTest.cpp
浏览文件 @ 3b090271
...@@ -48,10 +48,28 @@ TEST_F(PlanSubqeuryTest, doubleGroupBy) { ...@@ -48,10 +48,28 @@ TEST_F(PlanSubqeuryTest, doubleGroupBy) {
"WHERE a > 100 GROUP BY b"); "WHERE a > 100 GROUP BY b");
} }
TEST_F(PlanSubqeuryTest, withSetOperator) { TEST_F(PlanSubqeuryTest, innerSetOperator) {
useDb("root", "test"); useDb("root", "test");
run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)");
run("SELECT c1 FROM (SELECT c1 FROM t1 UNION SELECT c1 FROM t1)"); run("SELECT c1 FROM (SELECT c1 FROM t1 UNION SELECT c1 FROM t1)");
} }
TEST_F(PlanSubqeuryTest, innerFill) {
useDb("root", "test");
run("SELECT cnt FROM (SELECT _WSTART ts, COUNT(*) cnt FROM t1 "
"WHERE ts > '2022-04-01 00:00:00' and ts < '2022-04-30 23:59:59' INTERVAL(10s) FILL(LINEAR)) "
"WHERE ts > '2022-04-06 00:00:00'");
}
TEST_F(PlanSubqeuryTest, outerInterval) {
useDb("root", "test");
run("SELECT COUNT(*) FROM (SELECT * FROM st1) INTERVAL(5s)");
run("SELECT COUNT(*) + SUM(c1) FROM (SELECT * FROM st1) INTERVAL(5s)");
run("SELECT COUNT(*) FROM (SELECT ts, TOP(c1, 10) FROM st1s1) INTERVAL(5s)");
}
tests/script/jenkins/basic.txt
浏览文件 @ 3b090271
...@@ -81,6 +81,7 @@ ...@@ -81,6 +81,7 @@
./test.sh -f tsim/insert/query_multi_file.sim ./test.sh -f tsim/insert/query_multi_file.sim
./test.sh -f tsim/insert/tcp.sim ./test.sh -f tsim/insert/tcp.sim
./test.sh -f tsim/insert/update0.sim ./test.sh -f tsim/insert/update0.sim
./test.sh -f tsim/insert/update1_sort_merge.sim
# ---- parser # ---- parser
./test.sh -f tsim/parser/alter__for_community_version.sim ./test.sh -f tsim/parser/alter__for_community_version.sim
......
tests/script/tsim/insert/dupinsert.sim 已删除 100644 → 0
浏览文件 @ 881c0e44
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== create database
sql drop database if exists d0
sql create database d0 keep 365000d,365000d,365000d
sql use d0
print =============== create super table
sql create table if not exists stb (ts timestamp, c1 int unsigned, c2 double, c3 binary(10), c4 nchar(10), c5 double) tags (city binary(20),district binary(20));
sql show stables
if $rows != 1 then
return -1
endi
print =============== create child table
sql create table ct1 using stb tags("BeiJing", "ChaoYang")
sql create table ct2 using stb tags("BeiJing", "HaiDian")
sql create table ct3 using stb tags("BeiJing", "PingGu")
sql create table ct4 using stb tags("BeiJing", "YanQing")
sql show tables
if $rows != 4 then
print rows $rows != 4
return -1
endi
print =============== step 1 insert records into ct1 - taosd merge
sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10, 20);
sql insert into ct1(ts,c1,c2,c3,c4) values('2022-05-03 16:59:00.011', 11, NULL, 'binary', 'nchar');
sql insert into ct1 values('2022-05-03 16:59:00.016', 16, NULL, NULL, 'nchar', NULL);
sql insert into ct1 values('2022-05-03 16:59:00.016', 17, NULL, NULL, 'nchar', 170);
sql insert into ct1 values('2022-05-03 16:59:00.020', 20, NULL, NULL, 'nchar', 200);
sql insert into ct1 values('2022-05-03 16:59:00.016', 18, NULL, NULL, 'nchar', 180);
sql insert into ct1 values('2022-05-03 16:59:00.021', 21, NULL, NULL, 'nchar', 210);
sql insert into ct1 values('2022-05-03 16:59:00.022', 22, NULL, NULL, 'nchar', 220);
print =============== step 2 insert records into ct1/ct2 - taosc merge for 2022-05-03 16:59:00.010
sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10,10), ('2022-05-03 16:59:00.010',20,10.0), ('2022-05-03 16:59:00.010',30,NULL) ct2(ts,c1) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',20) ct1(ts,c2) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',100) ct1(ts,c3) values('2022-05-03 16:59:00.010','bin1'), ('2022-05-03 16:59:00.010','bin2') ct1(ts,c4,c5) values('2022-05-03 16:59:00.010',NULL,NULL), ('2022-05-03 16:59:00.010','nchar4',1000.01) ct2(ts,c2,c3,c4,c5) values('2022-05-03 16:59:00.010',20,'xkl','zxc',10);
print =============== step 3 insert records into ct3
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.020', 10,10);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.021', 10,10), ('2022-05-03 16:59:00.021',20,20.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.022', 30,30), ('2022-05-03 16:59:00.022',40,40.0),('2022-05-03 16:59:00.022',50,50.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.023', 60,60), ('2022-05-03 16:59:00.023',70,70.0),('2022-05-03 16:59:00.023',80,80.0), ('2022-05-03 16:59:00.023',90,90.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.024', 100,100), ('2022-05-03 16:59:00.025',110,110.0),('2022-05-03 16:59:00.025',120,120.0), ('2022-05-03 16:59:00.025',130,130.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.030', 140,140), ('2022-05-03 16:59:00.030',150,150.0),('2022-05-03 16:59:00.031',160,160.0), ('2022-05-03 16:59:00.030',170,170.0), ('2022-05-03 16:59:00.031',180,180.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.042', 190,190), ('2022-05-03 16:59:00.041',200,200.0),('2022-05-03 16:59:00.040',210,210.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.050', 220,220), ('2022-05-03 16:59:00.051',230,230.0),('2022-05-03 16:59:00.052',240,240.0);
print =============== step 4 insert records into ct4
sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.020', 10,'b0','n0');
sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.021', 20,'b1','n1'), ('2022-05-03 16:59:00.021',30,'b2','n2');
sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.022', 40,'b3','n3'), ('2022-05-03 16:59:00.022',40,'b4','n4'),('2022-05-03 16:59:00.022',50,'b5','n5');
sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.023', 60,'b6','n6'), ('2022-05-03 16:59:00.024',70,'b7','n7'),('2022-05-03 16:59:00.024',80,'b8','n8'), ('2022-05-03 16:59:00.023',90,'b9','n9');
print =============== step 5 query records of ct1 from memory(taosc and taosd merge)
sql select * from ct1;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
print $data50 $data51 $data52 $data53 $data54 $data55
print =============== step 6 query records of ct2 from memory(taosc and taosd merge)
sql select * from ct2;
print $data00 $data01 $data02 $data03 $data04 $data05
if $rows != 1 then
print rows $rows != 1
return -1
endi
print =============== step 7 query records of ct3 from memory
sql select * from ct3;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
print $data50 $data51 $data52 $data53 $data54 $data55
print $data60 $data61 $data62 $data63 $data64 $data65
print $data70 $data71 $data72 $data73 $data74 $data75
print $data80 $data81 $data82 $data83 $data84 $data85
print $data90 $data91 $data92 $data93 $data94 $data95
print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5]
print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5]
print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5]
print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5]
if $rows != 14 then
print rows $rows != 14
return -1
endi
print =============== step 8 query records of ct4 from memory
sql select * from ct4;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 5 then
print rows $rows != 5
return -1
endi
#==================== reboot to trigger commit data to file
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
print =============== step 9 query records of ct1 from file
sql select * from ct1;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
print $data50 $data51 $data52 $data53 $data54 $data55
if $rows != 6 then
print rows $rows != 6
return -1
endi
print =============== step 10 query records of ct2 from file
sql select * from ct2;
print $data00 $data01 $data02 $data03 $data04 $data05
if $rows != 1 then
print rows $rows != 1
return -1
endi
print =============== step 11 query records of ct3 from file
sql select * from ct3;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
print $data50 $data51 $data52 $data53 $data54 $data55
print $data60 $data61 $data62 $data63 $data64 $data65
print $data70 $data71 $data72 $data73 $data74 $data75
print $data80 $data81 $data82 $data83 $data84 $data85
print $data90 $data91 $data92 $data93 $data94 $data95
print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5]
print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5]
print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5]
print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5]
print =============== step 12 query records of ct4 from file
sql select * from ct4;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
\ No newline at end of file
tests/script/tsim/insert/update1_sort_merge.sim
浏览文件 @ 3b090271
...@@ -3,6 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 ...@@ -3,6 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
sql connect sql connect
print =============== test case: merge duplicated rows in taosc and taosd
print =============== create database print =============== create database
sql drop database if exists d0 sql drop database if exists d0
sql create database d0 keep 365000d,365000d,365000d sql create database d0 keep 365000d,365000d,365000d
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或