tsdb.h 19.7 KB
Newer Older
L
Liu Jicong 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
Hongze Cheng 已提交
15 16 17 18

#ifndef _TD_VNODE_TSDB_H_
#define _TD_VNODE_TSDB_H_

H
Hongze Cheng 已提交
19 20
#include "vnodeInt.h"

H
Hongze Cheng 已提交
21 22 23 24
#ifdef __cplusplus
extern "C" {
#endif

H
Hongze Cheng 已提交
25 26
// tsdbDebug ================
// clang-format off
H
Hongze Cheng 已提交
27 28 29 30 31 32
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSD FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSD ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSD WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSD ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSD ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
H
Hongze Cheng 已提交
33
// clang-format on
H
Hongze Cheng 已提交
34

H
Hongze Cheng 已提交
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
typedef struct TSDBROW       TSDBROW;
typedef struct TABLEID       TABLEID;
typedef struct TSDBKEY       TSDBKEY;
typedef struct SDelData      SDelData;
typedef struct SDelIdx       SDelIdx;
typedef struct STbData       STbData;
typedef struct SMemTable     SMemTable;
typedef struct STbDataIter   STbDataIter;
typedef struct STable        STable;
typedef struct SMapData      SMapData;
typedef struct SBlockIdx     SBlockIdx;
typedef struct SBlock        SBlock;
typedef struct SBlockStatis  SBlockStatis;
typedef struct SAggrBlkCol   SAggrBlkCol;
typedef struct SColData      SColData;
typedef struct SBlockDataHdr SBlockDataHdr;
typedef struct SBlockData    SBlockData;
typedef struct SDelFile      SDelFile;
typedef struct SHeadFile     SHeadFile;
typedef struct SDataFile     SDataFile;
typedef struct SLastFile     SLastFile;
typedef struct SSmaFile      SSmaFile;
typedef struct SDFileSet     SDFileSet;
typedef struct SDataFWriter  SDataFWriter;
typedef struct SDataFReader  SDataFReader;
typedef struct SDelFWriter   SDelFWriter;
typedef struct SDelFReader   SDelFReader;
typedef struct SRowIter      SRowIter;
typedef struct STsdbFS       STsdbFS;
typedef struct SRowMerger    SRowMerger;
typedef struct STsdbFSState  STsdbFSState;
typedef struct STsdbSnapHdr  STsdbSnapHdr;
H
Hongze Cheng 已提交
67

H
Hongze Cheng 已提交
68
#define TSDB_MAX_SUBBLOCKS 8
H
Hongze Cheng 已提交
69
#define TSDB_FHDR_SIZE     512
H
refact  
Hongze Cheng 已提交
70

H
Hongze Cheng 已提交
71 72 73
#define HAS_NONE  ((int8_t)0x1)
#define HAS_NULL  ((int8_t)0x2)
#define HAS_VALUE ((int8_t)0x4)
H
Hongze Cheng 已提交
74

H
Hongze Cheng 已提交
75 76 77
#define VERSION_MIN 0
#define VERSION_MAX INT64_MAX

H
Hongze Cheng 已提交
78 79 80
#define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN})
#define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX})

H
Hongze Cheng 已提交
81
// tsdbUtil.c ==============================================================================================
H
Hongze Cheng 已提交
82
// TSDBROW
H
Hongze Cheng 已提交
83 84
#define TSDBROW_TS(ROW)                       (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow])
#define TSDBROW_VERSION(ROW)                  (((ROW)->type == 0) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow])
H
Hongze Cheng 已提交
85
#define TSDBROW_SVERSION(ROW)                 TD_ROW_SVER((ROW)->pTSRow)
H
Hongze Cheng 已提交
86
#define TSDBROW_KEY(ROW)                      ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)})
H
Hongze Cheng 已提交
87 88
#define tsdbRowFromTSRow(VERSION, TSROW)      ((TSDBROW){.type = 0, .version = (VERSION), .pTSRow = (TSROW)})
#define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
H
Hongze Cheng 已提交
89 90 91
void    tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
H
Hongze Cheng 已提交
92
int32_t tsdbRowCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
93
// SRowIter
H
Hongze Cheng 已提交
94
void     tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);
H
Hongze Cheng 已提交
95
SColVal *tRowIterNext(SRowIter *pIter);
H
Hongze Cheng 已提交
96 97 98 99 100
// SRowMerger
int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
void    tRowMergerClear(SRowMerger *pMerger);
int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow);
int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow);
H
Hongze Cheng 已提交
101
// TABLEID
H
Hongze Cheng 已提交
102
int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
103
// TSDBKEY
H
Hongze Cheng 已提交
104
int32_t tsdbKeyCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
105 106
#define MIN_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) < 0) ? (KEY1) : (KEY2))
#define MAX_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) > 0) ? (KEY1) : (KEY2))
H
Hongze Cheng 已提交
107 108 109
// SBlockCol
int32_t tPutBlockCol(uint8_t *p, void *ph);
int32_t tGetBlockCol(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
110
int32_t tBlockColCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
111
// SBlock
H
Hongze Cheng 已提交
112
void    tBlockReset(SBlock *pBlock);
H
Hongze Cheng 已提交
113 114
int32_t tPutBlock(uint8_t *p, void *ph);
int32_t tGetBlock(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
115
int32_t tBlockCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
116
bool    tBlockHasSma(SBlock *pBlock);
H
Hongze Cheng 已提交
117 118 119
// SBlockIdx
int32_t tPutBlockIdx(uint8_t *p, void *ph);
int32_t tGetBlockIdx(uint8_t *p, void *ph);
120
int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
H
Hongze Cheng 已提交
121
// SColdata
H
Hongze Cheng 已提交
122 123
void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
void    tColDataReset(SColData *pColData);
H
Hongze Cheng 已提交
124
void    tColDataClear(void *ph);
H
Hongze Cheng 已提交
125
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
H
Hongze Cheng 已提交
126
int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
H
Hongze Cheng 已提交
127
int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
H
Hongze Cheng 已提交
128 129
int32_t tPutColData(uint8_t *p, SColData *pColData);
int32_t tGetColData(uint8_t *p, SColData *pColData);
H
Hongze Cheng 已提交
130
// SBlockData
H
Hongze Cheng 已提交
131 132
#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0)
#define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
H
Hongze Cheng 已提交
133 134
#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA))
#define tBlockDataLastKey(PBLOCKDATA)  TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA))
H
Hongze Cheng 已提交
135 136 137
int32_t   tBlockDataInit(SBlockData *pBlockData);
void      tBlockDataReset(SBlockData *pBlockData);
int32_t   tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema);
H
Hongze Cheng 已提交
138
int32_t   tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom);
H
Hongze Cheng 已提交
139
void      tBlockDataClearData(SBlockData *pBlockData);
H
Hongze Cheng 已提交
140
void      tBlockDataClear(SBlockData *pBlockData, int8_t deepClear);
H
Hongze Cheng 已提交
141 142 143 144 145 146
int32_t   tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
int32_t   tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema);
int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
int32_t   tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx);
void      tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
H
Hongze Cheng 已提交
147 148
int32_t   tPutBlockData(uint8_t *p, SBlockData *pBlockData);
int32_t   tGetBlockData(uint8_t *p, SBlockData *pBlockData);
H
Hongze Cheng 已提交
149
// SDelIdx
H
Hongze Cheng 已提交
150 151
int32_t tPutDelIdx(uint8_t *p, void *ph);
int32_t tGetDelIdx(uint8_t *p, void *ph);
152
int32_t tCmprDelIdx(void const *lhs, void const *rhs);
H
Hongze Cheng 已提交
153
// SDelData
H
Hongze Cheng 已提交
154 155
int32_t tPutDelData(uint8_t *p, void *ph);
int32_t tGetDelData(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
156
// SMapData
H
Hongze Cheng 已提交
157
#define tMapDataInit() ((SMapData){0})
H
Hongze Cheng 已提交
158 159 160
void    tMapDataReset(SMapData *pMapData);
void    tMapDataClear(SMapData *pMapData);
int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *));
H
Hongze Cheng 已提交
161
void    tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *));
H
Hongze Cheng 已提交
162 163
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
                       int32_t (*tItemCmprFn)(const void *, const void *), void *pItem);
H
Hongze Cheng 已提交
164 165 166 167 168
int32_t tPutMapData(uint8_t *p, SMapData *pMapData);
int32_t tGetMapData(uint8_t *p, SMapData *pMapData);
// other
int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision);
void    tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey);
H
Hongze Cheng 已提交
169
int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now);
H
Hongze Cheng 已提交
170
int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline);
H
Hongze Cheng 已提交
171
void    tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg);
H
Hongze Cheng 已提交
172 173 174 175 176
// tsdbMemTable ==============================================================================================
// SMemTable
int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
void    tsdbMemTableDestroy(SMemTable *pMemTable);
void    tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
H
Hongze Cheng 已提交
177 178 179 180
void    tsdbRefMemTable(SMemTable *pMemTable);
void    tsdbUnrefMemTable(SMemTable *pMemTable);
int32_t tsdbTakeMemSnapshot(STsdb *pTsdb, SMemTable **ppMem, SMemTable **ppIMem);
void    tsdbUntakeMemSnapshot(STsdb *pTsdb, SMemTable *pMem, SMemTable *pIMem);
H
Hongze Cheng 已提交
181 182 183 184 185 186
// STbDataIter
int32_t  tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
void    *tsdbTbDataIterDestroy(STbDataIter *pIter);
void     tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter);
bool     tsdbTbDataIterNext(STbDataIter *pIter);
H
Hongze Cheng 已提交
187 188
// STbData
int32_t tsdbGetNRowsInTbData(STbData *pTbData);
H
Hongze Cheng 已提交
189
// tsdbFile.c ==============================================================================================
H
Hongze Cheng 已提交
190
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
H
Hongze Cheng 已提交
191
void    tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]);
H
Hongze Cheng 已提交
192
bool    tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype);
H
Hongze Cheng 已提交
193
bool    tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
H
Hongze Cheng 已提交
194 195
int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
H
Hongze Cheng 已提交
196
int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype);
H
Hongze Cheng 已提交
197 198 199 200
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet);
int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet);
H
Hongze Cheng 已提交
201
// SDelFile
H
Hongze Cheng 已提交
202
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]);
H
Hongze Cheng 已提交
203
// tsdbFS.c ==============================================================================================
H
Hongze Cheng 已提交
204 205
int32_t tsdbFSOpen(STsdb *pTsdb, STsdbFS **ppFS);
int32_t tsdbFSClose(STsdbFS *pFS);
H
Hongze Cheng 已提交
206 207 208
int32_t tsdbFSBegin(STsdbFS *pFS);
int32_t tsdbFSCommit(STsdbFS *pFS);
int32_t tsdbFSRollback(STsdbFS *pFS);
H
Hongze Cheng 已提交
209 210 211

int32_t    tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile);
int32_t    tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet);
H
Hongze Cheng 已提交
212
void       tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid);
H
Hongze Cheng 已提交
213
SDelFile  *tsdbFSStateGetDelFile(STsdbFSState *pState);
H
Hongze Cheng 已提交
214
SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag);
H
Hongze Cheng 已提交
215 216 217
// tsdbReaderWriter.c ==============================================================================================
// SDataFWriter
int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet);
H
Hongze Cheng 已提交
218
int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
H
Hongze Cheng 已提交
219
int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
H
Hongze Cheng 已提交
220
int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf);
H
Hongze Cheng 已提交
221 222
int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx);
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2,
H
Hongze Cheng 已提交
223
                           SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg);
H
Hongze Cheng 已提交
224 225

SDFileSet *tsdbDataFWriterGetWSet(SDataFWriter *pWriter);
H
Hongze Cheng 已提交
226
int32_t    tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
H
Hongze Cheng 已提交
227 228
// SDataFReader
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet);
H
Hongze Cheng 已提交
229
int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
H
Hongze Cheng 已提交
230
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf);
H
Hongze Cheng 已提交
231
int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData, uint8_t **ppBuf);
H
Hongze Cheng 已提交
232
int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol,
H
Hongze Cheng 已提交
233
                        SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2);
H
Hongze Cheng 已提交
234
int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData,
H
Hongze Cheng 已提交
235
                          uint8_t **ppBuf1, uint8_t **ppBuf2);
H
Hongze Cheng 已提交
236
int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf);
H
Hongze Cheng 已提交
237 238
// SDelFWriter
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
H
Hongze Cheng 已提交
239
int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
H
Hongze Cheng 已提交
240 241 242
int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx);
int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf);
int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter);
H
Hongze Cheng 已提交
243 244
// SDelFReader
int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf);
H
Hongze Cheng 已提交
245
int32_t tsdbDelFReaderClose(SDelFReader **ppReader);
H
Hongze Cheng 已提交
246 247
int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf);
int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf);
H
Hongze Cheng 已提交
248

249 250 251 252
#define TSDB_CACHE_NO(c)       ((c).cacheLast == 0)
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
#define TSDB_CACHE_LAST(c)     (((c).cacheLast & 2) > 0)

253 254
// tsdbCache
int32_t tsdbOpenCache(STsdb *pTsdb);
H
Hongze Cheng 已提交
255
void    tsdbCloseCache(SLRUCache *pCache);
256
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
257
int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup);
258 259
int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
M
Minglei Jin 已提交
260
int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);
261

262 263
int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
264
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
265

266 267 268
void   tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode);

269 270
int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);

H
Hongze Cheng 已提交
271
// structs =======================
H
Hongze Cheng 已提交
272 273 274 275 276 277 278 279
typedef struct {
  int   minFid;
  int   midFid;
  int   maxFid;
  TSKEY minKey;
} SRtn;

struct STsdb {
H
Hongze Cheng 已提交
280 281 282 283 284 285 286 287
  char          *path;
  SVnode        *pVnode;
  TdThreadRwlock rwLock;
  STsdbKeepCfg   keepCfg;
  SMemTable     *mem;
  SMemTable     *imem;
  STsdbFS       *pFS;
  SLRUCache     *lruCache;
H
Hongze Cheng 已提交
288 289
};

H
Hongze Cheng 已提交
290
struct STable {
H
Hongze Cheng 已提交
291
  uint64_t  suid;
H
Hongze Cheng 已提交
292
  uint64_t  uid;
293 294
  STSchema *pSchema;       // latest schema
  STSchema *pCacheSchema;  // cached cache
H
Hongze Cheng 已提交
295
};
H
Hongze Cheng 已提交
296

H
refact  
Hongze Cheng 已提交
297 298 299 300 301
struct TSDBKEY {
  int64_t version;
  TSKEY   ts;
};

H
Hongze Cheng 已提交
302 303 304 305 306 307 308 309 310 311 312 313 314 315
typedef struct SMemSkipListNode SMemSkipListNode;
struct SMemSkipListNode {
  int8_t            level;
  SMemSkipListNode *forwards[0];
};
typedef struct SMemSkipList {
  uint32_t          seed;
  int64_t           size;
  int8_t            maxLevel;
  int8_t            level;
  SMemSkipListNode *pHead;
  SMemSkipListNode *pTail;
} SMemSkipList;

H
Hongze Cheng 已提交
316 317 318 319 320
struct SDelDataInfo {
  tb_uid_t suid;
  tb_uid_t uid;
};

H
Hongze Cheng 已提交
321
struct STbData {
H
Hongze Cheng 已提交
322 323
  tb_uid_t     suid;
  tb_uid_t     uid;
H
Hongze Cheng 已提交
324 325 326 327
  TSKEY        minKey;
  TSKEY        maxKey;
  int64_t      minVersion;
  int64_t      maxVersion;
H
Hongze Cheng 已提交
328
  int32_t      maxSkmVer;
H
Hongze Cheng 已提交
329 330
  SDelData    *pHead;
  SDelData    *pTail;
H
Hongze Cheng 已提交
331
  SMemSkipList sl;
H
Hongze Cheng 已提交
332
};
H
Hongze Cheng 已提交
333

H
refact  
Hongze Cheng 已提交
334
struct SMemTable {
H
Hongze Cheng 已提交
335 336 337 338 339 340 341 342 343 344 345
  SRWLatch         latch;
  STsdb           *pTsdb;
  SVBufPool       *pPool;
  volatile int32_t nRef;
  TSKEY            minKey;
  TSKEY            maxKey;
  int64_t          minVersion;
  int64_t          maxVersion;
  int64_t          nRow;
  int64_t          nDel;
  SArray          *aTbData;  // SArray<STbData*>
H
Hongze Cheng 已提交
346
};
H
Hongze Cheng 已提交
347

H
Hongze Cheng 已提交
348
struct TSDBROW {
H
Hongze Cheng 已提交
349
  int8_t type;  // 0 for row from tsRow, 1 for row from block data
H
Hongze Cheng 已提交
350 351 352 353 354 355
  union {
    struct {
      int64_t version;
      STSRow *pTSRow;
    };
    struct {
H
Hongze Cheng 已提交
356 357
      SBlockData *pBlockData;
      int32_t     iRow;
H
Hongze Cheng 已提交
358 359
    };
  };
H
Hongze Cheng 已提交
360 361
};

H
Hongze Cheng 已提交
362
struct SBlockIdx {
H
Hongze Cheng 已提交
363 364
  int64_t suid;
  int64_t uid;
H
Hongze Cheng 已提交
365 366 367 368
  int64_t offset;
  int64_t size;
};

H
Hongze Cheng 已提交
369 370
struct SMapData {
  int32_t  nItem;
H
Hongze Cheng 已提交
371 372
  int32_t *aOffset;
  int32_t  nData;
H
Hongze Cheng 已提交
373 374 375
  uint8_t *pData;
};

H
Hongze Cheng 已提交
376
typedef struct {
H
Hongze Cheng 已提交
377 378
  int16_t cid;
  int8_t  type;
H
Hongze Cheng 已提交
379
  int8_t  smaOn;
H
Hongze Cheng 已提交
380
  int8_t  flag;  // HAS_NONE|HAS_NULL|HAS_VALUE
H
Hongze Cheng 已提交
381 382 383 384 385
  int32_t offset;
  int32_t szBitmap;  // bitmap size
  int32_t szOffset;  // size of offset, only for variant-length data type
  int32_t szValue;   // compressed column value size
  int32_t szOrigin;  // original column value size (only save for variant data type)
H
Hongze Cheng 已提交
386 387 388
} SBlockCol;

typedef struct {
H
Hongze Cheng 已提交
389 390 391 392 393 394 395 396 397
  int32_t nRow;
  int8_t  cmprAlg;
  int64_t offset;      // block data offset
  int32_t szBlockCol;  // SBlockCol size
  int32_t szVersion;   // VERSION size
  int32_t szTSKEY;     // TSKEY size
  int32_t szBlock;     // total block size
  int64_t sOffset;     // sma offset
  int32_t nSma;        // sma size
H
Hongze Cheng 已提交
398 399
} SSubBlock;

H
Hongze Cheng 已提交
400
struct SBlock {
H
Hongze Cheng 已提交
401 402 403 404
  TSDBKEY   minKey;
  TSDBKEY   maxKey;
  int64_t   minVersion;
  int64_t   maxVersion;
H
Hongze Cheng 已提交
405
  int32_t   nRow;
H
Hongze Cheng 已提交
406 407 408
  int8_t    last;
  int8_t    hasDup;
  int8_t    nSubBlock;
H
Hongze Cheng 已提交
409
  SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS];
H
refact  
Hongze Cheng 已提交
410
};
H
Hongze Cheng 已提交
411

H
refact  
Hongze Cheng 已提交
412
struct SAggrBlkCol {
H
Hongze Cheng 已提交
413 414 415 416 417 418 419
  int16_t colId;
  int16_t maxIndex;
  int16_t minIndex;
  int16_t numOfNull;
  int64_t sum;
  int64_t max;
  int64_t min;
H
refact  
Hongze Cheng 已提交
420
};
H
Hongze Cheng 已提交
421

H
Hongze Cheng 已提交
422 423 424
struct SColData {
  int16_t  cid;
  int8_t   type;
H
Hongze Cheng 已提交
425
  int8_t   smaOn;
H
Hongze Cheng 已提交
426 427
  int32_t  nVal;
  uint8_t  flag;
H
Hongze Cheng 已提交
428
  uint8_t *pBitMap;
H
Hongze Cheng 已提交
429 430
  int32_t *aOffset;
  int32_t  nData;
H
Hongze Cheng 已提交
431
  uint8_t *pData;
H
refact  
Hongze Cheng 已提交
432
};
H
Hongze Cheng 已提交
433

H
Hongze Cheng 已提交
434
struct SBlockData {
H
Hongze Cheng 已提交
435 436 437
  int32_t  nRow;
  int64_t *aVersion;
  TSKEY   *aTSKEY;
H
Hongze Cheng 已提交
438 439
  SArray  *aIdx;      // SArray<int32_t>
  SArray  *aColData;  // SArray<SColData>
H
Hongze Cheng 已提交
440
};
H
Hongze Cheng 已提交
441 442 443 444

// ================== TSDB global config
extern bool tsdbForceKeepFile;

H
Hongze Cheng 已提交
445
#define TSDB_FS_ITER_FORWARD  TSDB_ORDER_ASC
H
Hongze Cheng 已提交
446 447
#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC

H
Hongze Cheng 已提交
448 449 450 451 452
struct TABLEID {
  tb_uid_t suid;
  tb_uid_t uid;
};

H
Hongze Cheng 已提交
453 454 455 456
struct STbDataIter {
  STbData          *pTbData;
  int8_t            backward;
  SMemSkipListNode *pNode;
H
Hongze Cheng 已提交
457 458
  TSDBROW          *pRow;
  TSDBROW           row;
H
Hongze Cheng 已提交
459 460
};

H
Hongze Cheng 已提交
461
struct SDelData {
H
Hongze Cheng 已提交
462 463 464 465
  int64_t   version;
  TSKEY     sKey;
  TSKEY     eKey;
  SDelData *pNext;
H
Hongze Cheng 已提交
466
};
H
Hongze Cheng 已提交
467

H
Hongze Cheng 已提交
468
struct SDelIdx {
H
Hongze Cheng 已提交
469 470 471 472
  tb_uid_t suid;
  tb_uid_t uid;
  int64_t  offset;
  int64_t  size;
H
Hongze Cheng 已提交
473
};
H
Hongze Cheng 已提交
474

H
Hongze Cheng 已提交
475
struct SDelFile {
H
Hongze Cheng 已提交
476
  int64_t commitID;
H
Hongze Cheng 已提交
477 478 479
  int64_t size;
  int64_t offset;
};
H
Hongze Cheng 已提交
480

H
Hongze Cheng 已提交
481
#pragma pack(push, 1)
H
Hongze Cheng 已提交
482
struct SBlockDataHdr {
H
more  
Hongze Cheng 已提交
483 484 485
  uint32_t delimiter;
  int64_t  suid;
  int64_t  uid;
H
Hongze Cheng 已提交
486
};
H
Hongze Cheng 已提交
487
#pragma pack(pop)
H
Hongze Cheng 已提交
488

H
Hongze Cheng 已提交
489
struct SHeadFile {
H
Hongze Cheng 已提交
490
  int64_t commitID;
H
Hongze Cheng 已提交
491 492 493 494 495
  int64_t size;
  int64_t offset;
};

struct SDataFile {
H
Hongze Cheng 已提交
496
  int64_t commitID;
H
Hongze Cheng 已提交
497 498 499 500
  int64_t size;
};

struct SLastFile {
H
Hongze Cheng 已提交
501
  int64_t commitID;
H
Hongze Cheng 已提交
502 503 504 505
  int64_t size;
};

struct SSmaFile {
H
Hongze Cheng 已提交
506
  int64_t commitID;
H
Hongze Cheng 已提交
507 508 509 510
  int64_t size;
};

struct SDFileSet {
H
Hongze Cheng 已提交
511 512 513 514 515 516
  SDiskID   diskId;
  int32_t   fid;
  SHeadFile fHead;
  SDataFile fData;
  SLastFile fLast;
  SSmaFile  fSma;
H
Hongze Cheng 已提交
517 518 519 520
};

struct SRowIter {
  TSDBROW  *pRow;
H
Hongze Cheng 已提交
521
  STSchema *pTSchema;
H
Hongze Cheng 已提交
522 523
  SColVal   colVal;
  int32_t   i;
H
Hongze Cheng 已提交
524
};
H
Hongze Cheng 已提交
525 526 527 528 529
struct SRowMerger {
  STSchema *pTSchema;
  int64_t   version;
  SArray   *pArray;  // SArray<SColVal>
};
H
Hongze Cheng 已提交
530

H
more  
Hongze Cheng 已提交
531 532 533 534 535 536
struct STsdbFSState {
  SDelFile *pDelFile;
  SArray   *aDFileSet;  // SArray<aDFileSet>
  SDelFile  delFile;
};

H
Hongze Cheng 已提交
537 538 539 540 541 542 543 544
struct STsdbFS {
  STsdb         *pTsdb;
  TdThreadRwlock lock;
  int8_t         inTxn;
  STsdbFSState  *cState;
  STsdbFSState  *nState;
};

H
Hongze Cheng 已提交
545 546 547 548 549 550
struct SDelFWriter {
  STsdb    *pTsdb;
  SDelFile  fDel;
  TdFilePtr pWriteH;
};

H
Hongze Cheng 已提交
551 552 553 554
#ifdef __cplusplus
}
#endif

555
#endif /*_TD_VNODE_TSDB_H_*/