tsdb.h 18.6 KB
Newer Older
L
Liu Jicong 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
Hongze Cheng 已提交
15 16 17 18

#ifndef _TD_VNODE_TSDB_H_
#define _TD_VNODE_TSDB_H_

H
Hongze Cheng 已提交
19 20
#include "vnodeInt.h"

H
Hongze Cheng 已提交
21 22 23 24
#ifdef __cplusplus
extern "C" {
#endif

H
Hongze Cheng 已提交
25 26 27 28 29 30 31 32 33
// tsdbDebug ================
// clang-format off
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
H
Hongze Cheng 已提交
34

H
Hongze Cheng 已提交
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
typedef struct TSDBROW        TSDBROW;
typedef struct TABLEID        TABLEID;
typedef struct TSDBKEY        TSDBKEY;
typedef struct SDelData       SDelData;
typedef struct SDelIdx        SDelIdx;
typedef struct STbData        STbData;
typedef struct SMemTable      SMemTable;
typedef struct STbDataIter    STbDataIter;
typedef struct STable         STable;
typedef struct SMapData       SMapData;
typedef struct SBlockIdx      SBlockIdx;
typedef struct SBlock         SBlock;
typedef struct SBlockStatis   SBlockStatis;
typedef struct SAggrBlkCol    SAggrBlkCol;
typedef struct SColData       SColData;
typedef struct SBlockDataHdr  SBlockDataHdr;
typedef struct SBlockData     SBlockData;
H
Hongze Cheng 已提交
52 53
typedef struct SDelFile       SDelFile;
typedef struct STsdbCacheFile STsdbCacheFile;
H
Hongze Cheng 已提交
54 55 56 57
typedef struct SHeadFile      SHeadFile;
typedef struct SDataFile      SDataFile;
typedef struct SLastFile      SLastFile;
typedef struct SSmaFile       SSmaFile;
H
Hongze Cheng 已提交
58
typedef struct SDFileSet      SDFileSet;
H
Hongze Cheng 已提交
59 60 61 62
typedef struct SDataFWriter   SDataFWriter;
typedef struct SDataFReader   SDataFReader;
typedef struct SDelFWriter    SDelFWriter;
typedef struct SDelFReader    SDelFReader;
H
Hongze Cheng 已提交
63 64
typedef struct SRowIter       SRowIter;
typedef struct STsdbFS        STsdbFS;
H
Hongze Cheng 已提交
65
typedef struct SRowMerger     SRowMerger;
H
Hongze Cheng 已提交
66
typedef struct STsdbFSState   STsdbFSState;
H
Hongze Cheng 已提交
67
typedef struct STsdbSnapHdr   STsdbSnapHdr;
H
Hongze Cheng 已提交
68

H
Hongze Cheng 已提交
69
#define TSDB_MAX_SUBBLOCKS 8
H
Hongze Cheng 已提交
70
#define TSDB_FHDR_SIZE     512
H
refact  
Hongze Cheng 已提交
71

H
Hongze Cheng 已提交
72 73 74
#define HAS_NONE  ((int8_t)0x1)
#define HAS_NULL  ((int8_t)0x2)
#define HAS_VALUE ((int8_t)0x4)
H
Hongze Cheng 已提交
75

H
Hongze Cheng 已提交
76 77 78
#define VERSION_MIN 0
#define VERSION_MAX INT64_MAX

H
Hongze Cheng 已提交
79 80 81
#define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN})
#define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX})

H
Hongze Cheng 已提交
82
// tsdbUtil.c ==============================================================================================
H
Hongze Cheng 已提交
83
// TSDBROW
H
Hongze Cheng 已提交
84 85
#define TSDBROW_TS(ROW)                       (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow])
#define TSDBROW_VERSION(ROW)                  (((ROW)->type == 0) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow])
H
Hongze Cheng 已提交
86
#define TSDBROW_SVERSION(ROW)                 TD_ROW_SVER((ROW)->pTSRow)
H
Hongze Cheng 已提交
87
#define TSDBROW_KEY(ROW)                      ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)})
H
Hongze Cheng 已提交
88 89
#define tsdbRowFromTSRow(VERSION, TSROW)      ((TSDBROW){.type = 0, .version = (VERSION), .pTSRow = (TSROW)})
#define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
H
Hongze Cheng 已提交
90 91 92
void    tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
H
Hongze Cheng 已提交
93
int32_t tsdbRowCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
94
// SRowIter
H
Hongze Cheng 已提交
95
void     tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);
H
Hongze Cheng 已提交
96
SColVal *tRowIterNext(SRowIter *pIter);
H
Hongze Cheng 已提交
97 98 99 100 101
// SRowMerger
int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
void    tRowMergerClear(SRowMerger *pMerger);
int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow);
int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow);
H
Hongze Cheng 已提交
102
// TABLEID
H
Hongze Cheng 已提交
103
int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
104
// TSDBKEY
H
Hongze Cheng 已提交
105
int32_t tsdbKeyCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
106 107
#define MIN_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) < 0) ? (KEY1) : (KEY2))
#define MAX_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) > 0) ? (KEY1) : (KEY2))
H
Hongze Cheng 已提交
108 109 110
// SBlockCol
int32_t tPutBlockCol(uint8_t *p, void *ph);
int32_t tGetBlockCol(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
111
int32_t tBlockColCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
112
// SBlock
H
Hongze Cheng 已提交
113
void    tBlockReset(SBlock *pBlock);
H
Hongze Cheng 已提交
114 115
int32_t tPutBlock(uint8_t *p, void *ph);
int32_t tGetBlock(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
116
int32_t tBlockCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
117
bool    tBlockHasSma(SBlock *pBlock);
H
Hongze Cheng 已提交
118
// SBlockIdx
H
Hongze Cheng 已提交
119
void    tBlockIdxReset(SBlockIdx *pBlockIdx);
H
Hongze Cheng 已提交
120 121
int32_t tPutBlockIdx(uint8_t *p, void *ph);
int32_t tGetBlockIdx(uint8_t *p, void *ph);
122
int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
H
Hongze Cheng 已提交
123
// SColdata
H
Hongze Cheng 已提交
124 125
void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
void    tColDataReset(SColData *pColData);
H
Hongze Cheng 已提交
126
void    tColDataClear(void *ph);
H
Hongze Cheng 已提交
127
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
H
Hongze Cheng 已提交
128
int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
H
Hongze Cheng 已提交
129
int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
H
Hongze Cheng 已提交
130
// SBlockData
H
Hongze Cheng 已提交
131 132
#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0)
#define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
H
Hongze Cheng 已提交
133 134
#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA))
#define tBlockDataLastKey(PBLOCKDATA)  TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA))
H
Hongze Cheng 已提交
135 136 137 138 139 140 141 142 143 144 145
int32_t   tBlockDataInit(SBlockData *pBlockData);
void      tBlockDataReset(SBlockData *pBlockData);
int32_t   tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema);
void      tBlockDataClearData(SBlockData *pBlockData);
void      tBlockDataClear(SBlockData *pBlockData);
int32_t   tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
int32_t   tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema);
int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
int32_t   tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx);
void      tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
H
Hongze Cheng 已提交
146
// SDelIdx
H
Hongze Cheng 已提交
147 148
int32_t tPutDelIdx(uint8_t *p, void *ph);
int32_t tGetDelIdx(uint8_t *p, void *ph);
149
int32_t tCmprDelIdx(void const *lhs, void const *rhs);
H
Hongze Cheng 已提交
150
// SDelData
H
Hongze Cheng 已提交
151 152
int32_t tPutDelData(uint8_t *p, void *ph);
int32_t tGetDelData(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
153
// SMapData
H
Hongze Cheng 已提交
154
#define tMapDataInit() ((SMapData){0})
H
Hongze Cheng 已提交
155 156 157
void    tMapDataReset(SMapData *pMapData);
void    tMapDataClear(SMapData *pMapData);
int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *));
H
Hongze Cheng 已提交
158
void    tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *));
H
Hongze Cheng 已提交
159 160
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
                       int32_t (*tItemCmprFn)(const void *, const void *), void *pItem);
H
Hongze Cheng 已提交
161 162 163 164 165 166
int32_t tPutMapData(uint8_t *p, SMapData *pMapData);
int32_t tGetMapData(uint8_t *p, SMapData *pMapData);
// other
int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision);
void    tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey);
int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline);
H
Hongze Cheng 已提交
167
void    tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg);
H
Hongze Cheng 已提交
168 169 170 171 172 173 174 175 176 177 178
// tsdbMemTable ==============================================================================================
// SMemTable
int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
void    tsdbMemTableDestroy(SMemTable *pMemTable);
void    tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
// STbDataIter
int32_t  tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
void    *tsdbTbDataIterDestroy(STbDataIter *pIter);
void     tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter);
bool     tsdbTbDataIterNext(STbDataIter *pIter);
H
Hongze Cheng 已提交
179 180
// STbData
int32_t tsdbGetNRowsInTbData(STbData *pTbData);
H
Hongze Cheng 已提交
181
// tsdbFile.c ==============================================================================================
H
Hongze Cheng 已提交
182
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
H
Hongze Cheng 已提交
183
void    tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]);
H
Hongze Cheng 已提交
184 185 186
bool    tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype);
int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
H
Hongze Cheng 已提交
187
int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype);
H
Hongze Cheng 已提交
188 189 190 191
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet);
int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet);
H
Hongze Cheng 已提交
192
// SDelFile
H
Hongze Cheng 已提交
193
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]);
H
Hongze Cheng 已提交
194
// tsdbFS.c ==============================================================================================
H
Hongze Cheng 已提交
195 196
int32_t tsdbFSOpen(STsdb *pTsdb, STsdbFS **ppFS);
int32_t tsdbFSClose(STsdbFS *pFS);
H
Hongze Cheng 已提交
197 198 199
int32_t tsdbFSBegin(STsdbFS *pFS);
int32_t tsdbFSCommit(STsdbFS *pFS);
int32_t tsdbFSRollback(STsdbFS *pFS);
H
Hongze Cheng 已提交
200 201 202 203 204

int32_t    tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile);
int32_t    tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet);
SDelFile  *tsdbFSStateGetDelFile(STsdbFSState *pState);
SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid);
H
Hongze Cheng 已提交
205 206 207
// tsdbReaderWriter.c ==============================================================================================
// SDataFWriter
int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet);
H
Hongze Cheng 已提交
208
int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
H
Hongze Cheng 已提交
209
int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
H
Hongze Cheng 已提交
210
int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx, uint8_t **ppBuf);
H
Hongze Cheng 已提交
211 212
int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, uint8_t **ppBuf, SBlockIdx *pBlockIdx);
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2,
H
Hongze Cheng 已提交
213
                           SBlockIdx *pBlockIdx, SBlock *pBlock, int8_t cmprAlg);
H
Hongze Cheng 已提交
214 215

SDFileSet *tsdbDataFWriterGetWSet(SDataFWriter *pWriter);
H
Hongze Cheng 已提交
216 217
// SDataFReader
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet);
H
Hongze Cheng 已提交
218
int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
H
Hongze Cheng 已提交
219
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx, uint8_t **ppBuf);
H
Hongze Cheng 已提交
220
int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData, uint8_t **ppBuf);
H
Hongze Cheng 已提交
221
int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, int16_t *aColId, int32_t nCol,
H
Hongze Cheng 已提交
222
                        SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2);
H
Hongze Cheng 已提交
223
int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData,
H
Hongze Cheng 已提交
224
                          uint8_t **ppBuf1, uint8_t **ppBuf2);
H
Hongze Cheng 已提交
225
int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg, uint8_t **ppBuf);
H
Hongze Cheng 已提交
226 227
// SDelFWriter
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
H
Hongze Cheng 已提交
228
int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
H
Hongze Cheng 已提交
229 230 231
int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, uint8_t **ppBuf, SDelIdx *pDelIdx);
int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx, uint8_t **ppBuf);
int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter);
H
Hongze Cheng 已提交
232 233
// SDelFReader
int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf);
H
Hongze Cheng 已提交
234
int32_t tsdbDelFReaderClose(SDelFReader **ppReader);
H
Hongze Cheng 已提交
235 236
int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, uint8_t **ppBuf);
int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf);
H
Hongze Cheng 已提交
237

238 239
// tsdbCache
int32_t tsdbOpenCache(STsdb *pTsdb);
H
Hongze Cheng 已提交
240
void    tsdbCloseCache(SLRUCache *pCache);
241
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
242
int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup);
243 244
int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
M
Minglei Jin 已提交
245
int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);
246

247
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
248

249 250
int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);

H
Hongze Cheng 已提交
251
// structs =======================
H
Hongze Cheng 已提交
252 253 254 255 256 257 258 259
typedef struct {
  int   minFid;
  int   midFid;
  int   maxFid;
  TSKEY minKey;
} SRtn;

struct STsdb {
H
refact  
Hongze Cheng 已提交
260 261 262 263 264 265 266 267 268
  char         *path;
  SVnode       *pVnode;
  TdThreadMutex mutex;
  bool          repoLocked;
  STsdbKeepCfg  keepCfg;
  SMemTable    *mem;
  SMemTable    *imem;
  SRtn          rtn;
  STsdbFS      *fs;
269
  SLRUCache    *lruCache;
H
Hongze Cheng 已提交
270 271
};

H
Hongze Cheng 已提交
272
struct STable {
H
Hongze Cheng 已提交
273
  uint64_t  suid;
H
Hongze Cheng 已提交
274
  uint64_t  uid;
275 276
  STSchema *pSchema;       // latest schema
  STSchema *pCacheSchema;  // cached cache
H
Hongze Cheng 已提交
277
};
H
Hongze Cheng 已提交
278

H
refact  
Hongze Cheng 已提交
279 280 281 282 283
struct TSDBKEY {
  int64_t version;
  TSKEY   ts;
};

H
Hongze Cheng 已提交
284 285 286 287 288 289 290 291 292 293 294 295 296 297
typedef struct SMemSkipListNode SMemSkipListNode;
struct SMemSkipListNode {
  int8_t            level;
  SMemSkipListNode *forwards[0];
};
typedef struct SMemSkipList {
  uint32_t          seed;
  int64_t           size;
  int8_t            maxLevel;
  int8_t            level;
  SMemSkipListNode *pHead;
  SMemSkipListNode *pTail;
} SMemSkipList;

H
Hongze Cheng 已提交
298 299 300 301 302
struct SDelDataInfo {
  tb_uid_t suid;
  tb_uid_t uid;
};

H
Hongze Cheng 已提交
303
struct STbData {
H
Hongze Cheng 已提交
304 305
  tb_uid_t     suid;
  tb_uid_t     uid;
H
Hongze Cheng 已提交
306 307 308 309
  TSKEY        minKey;
  TSKEY        maxKey;
  int64_t      minVersion;
  int64_t      maxVersion;
H
Hongze Cheng 已提交
310
  int32_t      maxSkmVer;
H
Hongze Cheng 已提交
311 312
  SDelData    *pHead;
  SDelData    *pTail;
H
Hongze Cheng 已提交
313
  SMemSkipList sl;
H
Hongze Cheng 已提交
314
};
H
Hongze Cheng 已提交
315

H
refact  
Hongze Cheng 已提交
316
struct SMemTable {
H
Hongze Cheng 已提交
317 318 319
  SRWLatch latch;
  STsdb   *pTsdb;
  int32_t  nRef;
H
Hongze Cheng 已提交
320 321 322 323
  TSKEY    minKey;
  TSKEY    maxKey;
  int64_t  minVersion;
  int64_t  maxVersion;
H
Hongze Cheng 已提交
324
  int64_t  nRow;
H
Hongze Cheng 已提交
325
  int64_t  nDel;
H
Hongze Cheng 已提交
326
  SArray  *aTbData;  // SArray<STbData*>
H
Hongze Cheng 已提交
327
};
H
Hongze Cheng 已提交
328 329 330 331

int tsdbLockRepo(STsdb *pTsdb);
int tsdbUnlockRepo(STsdb *pTsdb);

H
Hongze Cheng 已提交
332
struct TSDBROW {
H
Hongze Cheng 已提交
333
  int8_t type;  // 0 for row from tsRow, 1 for row from block data
H
Hongze Cheng 已提交
334 335 336 337 338 339
  union {
    struct {
      int64_t version;
      STSRow *pTSRow;
    };
    struct {
H
Hongze Cheng 已提交
340 341
      SBlockData *pBlockData;
      int32_t     iRow;
H
Hongze Cheng 已提交
342 343
    };
  };
H
Hongze Cheng 已提交
344 345
};

H
Hongze Cheng 已提交
346
struct SBlockIdx {
H
Hongze Cheng 已提交
347 348
  int64_t suid;
  int64_t uid;
H
Hongze Cheng 已提交
349 350 351 352
  TSKEY   minKey;
  TSKEY   maxKey;
  int64_t minVersion;
  int64_t maxVersion;
H
Hongze Cheng 已提交
353 354 355 356
  int64_t offset;
  int64_t size;
};

H
Hongze Cheng 已提交
357 358
struct SMapData {
  int32_t  nItem;
H
Hongze Cheng 已提交
359 360
  int32_t *aOffset;
  int32_t  nData;
H
Hongze Cheng 已提交
361 362 363
  uint8_t *pData;
};

H
Hongze Cheng 已提交
364
typedef struct {
H
Hongze Cheng 已提交
365 366
  int16_t cid;
  int8_t  type;
H
Hongze Cheng 已提交
367
  int8_t  smaOn;
H
Hongze Cheng 已提交
368
  int8_t  flag;  // HAS_NONE|HAS_NULL|HAS_VALUE
H
Hongze Cheng 已提交
369 370 371 372 373
  int32_t offset;
  int32_t szBitmap;  // bitmap size
  int32_t szOffset;  // size of offset, only for variant-length data type
  int32_t szValue;   // compressed column value size
  int32_t szOrigin;  // original column value size (only save for variant data type)
H
Hongze Cheng 已提交
374 375 376
} SBlockCol;

typedef struct {
H
Hongze Cheng 已提交
377 378 379 380 381 382 383 384 385
  int32_t nRow;
  int8_t  cmprAlg;
  int64_t offset;      // block data offset
  int32_t szBlockCol;  // SBlockCol size
  int32_t szVersion;   // VERSION size
  int32_t szTSKEY;     // TSKEY size
  int32_t szBlock;     // total block size
  int64_t sOffset;     // sma offset
  int32_t nSma;        // sma size
H
Hongze Cheng 已提交
386 387
} SSubBlock;

H
Hongze Cheng 已提交
388
struct SBlock {
H
Hongze Cheng 已提交
389 390 391 392
  TSDBKEY   minKey;
  TSDBKEY   maxKey;
  int64_t   minVersion;
  int64_t   maxVersion;
H
Hongze Cheng 已提交
393
  int32_t   nRow;
H
Hongze Cheng 已提交
394 395 396
  int8_t    last;
  int8_t    hasDup;
  int8_t    nSubBlock;
H
Hongze Cheng 已提交
397
  SSubBlock aSubBlock[TSDB_MAX_SUBBLOCKS];
H
refact  
Hongze Cheng 已提交
398
};
H
Hongze Cheng 已提交
399

H
refact  
Hongze Cheng 已提交
400
struct SAggrBlkCol {
H
Hongze Cheng 已提交
401 402 403 404 405 406 407
  int16_t colId;
  int16_t maxIndex;
  int16_t minIndex;
  int16_t numOfNull;
  int64_t sum;
  int64_t max;
  int64_t min;
H
refact  
Hongze Cheng 已提交
408
};
H
Hongze Cheng 已提交
409

H
Hongze Cheng 已提交
410 411 412
struct SColData {
  int16_t  cid;
  int8_t   type;
H
Hongze Cheng 已提交
413
  int8_t   smaOn;
H
Hongze Cheng 已提交
414 415
  int32_t  nVal;
  uint8_t  flag;
H
Hongze Cheng 已提交
416
  uint8_t *pBitMap;
H
Hongze Cheng 已提交
417 418
  int32_t *aOffset;
  int32_t  nData;
H
Hongze Cheng 已提交
419
  uint8_t *pData;
H
refact  
Hongze Cheng 已提交
420
};
H
Hongze Cheng 已提交
421

H
Hongze Cheng 已提交
422
struct SBlockData {
H
Hongze Cheng 已提交
423 424 425
  int32_t  nRow;
  int64_t *aVersion;
  TSKEY   *aTSKEY;
H
Hongze Cheng 已提交
426 427
  SArray  *aIdx;      // SArray<int32_t>
  SArray  *aColData;  // SArray<SColData>
H
Hongze Cheng 已提交
428
};
H
Hongze Cheng 已提交
429 430 431 432

// ================== TSDB global config
extern bool tsdbForceKeepFile;

H
Hongze Cheng 已提交
433
#define TSDB_FS_ITER_FORWARD  TSDB_ORDER_ASC
H
Hongze Cheng 已提交
434 435
#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC

H
Hongze Cheng 已提交
436 437 438 439 440
struct TABLEID {
  tb_uid_t suid;
  tb_uid_t uid;
};

H
Hongze Cheng 已提交
441 442 443 444
struct STbDataIter {
  STbData          *pTbData;
  int8_t            backward;
  SMemSkipListNode *pNode;
H
Hongze Cheng 已提交
445 446
  TSDBROW          *pRow;
  TSDBROW           row;
H
Hongze Cheng 已提交
447 448
};

H
Hongze Cheng 已提交
449
struct SDelData {
H
Hongze Cheng 已提交
450 451 452 453
  int64_t   version;
  TSKEY     sKey;
  TSKEY     eKey;
  SDelData *pNext;
H
Hongze Cheng 已提交
454
};
H
Hongze Cheng 已提交
455

H
Hongze Cheng 已提交
456
struct SDelIdx {
H
Hongze Cheng 已提交
457 458 459 460
  tb_uid_t suid;
  tb_uid_t uid;
  int64_t  offset;
  int64_t  size;
H
Hongze Cheng 已提交
461
};
H
Hongze Cheng 已提交
462

H
Hongze Cheng 已提交
463
struct SDelFile {
H
Hongze Cheng 已提交
464
  int64_t commitID;
H
Hongze Cheng 已提交
465 466 467
  int64_t size;
  int64_t offset;
};
H
Hongze Cheng 已提交
468

H
Hongze Cheng 已提交
469
#pragma pack(push, 1)
H
Hongze Cheng 已提交
470
struct SBlockDataHdr {
H
more  
Hongze Cheng 已提交
471 472 473
  uint32_t delimiter;
  int64_t  suid;
  int64_t  uid;
H
Hongze Cheng 已提交
474
};
H
Hongze Cheng 已提交
475
#pragma pack(pop)
H
Hongze Cheng 已提交
476

H
Hongze Cheng 已提交
477
struct SHeadFile {
H
Hongze Cheng 已提交
478
  int64_t commitID;
H
Hongze Cheng 已提交
479 480 481 482 483
  int64_t size;
  int64_t offset;
};

struct SDataFile {
H
Hongze Cheng 已提交
484
  int64_t commitID;
H
Hongze Cheng 已提交
485 486 487 488
  int64_t size;
};

struct SLastFile {
H
Hongze Cheng 已提交
489
  int64_t commitID;
H
Hongze Cheng 已提交
490 491 492 493
  int64_t size;
};

struct SSmaFile {
H
Hongze Cheng 已提交
494
  int64_t commitID;
H
Hongze Cheng 已提交
495 496 497 498
  int64_t size;
};

struct SDFileSet {
H
Hongze Cheng 已提交
499 500 501 502 503 504
  SDiskID   diskId;
  int32_t   fid;
  SHeadFile fHead;
  SDataFile fData;
  SLastFile fLast;
  SSmaFile  fSma;
H
Hongze Cheng 已提交
505 506 507 508
};

struct SRowIter {
  TSDBROW  *pRow;
H
Hongze Cheng 已提交
509
  STSchema *pTSchema;
H
Hongze Cheng 已提交
510 511
  SColVal   colVal;
  int32_t   i;
H
Hongze Cheng 已提交
512
};
H
Hongze Cheng 已提交
513 514 515 516 517
struct SRowMerger {
  STSchema *pTSchema;
  int64_t   version;
  SArray   *pArray;  // SArray<SColVal>
};
H
Hongze Cheng 已提交
518

H
more  
Hongze Cheng 已提交
519 520 521 522 523 524
struct STsdbFSState {
  SDelFile *pDelFile;
  SArray   *aDFileSet;  // SArray<aDFileSet>
  SDelFile  delFile;
};

H
Hongze Cheng 已提交
525 526 527 528 529 530 531 532
struct STsdbFS {
  STsdb         *pTsdb;
  TdThreadRwlock lock;
  int8_t         inTxn;
  STsdbFSState  *cState;
  STsdbFSState  *nState;
};

H
Hongze Cheng 已提交
533 534 535 536 537 538
struct SDelFWriter {
  STsdb    *pTsdb;
  SDelFile  fDel;
  TdFilePtr pWriteH;
};

H
Hongze Cheng 已提交
539 540 541 542
#ifdef __cplusplus
}
#endif

543
#endif /*_TD_VNODE_TSDB_H_*/