tsdb.h 31.6 KB
Newer Older
L
Liu Jicong 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
Hongze Cheng 已提交
15 16 17 18

#ifndef _TD_VNODE_TSDB_H_
#define _TD_VNODE_TSDB_H_

H
Hongze Cheng 已提交
19 20
#include "vnodeInt.h"

H
Hongze Cheng 已提交
21 22 23 24
#ifdef __cplusplus
extern "C" {
#endif

H
Hongze Cheng 已提交
25 26
// tsdbDebug ================
// clang-format off
H
Hongze Cheng 已提交
27 28 29 30 31 32
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSD FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSD ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSD WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSD ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSD ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
H
Hongze Cheng 已提交
33
// clang-format on
H
Hongze Cheng 已提交
34

H
Hongze Cheng 已提交
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
typedef struct TSDBROW          TSDBROW;
typedef struct TABLEID          TABLEID;
typedef struct TSDBKEY          TSDBKEY;
typedef struct SDelData         SDelData;
typedef struct SDelIdx          SDelIdx;
typedef struct STbData          STbData;
typedef struct SMemTable        SMemTable;
typedef struct STbDataIter      STbDataIter;
typedef struct SMapData         SMapData;
typedef struct SBlockIdx        SBlockIdx;
typedef struct SDataBlk         SDataBlk;
typedef struct SSttBlk          SSttBlk;
typedef struct SDiskDataHdr     SDiskDataHdr;
typedef struct SBlockData       SBlockData;
typedef struct SDelFile         SDelFile;
typedef struct SHeadFile        SHeadFile;
typedef struct SDataFile        SDataFile;
typedef struct SSttFile         SSttFile;
typedef struct SSmaFile         SSmaFile;
typedef struct SDFileSet        SDFileSet;
typedef struct SDataFWriter     SDataFWriter;
typedef struct SDataFReader     SDataFReader;
typedef struct SDelFWriter      SDelFWriter;
typedef struct SDelFReader      SDelFReader;
59
typedef struct STSDBRowIter     STSDBRowIter;
H
Hongze Cheng 已提交
60 61 62 63 64 65 66 67 68 69 70
typedef struct STsdbFS          STsdbFS;
typedef struct SRowMerger       SRowMerger;
typedef struct STsdbReadSnap    STsdbReadSnap;
typedef struct SBlockInfo       SBlockInfo;
typedef struct SSmaInfo         SSmaInfo;
typedef struct SBlockCol        SBlockCol;
typedef struct SVersionRange    SVersionRange;
typedef struct SLDataIter       SLDataIter;
typedef struct SDiskCol         SDiskCol;
typedef struct SDiskData        SDiskData;
typedef struct SDiskDataBuilder SDiskDataBuilder;
H
Hongze Cheng 已提交
71
typedef struct SBlkInfo         SBlkInfo;
H
Hongze Cheng 已提交
72 73
typedef struct STsdbDataIter2   STsdbDataIter2;
typedef struct STsdbFilterInfo  STsdbFilterInfo;
H
Hongze Cheng 已提交
74

H
Hongze Cheng 已提交
75 76 77
#define TSDBROW_ROW_FMT ((int8_t)0x0)
#define TSDBROW_COL_FMT ((int8_t)0x1)

H
Hongze Cheng 已提交
78 79 80
#define TSDB_FILE_DLMT     ((uint32_t)0xF00AFA0F)
#define TSDB_MAX_SUBBLOCKS 8
#define TSDB_FHDR_SIZE     512
H
refact  
Hongze Cheng 已提交
81

H
Hongze Cheng 已提交
82 83 84
#define VERSION_MIN 0
#define VERSION_MAX INT64_MAX

H
Hongze Cheng 已提交
85 86 87
#define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN})
#define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX})

H
Hongze Cheng 已提交
88 89
#define TABLE_SAME_SCHEMA(SUID1, UID1, SUID2, UID2) ((SUID1) ? (SUID1) == (SUID2) : (UID1) == (UID2))

H
Hongze Cheng 已提交
90 91 92 93 94 95
#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM))
#define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \
  ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE))
#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE))
#define PAGE_OFFSET(PGNO, PAGE)            (((PGNO)-1) * (PAGE))
#define OFFSET_PGNO(OFFSET, PAGE)          ((OFFSET) / (PAGE) + 1)
H
Hongze Cheng 已提交
96 97 98 99 100

static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
  int64_t fOffSet = LOGIC_TO_FILE_OFFSET(lSize, szPage);
  int64_t pgno = OFFSET_PGNO(fOffSet, szPage);

H
Hongze Cheng 已提交
101
  if (fOffSet % szPage == 0) {
H
Hongze Cheng 已提交
102 103 104 105 106
    pgno--;
  }

  return pgno * szPage;
}
H
Hongze Cheng 已提交
107

H
Hongze Cheng 已提交
108
// tsdbUtil.c ==============================================================================================
H
Hongze Cheng 已提交
109
// TSDBROW
H
Hongze Cheng 已提交
110 111 112
#define TSDBROW_TS(ROW) (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow])
#define TSDBROW_VERSION(ROW) \
  (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow])
H
Hongze Cheng 已提交
113
#define TSDBROW_SVERSION(ROW)            ((ROW)->type == TSDBROW_ROW_FMT ? (ROW)->pTSRow->sver : -1)
H
Hongze Cheng 已提交
114 115 116 117
#define TSDBROW_KEY(ROW)                 ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)})
#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = TSDBROW_ROW_FMT, .version = (VERSION), .pTSRow = (TSROW)})
#define tsdbRowFromBlockData(BLOCKDATA, IROW) \
  ((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
H
Hongze Cheng 已提交
118 119

void    tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
H
Hongze Cheng 已提交
120
int32_t tsdbRowCmprFn(const void *p1, const void *p2);
121
// STSDBRowIter
H
Hongze Cheng 已提交
122 123
int32_t  tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);
void     tsdbRowClose(STSDBRowIter *pIter);
124
SColVal *tsdbRowIterNext(STSDBRowIter *pIter);
H
Hongze Cheng 已提交
125
// SRowMerger
M
Minglei Jin 已提交
126
int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema);
H
Hongze Cheng 已提交
127
int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
128

M
Minglei Jin 已提交
129 130
// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
void tsdbRowMergerClear(SRowMerger *pMerger);
131
// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow);
H
Hongze Cheng 已提交
132
int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow);
H
Hongze Cheng 已提交
133
// TABLEID
H
Hongze Cheng 已提交
134
int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
135
// TSDBKEY
H
Hongze Cheng 已提交
136 137
#define MIN_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) < 0) ? (KEY1) : (KEY2))
#define MAX_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) > 0) ? (KEY1) : (KEY2))
H
Hongze Cheng 已提交
138 139 140
// SBlockCol
int32_t tPutBlockCol(uint8_t *p, void *ph);
int32_t tGetBlockCol(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
141
int32_t tBlockColCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
142
// SDataBlk
H
Hongze Cheng 已提交
143
void    tDataBlkReset(SDataBlk *pBlock);
H
Hongze Cheng 已提交
144 145
int32_t tPutDataBlk(uint8_t *p, void *ph);
int32_t tGetDataBlk(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
146 147
int32_t tDataBlkCmprFn(const void *p1, const void *p2);
bool    tDataBlkHasSma(SDataBlk *pDataBlk);
H
Hongze Cheng 已提交
148 149 150
// SSttBlk
int32_t tPutSttBlk(uint8_t *p, void *ph);
int32_t tGetSttBlk(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
151 152 153
// SBlockIdx
int32_t tPutBlockIdx(uint8_t *p, void *ph);
int32_t tGetBlockIdx(uint8_t *p, void *ph);
154
int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
155
int32_t tCmprBlockL(void const *lhs, void const *rhs);
H
Hongze Cheng 已提交
156
// SBlockData
H
Hongze Cheng 已提交
157 158 159 160 161 162 163 164 165 166 167
#define tBlockDataFirstRow(PBLOCKDATA)             tsdbRowFromBlockData(PBLOCKDATA, 0)
#define tBlockDataLastRow(PBLOCKDATA)              tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
#define tBlockDataFirstKey(PBLOCKDATA)             TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA))
#define tBlockDataLastKey(PBLOCKDATA)              TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA))
#define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX])

int32_t tBlockDataCreate(SBlockData *pBlockData);
void    tBlockDataDestroy(SBlockData *pBlockData);
int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid);
void    tBlockDataReset(SBlockData *pBlockData);
int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
H
Hongze Cheng 已提交
168
int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid);
H
Hongze Cheng 已提交
169
int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
H
Hongze Cheng 已提交
170 171 172 173 174
void    tBlockDataClear(SBlockData *pBlockData);
void    tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
                       int32_t aBufN[]);
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
H
Hongze Cheng 已提交
175
// SDiskDataHdr
H
Hongze Cheng 已提交
176
int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr);
H
Hongze Cheng 已提交
177
int32_t tGetDiskDataHdr(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
178
// SDelIdx
H
Hongze Cheng 已提交
179 180
int32_t tPutDelIdx(uint8_t *p, void *ph);
int32_t tGetDelIdx(uint8_t *p, void *ph);
181
int32_t tCmprDelIdx(void const *lhs, void const *rhs);
H
Hongze Cheng 已提交
182
// SDelData
H
Hongze Cheng 已提交
183 184
int32_t tPutDelData(uint8_t *p, void *ph);
int32_t tGetDelData(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
185
// SMapData
H
Hongze Cheng 已提交
186
#define tMapDataInit() ((SMapData){0})
H
Hongze Cheng 已提交
187 188 189
void    tMapDataReset(SMapData *pMapData);
void    tMapDataClear(SMapData *pMapData);
int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *));
H
Hongze Cheng 已提交
190
int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo);
H
Hongze Cheng 已提交
191
void    tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *));
H
Hongze Cheng 已提交
192 193
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
                       int32_t (*tItemCmprFn)(const void *, const void *), void *pItem);
H
Hongze Cheng 已提交
194 195
int32_t tPutMapData(uint8_t *p, SMapData *pMapData);
int32_t tGetMapData(uint8_t *p, SMapData *pMapData);
H
Hongze Cheng 已提交
196 197
int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItemFn)(uint8_t *, void *),
                        SArray **ppArray);
H
Hongze Cheng 已提交
198 199 200
// other
int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision);
void    tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey);
H
Hongze Cheng 已提交
201
int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now);
H
Hongze Cheng 已提交
202
int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline);
H
Hongze Cheng 已提交
203 204
int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg);
int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg);
H
Hongze Cheng 已提交
205 206
int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut,
                     int32_t *szOut, uint8_t **ppBuf);
H
Hongze Cheng 已提交
207 208
int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut,
                       uint8_t **ppBuf);
H
Hongze Cheng 已提交
209
int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int32_t nOut,
H
Hongze Cheng 已提交
210
                        uint8_t **ppBuf);
H
Hongze Cheng 已提交
211 212
int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData,
                          uint8_t **ppBuf);
H
Hongze Cheng 已提交
213
int32_t tRowInfoCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
214 215
// tsdbMemTable ==============================================================================================
// SMemTable
H
Hongze Cheng 已提交
216
int32_t  tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
H
Hongze Cheng 已提交
217
void     tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive);
H
Hongze Cheng 已提交
218
STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid);
H
Hongze Cheng 已提交
219
int32_t  tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode);
H
Hongze Cheng 已提交
220
int32_t  tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive);
H
Hongze Cheng 已提交
221
SArray  *tsdbMemTableGetTbDataArray(SMemTable *pMemTable);
H
Hongze Cheng 已提交
222
// STbDataIter
H
Hongze Cheng 已提交
223 224 225 226
int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
void   *tsdbTbDataIterDestroy(STbDataIter *pIter);
void    tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
bool    tsdbTbDataIterNext(STbDataIter *pIter);
H
Hongze Cheng 已提交
227
void    tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum);
D
dapan1121 已提交
228

H
Hongze Cheng 已提交
229 230
// STbData
int32_t tsdbGetNRowsInTbData(STbData *pTbData);
H
Hongze Cheng 已提交
231
// tsdbFile.c ==============================================================================================
H
Hongze Cheng 已提交
232
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
H
Hongze Cheng 已提交
233

H
Hongze Cheng 已提交
234
bool    tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
H
Hongze Cheng 已提交
235
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
H
Hongze Cheng 已提交
236 237
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
H
Hongze Cheng 已提交
238
int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile);
H
Hongze Cheng 已提交
239
int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile);
H
Hongze Cheng 已提交
240 241 242 243
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet);
int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet);
H
Hongze Cheng 已提交
244 245 246

void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]);
void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]);
H
Hongze Cheng 已提交
247
void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]);
H
Hongze Cheng 已提交
248
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]);
249

H
Hongze Cheng 已提交
250
// SDelFile
H
Hongze Cheng 已提交
251
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]);
H
Hongze Cheng 已提交
252
// tsdbFS.c ==============================================================================================
H
Hongze Cheng 已提交
253
int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback);
H
Hongze Cheng 已提交
254 255 256 257
int32_t tsdbFSClose(STsdb *pTsdb);
int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS);
void    tsdbFSDestroy(STsdbFS *pFS);
int32_t tDFileSetCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
258 259 260
int32_t tsdbFSCommit(STsdb *pTsdb);
int32_t tsdbFSRollback(STsdb *pTsdb);
int32_t tsdbFSPrepareCommit(STsdb *pTsdb, STsdbFS *pFS);
H
Hongze Cheng 已提交
261 262 263 264 265
int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS);
void    tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS);

int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet);
int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile);
H
Hongze Cheng 已提交
266 267 268
// tsdbReaderWriter.c ==============================================================================================
// SDataFWriter
int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet);
H
Hongze Cheng 已提交
269
int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
H
Hongze Cheng 已提交
270
int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
H
Hongze Cheng 已提交
271
int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx);
H
Hongze Cheng 已提交
272
int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx);
H
Hongze Cheng 已提交
273
int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk);
H
Hongze Cheng 已提交
274
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
H
Hongze Cheng 已提交
275
                           int8_t cmprAlg, int8_t toLast);
H
Hongze Cheng 已提交
276
int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo);
H
Hongze Cheng 已提交
277 278

int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
H
Hongze Cheng 已提交
279 280
// SDataFReader
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet);
H
Hongze Cheng 已提交
281
int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
H
Hongze Cheng 已提交
282
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx);
H
Hongze Cheng 已提交
283
int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk);
H
Hongze Cheng 已提交
284
int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk);
H
Hongze Cheng 已提交
285 286
int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
H
Hongze Cheng 已提交
287
int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData);
H
Hongze Cheng 已提交
288
int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
H
Hongze Cheng 已提交
289
int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
H
Hongze Cheng 已提交
290 291
// SDelFWriter
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
H
Hongze Cheng 已提交
292
int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
H
Hongze Cheng 已提交
293 294
int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx);
int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx);
H
Hongze Cheng 已提交
295
int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter);
H
Hongze Cheng 已提交
296
// SDelFReader
H
Hongze Cheng 已提交
297
int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb);
H
Hongze Cheng 已提交
298
int32_t tsdbDelFReaderClose(SDelFReader **ppReader);
H
Hongze Cheng 已提交
299 300
int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData);
int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx);
H
Hongze Cheng 已提交
301
// tsdbRead.c ==============================================================================================
H
Hongze Cheng 已提交
302
int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap);
303
void    tsdbUntakeReadSnap(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive);
H
Hongze Cheng 已提交
304 305
// tsdbMerge.c ==============================================================================================
int32_t tsdbMerge(STsdb *pTsdb);
H
Hongze Cheng 已提交
306

307 308 309 310
#define TSDB_CACHE_NO(c)       ((c).cacheLast == 0)
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
#define TSDB_CACHE_LAST(c)     (((c).cacheLast & 2) > 0)

H
Hongze Cheng 已提交
311 312 313 314 315
// tsdbDiskData ==============================================================================================
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder);
void   *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder);
int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
                             uint8_t calcSma);
H
Hongze Cheng 已提交
316 317
int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder);
int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId);
H
Hongze Cheng 已提交
318
int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo);
H
Hongze Cheng 已提交
319 320 321 322 323 324 325
// tsdbDataIter.c ==============================================================================================
#define TSDB_MEM_TABLE_DATA_ITER 0
#define TSDB_DATA_FILE_DATA_ITER 1
#define TSDB_STT_FILE_DATA_ITER  2
#define TSDB_TOMB_FILE_DATA_ITER 3

#define TSDB_FILTER_FLAG_BY_VERSION 0x1
H
Hongze Cheng 已提交
326
#define TSDB_FILTER_FLAG_BY_TABLEID 0x2
H
Hongze Cheng 已提交
327 328 329 330 331 332 333 334 335 336 337 338

#define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn)))
/* open */
int32_t tsdbOpenDataFileDataIter(SDataFReader *pReader, STsdbDataIter2 **ppIter);
int32_t tsdbOpenSttFileDataIter(SDataFReader *pReader, int32_t iStt, STsdbDataIter2 **ppIter);
int32_t tsdbOpenTombFileDataIter(SDelFReader *pReader, STsdbDataIter2 **ppIter);
/* close */
void tsdbCloseDataIter2(STsdbDataIter2 *pIter);
/* cmpr */
int32_t tsdbDataIterCmprFn(const SRBTreeNode *pNode1, const SRBTreeNode *pNode2);
/* next */
int32_t tsdbDataIterNext2(STsdbDataIter2 *pIter, STsdbFilterInfo *pFilterInfo);
H
Hongze Cheng 已提交
339

H
Hongze Cheng 已提交
340
// structs =======================
H
Hongze Cheng 已提交
341 342 343 344 345
struct STsdbFS {
  SDelFile *pDelFile;
  SArray   *aDFileSet;  // SArray<SDFileSet>
};

H
Hongze Cheng 已提交
346
struct STsdb {
H
Hongze Cheng 已提交
347 348 349
  char          *path;
  SVnode        *pVnode;
  STsdbKeepCfg   keepCfg;
H
more  
Hongze Cheng 已提交
350
  TdThreadRwlock rwLock;
H
Hongze Cheng 已提交
351 352
  SMemTable     *mem;
  SMemTable     *imem;
H
Hongze Cheng 已提交
353
  STsdbFS        fs;
H
Hongze Cheng 已提交
354
  SLRUCache     *lruCache;
355
  TdThreadMutex  lruMutex;
356 357
  SLRUCache     *biCache;
  TdThreadMutex  biMutex;
H
Hongze Cheng 已提交
358 359 360
#ifdef USE_DEV_CODE
  struct STFileSystem *pFS;
#endif
H
Hongze Cheng 已提交
361 362
};

H
refact  
Hongze Cheng 已提交
363 364 365 366 367
struct TSDBKEY {
  int64_t version;
  TSKEY   ts;
};

368 369 370 371 372
struct SVersionRange {
  uint64_t minVer;
  uint64_t maxVer;
};

H
Hongze Cheng 已提交
373 374 375
typedef struct SMemSkipListNode SMemSkipListNode;
struct SMemSkipListNode {
  int8_t            level;
H
Hongze Cheng 已提交
376 377
  int8_t            flag;  // TSDBROW_ROW_FMT for row format, TSDBROW_COL_FMT for col format
  int32_t           iRow;
H
Hongze Cheng 已提交
378
  int64_t           version;
H
Hongze Cheng 已提交
379
  void             *pData;
H
Hongze Cheng 已提交
380 381
  SMemSkipListNode *forwards[0];
};
H
Hongze Cheng 已提交
382

H
Hongze Cheng 已提交
383 384
typedef struct SMemSkipList {
  int64_t           size;
H
Hongze Cheng 已提交
385
  uint32_t          seed;
H
Hongze Cheng 已提交
386 387 388 389 390 391
  int8_t            maxLevel;
  int8_t            level;
  SMemSkipListNode *pHead;
  SMemSkipListNode *pTail;
} SMemSkipList;

H
Hongze Cheng 已提交
392
struct STbData {
H
Hongze Cheng 已提交
393 394
  tb_uid_t     suid;
  tb_uid_t     uid;
H
Hongze Cheng 已提交
395 396
  TSKEY        minKey;
  TSKEY        maxKey;
H
Hongze Cheng 已提交
397 398
  SDelData    *pHead;
  SDelData    *pTail;
H
Hongze Cheng 已提交
399
  SMemSkipList sl;
H
Hongze Cheng 已提交
400
  STbData     *next;
H
Hongze Cheng 已提交
401
};
H
Hongze Cheng 已提交
402

H
refact  
Hongze Cheng 已提交
403
struct SMemTable {
H
Hongze Cheng 已提交
404 405 406 407
  SRWLatch         latch;
  STsdb           *pTsdb;
  SVBufPool       *pPool;
  volatile int32_t nRef;
H
Hongze Cheng 已提交
408 409
  int64_t          minVer;
  int64_t          maxVer;
H
Hongze Cheng 已提交
410 411 412 413
  TSKEY            minKey;
  TSKEY            maxKey;
  int64_t          nRow;
  int64_t          nDel;
H
Hongze Cheng 已提交
414 415 416
  int32_t          nTbData;
  int32_t          nBucket;
  STbData        **aBucket;
H
Hongze Cheng 已提交
417
};
H
Hongze Cheng 已提交
418

H
Hongze Cheng 已提交
419
struct TSDBROW {
H
Hongze Cheng 已提交
420
  int8_t type;  // TSDBROW_ROW_FMT for row from tsRow, TSDBROW_COL_FMT for row from block data
H
Hongze Cheng 已提交
421 422 423
  union {
    struct {
      int64_t version;
H
Hongze Cheng 已提交
424
      SRow   *pTSRow;
H
Hongze Cheng 已提交
425 426
    };
    struct {
H
Hongze Cheng 已提交
427 428
      SBlockData *pBlockData;
      int32_t     iRow;
H
Hongze Cheng 已提交
429 430
    };
  };
H
Hongze Cheng 已提交
431 432
};

H
Hongze Cheng 已提交
433
struct SBlockIdx {
H
Hongze Cheng 已提交
434 435
  int64_t suid;
  int64_t uid;
H
Hongze Cheng 已提交
436 437 438 439
  int64_t offset;
  int64_t size;
};

H
Hongze Cheng 已提交
440 441
struct SMapData {
  int32_t  nItem;
H
Hongze Cheng 已提交
442
  int32_t  nData;
443
  int32_t *aOffset;
H
Hongze Cheng 已提交
444 445 446
  uint8_t *pData;
};

H
Hongze Cheng 已提交
447
struct SBlockCol {
H
Hongze Cheng 已提交
448 449 450 451 452 453 454 455 456
  int16_t cid;
  int8_t  type;
  int8_t  smaOn;
  int8_t  flag;      // HAS_NONE|HAS_NULL|HAS_VALUE
  int32_t szOrigin;  // original column value size (only save for variant data type)
  int32_t szBitmap;  // bitmap size, 0 only for flag == HAS_VAL
  int32_t szOffset;  // offset size, 0 only for non-variant-length type
  int32_t szValue;   // value size, 0 when flag == (HAS_NULL | HAS_NONE)
  int32_t offset;
H
Hongze Cheng 已提交
457
};
H
Hongze Cheng 已提交
458

H
Hongze Cheng 已提交
459
struct SBlockInfo {
H
Hongze Cheng 已提交
460 461 462
  int64_t offset;  // block data offset
  int32_t szBlock;
  int32_t szKey;
H
Hongze Cheng 已提交
463
};
H
Hongze Cheng 已提交
464

H
Hongze Cheng 已提交
465
struct SSmaInfo {
H
Hongze Cheng 已提交
466 467
  int64_t offset;
  int32_t size;
H
Hongze Cheng 已提交
468
};
H
Hongze Cheng 已提交
469

H
Hongze Cheng 已提交
470
struct SBlkInfo {
H
Hongze Cheng 已提交
471 472
  int64_t minUid;
  int64_t maxUid;
H
Hongze Cheng 已提交
473 474
  TSKEY   minKey;
  TSKEY   maxKey;
H
Hongze Cheng 已提交
475 476
  int64_t minVer;
  int64_t maxVer;
H
Hongze Cheng 已提交
477 478 479
  TSDBKEY minTKey;
  TSDBKEY maxTKey;
};
H
Hongze Cheng 已提交
480

H
Hongze Cheng 已提交
481
struct SDataBlk {
H
Hongze Cheng 已提交
482 483 484 485 486 487 488 489 490
  TSDBKEY    minKey;
  TSDBKEY    maxKey;
  int64_t    minVer;
  int64_t    maxVer;
  int32_t    nRow;
  int8_t     hasDup;
  int8_t     nSubBlock;
  SBlockInfo aSubBlock[TSDB_MAX_SUBBLOCKS];
  SSmaInfo   smaInfo;
H
refact  
Hongze Cheng 已提交
491
};
H
Hongze Cheng 已提交
492

H
Hongze Cheng 已提交
493
struct SSttBlk {
H
Hongze Cheng 已提交
494 495 496
  int64_t    suid;
  int64_t    minUid;
  int64_t    maxUid;
H
Hongze Cheng 已提交
497 498
  TSKEY      minKey;
  TSKEY      maxKey;
H
Hongze Cheng 已提交
499 500 501 502
  int64_t    minVer;
  int64_t    maxVer;
  int32_t    nRow;
  SBlockInfo bInfo;
H
Hongze Cheng 已提交
503 504
};

H
Hongze Cheng 已提交
505 506 507 508
// (SBlockData){.suid = 0, .uid = 0}: block data not initialized
// (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file
// (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file
// (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file
H
Hongze Cheng 已提交
509
struct SBlockData {
H
Hongze Cheng 已提交
510 511 512 513 514 515 516 517
  int64_t   suid;      // 0 means normal table block data, otherwise child table block data
  int64_t   uid;       // 0 means block data in .last file, otherwise in .data file
  int32_t   nRow;      // number of rows
  int64_t  *aUid;      // uids of each row, only exist in block data in .last file (uid == 0)
  int64_t  *aVersion;  // versions of each row
  TSKEY    *aTSKEY;    // timestamp of each row
  int32_t   nColData;
  SColData *aColData;
H
Hongze Cheng 已提交
518
};
H
Hongze Cheng 已提交
519

H
Hongze Cheng 已提交
520 521 522 523 524
struct TABLEID {
  tb_uid_t suid;
  tb_uid_t uid;
};

H
Hongze Cheng 已提交
525 526 527 528
struct STbDataIter {
  STbData          *pTbData;
  int8_t            backward;
  SMemSkipListNode *pNode;
H
Hongze Cheng 已提交
529 530
  TSDBROW          *pRow;
  TSDBROW           row;
H
Hongze Cheng 已提交
531 532
};

H
Hongze Cheng 已提交
533
struct SDelData {
H
Hongze Cheng 已提交
534 535 536 537
  int64_t   version;
  TSKEY     sKey;
  TSKEY     eKey;
  SDelData *pNext;
H
Hongze Cheng 已提交
538
};
H
Hongze Cheng 已提交
539

H
Hongze Cheng 已提交
540
struct SDelIdx {
H
Hongze Cheng 已提交
541 542 543 544
  tb_uid_t suid;
  tb_uid_t uid;
  int64_t  offset;
  int64_t  size;
H
Hongze Cheng 已提交
545
};
H
Hongze Cheng 已提交
546

H
Hongze Cheng 已提交
547
struct SDiskDataHdr {
H
more  
Hongze Cheng 已提交
548
  uint32_t delimiter;
H
Hongze Cheng 已提交
549
  uint32_t fmtVer;
H
more  
Hongze Cheng 已提交
550 551
  int64_t  suid;
  int64_t  uid;
H
Hongze Cheng 已提交
552 553 554 555
  int32_t  szUid;
  int32_t  szVer;
  int32_t  szKey;
  int32_t  szBlkCol;
H
Hongze Cheng 已提交
556
  int32_t  nRow;
H
Hongze Cheng 已提交
557
  int8_t   cmprAlg;
H
Hongze Cheng 已提交
558 559
};

H
Hongze Cheng 已提交
560 561 562 563 564 565 566 567
struct SDelFile {
  volatile int32_t nRef;

  int64_t commitID;
  int64_t size;
  int64_t offset;
};

H
Hongze Cheng 已提交
568
struct SHeadFile {
H
Hongze Cheng 已提交
569 570
  volatile int32_t nRef;

H
Hongze Cheng 已提交
571
  int64_t commitID;
H
Hongze Cheng 已提交
572 573 574 575 576
  int64_t size;
  int64_t offset;
};

struct SDataFile {
H
Hongze Cheng 已提交
577 578
  volatile int32_t nRef;

H
Hongze Cheng 已提交
579
  int64_t commitID;
H
Hongze Cheng 已提交
580 581 582
  int64_t size;
};

H
Hongze Cheng 已提交
583
struct SSttFile {
H
Hongze Cheng 已提交
584 585
  volatile int32_t nRef;

H
Hongze Cheng 已提交
586
  int64_t commitID;
H
Hongze Cheng 已提交
587
  int64_t size;
H
Hongze Cheng 已提交
588
  int64_t offset;
H
Hongze Cheng 已提交
589 590 591
};

struct SSmaFile {
H
Hongze Cheng 已提交
592 593
  volatile int32_t nRef;

H
Hongze Cheng 已提交
594
  int64_t commitID;
H
Hongze Cheng 已提交
595 596 597 598
  int64_t size;
};

struct SDFileSet {
H
Hongze Cheng 已提交
599 600 601 602 603
  SDiskID    diskId;
  int32_t    fid;
  SHeadFile *pHeadF;
  SDataFile *pDataF;
  SSmaFile  *pSmaF;
H
Hongze Cheng 已提交
604
  uint8_t    nSttF;
H
Hongze Cheng 已提交
605
  SSttFile  *aSttF[TSDB_MAX_STT_TRIGGER];
H
Hongze Cheng 已提交
606 607
};

608
struct STSDBRowIter {
H
Hongze Cheng 已提交
609 610 611 612 613 614 615 616
  TSDBROW *pRow;
  union {
    SRowIter *pIter;
    struct {
      int32_t iColData;
      SColVal cv;
    };
  };
H
Hongze Cheng 已提交
617
};
H
Hongze Cheng 已提交
618 619 620 621 622
struct SRowMerger {
  STSchema *pTSchema;
  int64_t   version;
  SArray   *pArray;  // SArray<SColVal>
};
H
Hongze Cheng 已提交
623

H
more  
Hongze Cheng 已提交
624
typedef struct {
H
Hongze Cheng 已提交
625
  char     *path;
H
more  
Hongze Cheng 已提交
626
  int32_t   szPage;
H
Hongze Cheng 已提交
627 628 629
  int32_t   flag;
  TdFilePtr pFD;
  int64_t   pgno;
H
more  
Hongze Cheng 已提交
630
  uint8_t  *pBuf;
H
Hongze Cheng 已提交
631
  int64_t   szFile;
H
more  
Hongze Cheng 已提交
632
} STsdbFD;
H
Hongze Cheng 已提交
633

H
Hongze Cheng 已提交
634 635 636 637
struct SDelFWriter {
  STsdb   *pTsdb;
  SDelFile fDel;
  STsdbFD *pWriteH;
H
Hongze Cheng 已提交
638
  uint8_t *aBuf[1];
H
Hongze Cheng 已提交
639 640
};

H
Hongze Cheng 已提交
641
struct STsdbReadSnap {
H
Hongze Cheng 已提交
642 643 644 645 646
  SMemTable  *pMem;
  SQueryNode *pNode;
  SMemTable  *pIMem;
  SQueryNode *pINode;
  STsdbFS     fs;
H
Hongze Cheng 已提交
647 648
};

H
Hongze Cheng 已提交
649 650 651 652
struct SDataFWriter {
  STsdb    *pTsdb;
  SDFileSet wSet;

H
Hongze Cheng 已提交
653 654 655
  STsdbFD *pHeadFD;
  STsdbFD *pDataFD;
  STsdbFD *pSmaFD;
H
Hongze Cheng 已提交
656
  STsdbFD *pSttFD;
H
Hongze Cheng 已提交
657 658 659 660

  SHeadFile fHead;
  SDataFile fData;
  SSmaFile  fSma;
H
Hongze Cheng 已提交
661
  SSttFile  fStt[TSDB_MAX_STT_TRIGGER];
H
Hongze Cheng 已提交
662

H
Hongze Cheng 已提交
663
  uint8_t *aBuf[4];
H
Hongze Cheng 已提交
664 665
};

H
Hongze Cheng 已提交
666 667 668
struct SDataFReader {
  STsdb     *pTsdb;
  SDFileSet *pSet;
H
Hongze Cheng 已提交
669 670 671
  STsdbFD   *pHeadFD;
  STsdbFD   *pDataFD;
  STsdbFD   *pSmaFD;
H
Hongze Cheng 已提交
672
  STsdbFD   *aSttFD[TSDB_MAX_STT_TRIGGER];
H
Hongze Cheng 已提交
673
  uint8_t   *aBuf[3];
H
Hongze Cheng 已提交
674 675
};

H
Hongze Cheng 已提交
676
// NOTE: do NOT change the order of the fields
H
Hongze Cheng 已提交
677 678 679 680 681 682
typedef struct {
  int64_t suid;
  int64_t uid;
  TSDBROW row;
} SRowInfo;

683
typedef struct SSttBlockLoadInfo {
H
Hongze Cheng 已提交
684 685 686 687 688 689
  SBlockData blockData[2];
  SArray    *aSttBlk;
  int32_t    blockIndex[2];  // to denote the loaded block in the corresponding position.
  int32_t    currentLoadBlockIndex;
  int32_t    loadBlocks;
  double     elapsedTime;
690 691 692
  STSchema  *pSchema;
  int16_t   *colIds;
  int32_t    numOfCols;
693 694
  bool       checkRemainingRow;
  bool       isLast;
695
  bool       sttBlockLoaded;
696
  int32_t    numOfStt;
H
Haojun Liao 已提交
697 698 699 700 701 702 703

  // keep the last access position, this position may be used to reduce the binary times for
  // starting last block data for a new table
  struct {
    int32_t blockIndex;
    int32_t rowIndex;
  } prevEndPos;
704 705
} SSttBlockLoadInfo;

706
typedef struct SMergeTree {
H
Hongze Cheng 已提交
707 708 709 710 711 712 713
  int8_t             backward;
  SRBTree            rbt;
  SArray            *pIterList;
  SLDataIter        *pIter;
  bool               destroyLoadInfo;
  SSttBlockLoadInfo *pLoadInfo;
  const char        *idStr;
D
dapan1121 已提交
714
  bool               ignoreEarlierTs;
715 716
} SMergeTree;

H
Hongze Cheng 已提交
717 718 719 720 721 722
typedef struct {
  int64_t   suid;
  int64_t   uid;
  STSchema *pTSchema;
} SSkmInfo;

H
Hongze Cheng 已提交
723 724 725 726 727
struct SDiskCol {
  SBlockCol      bCol;
  const uint8_t *pBit;
  const uint8_t *pOff;
  const uint8_t *pVal;
H
Hongze Cheng 已提交
728
  SColumnDataAgg agg;
H
Hongze Cheng 已提交
729 730 731 732 733 734 735 736 737 738
};

struct SDiskData {
  SDiskDataHdr   hdr;
  const uint8_t *pUid;
  const uint8_t *pVer;
  const uint8_t *pKey;
  SArray        *aDiskCol;  // SArray<SDiskCol>
};

H
Hongze Cheng 已提交
739 740 741 742 743 744 745 746 747 748 749 750
struct SDiskDataBuilder {
  int64_t      suid;
  int64_t      uid;
  int32_t      nRow;
  uint8_t      cmprAlg;
  uint8_t      calcSma;
  SCompressor *pUidC;
  SCompressor *pVerC;
  SCompressor *pKeyC;
  int32_t      nBuilder;
  SArray      *aBuilder;  // SArray<SDiskColBuilder>
  uint8_t     *aBuf[2];
H
Hongze Cheng 已提交
751
  SDiskData    dd;
H
Hongze Cheng 已提交
752
  SBlkInfo     bi;
H
Hongze Cheng 已提交
753 754
};

H
Haojun Liao 已提交
755
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
756
                       STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
D
dapan1121 已提交
757
                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange);
H
Hongze Cheng 已提交
758
void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
H
Hongze Cheng 已提交
759
bool    tMergeTreeNext(SMergeTree *pMTree);
D
dapan1121 已提交
760
bool    tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
H
refact  
Hongze Cheng 已提交
761 762
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
void    tMergeTreeClose(SMergeTree *pMTree);
763

764
SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt);
H
Hongze Cheng 已提交
765 766 767
void               resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void               getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el);
void              *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
768

769 770
// tsdbCache ==============================================================================================
typedef struct SCacheRowsReader {
771 772 773
  STsdb             *pTsdb;
  SVersionRange      verRange;
  TdThreadMutex      readerMutex;
H
Haojun Liao 已提交
774 775
  SVnode            *pVnode;
  STSchema          *pSchema;
776
  STSchema          *pCurrSchema;
H
Haojun Liao 已提交
777 778 779 780 781 782
  uint64_t           uid;
  uint64_t           suid;
  char             **transferBuf;  // todo remove it soon
  int32_t            numOfCols;
  int32_t            type;
  int32_t            tableIndex;  // currently returned result tables
783 784
  STableKeyInfo     *pTableList;  // table id list
  int32_t            numOfTables;
785 786 787
  SSttBlockLoadInfo *pLoadInfo;
  STsdbReadSnap     *pReadSnap;
  SDataFReader      *pDataFReader;
788
  SDataFReader      *pDataFReaderLast;
H
Haojun Liao 已提交
789
  const char        *idstr;
790
  int64_t            lastTs;
791 792 793 794 795 796 797 798 799
} SCacheRowsReader;

typedef struct {
  TSKEY   ts;
  SColVal colVal;
} SLastCol;

int32_t tsdbOpenCache(STsdb *pTsdb);
void    tsdbCloseCache(STsdb *pTsdb);
H
Hongze Cheng 已提交
800 801
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb);
int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup);
802 803 804 805
int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h);
int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h);
int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);

806 807 808
int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHandle **handle);
int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h);

809 810 811 812 813 814 815
int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);

void   tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode);

H
Hongze Cheng 已提交
816
// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
817

818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837
// ========== inline functions ==========
static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
  TSDBKEY *pKey1 = (TSDBKEY *)p1;
  TSDBKEY *pKey2 = (TSDBKEY *)p2;

  if (pKey1->ts < pKey2->ts) {
    return -1;
  } else if (pKey1->ts > pKey2->ts) {
    return 1;
  }

  if (pKey1->version < pKey2->version) {
    return -1;
  } else if (pKey1->version > pKey2->version) {
    return 1;
  }

  return 0;
}

H
Hongze Cheng 已提交
838 839
// #define SL_NODE_FORWARD(n, l)  ((n)->forwards[l])
// #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
H
Hongze Cheng 已提交
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858

static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
  if (pIter == NULL) return NULL;

  if (pIter->pRow) {
    return pIter->pRow;
  }

  if (pIter->backward) {
    if (pIter->pNode == pIter->pTbData->sl.pHead) {
      return NULL;
    }
  } else {
    if (pIter->pNode == pIter->pTbData->sl.pTail) {
      return NULL;
    }
  }

  pIter->pRow = &pIter->row;
H
Hongze Cheng 已提交
859 860 861 862 863 864 865
  if (pIter->pNode->flag == TSDBROW_ROW_FMT) {
    pIter->row = tsdbRowFromTSRow(pIter->pNode->version, pIter->pNode->pData);
  } else if (pIter->pNode->flag == TSDBROW_COL_FMT) {
    pIter->row = tsdbRowFromBlockData(pIter->pNode->pData, pIter->pNode->iRow);
  } else {
    ASSERT(0);
  }
H
Hongze Cheng 已提交
866 867 868 869

  return pIter->pRow;
}

H
Hongze Cheng 已提交
870 871
int32_t tRowInfoCmprFn(const void *p1, const void *p2);

H
Hongze Cheng 已提交
872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925
typedef struct {
  int64_t  suid;
  int64_t  uid;
  SDelData delData;
} SDelInfo;

struct STsdbDataIter2 {
  STsdbDataIter2 *next;
  SRBTreeNode     rbtn;

  int32_t  type;
  SRowInfo rowInfo;
  SDelInfo delInfo;
  union {
    // TSDB_MEM_TABLE_DATA_ITER
    struct {
      SMemTable *pMemTable;
    } mIter;

    // TSDB_DATA_FILE_DATA_ITER
    struct {
      SDataFReader *pReader;
      SArray       *aBlockIdx;  // SArray<SBlockIdx>
      SMapData      mDataBlk;
      SBlockData    bData;
      int32_t       iBlockIdx;
      int32_t       iDataBlk;
      int32_t       iRow;
    } dIter;

    // TSDB_STT_FILE_DATA_ITER
    struct {
      SDataFReader *pReader;
      int32_t       iStt;
      SArray       *aSttBlk;
      SBlockData    bData;
      int32_t       iSttBlk;
      int32_t       iRow;
    } sIter;
    // TSDB_TOMB_FILE_DATA_ITER
    struct {
      SDelFReader *pReader;
      SArray      *aDelIdx;
      SArray      *aDelData;
      int32_t      iDelIdx;
      int32_t      iDelData;
    } tIter;
  };
};

struct STsdbFilterInfo {
  int32_t flag;
  int64_t sver;
  int64_t ever;
H
Hongze Cheng 已提交
926
  TABLEID tbid;
H
Hongze Cheng 已提交
927 928
};

H
Hongze Cheng 已提交
929 930 931 932
#ifdef __cplusplus
}
#endif

933
#endif /*_TD_VNODE_TSDB_H_*/