tsdb.h 24.2 KB
Newer Older
L
Liu Jicong 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
Hongze Cheng 已提交
15 16 17 18

#ifndef _TD_VNODE_TSDB_H_
#define _TD_VNODE_TSDB_H_

H
Hongze Cheng 已提交
19 20
#include "vnodeInt.h"

H
Hongze Cheng 已提交
21 22 23 24
#ifdef __cplusplus
extern "C" {
#endif

H
Hongze Cheng 已提交
25 26
// tsdbDebug ================
// clang-format off
H
Hongze Cheng 已提交
27 28 29 30 31 32
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSD FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSD ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSD WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSD ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSD ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
H
Hongze Cheng 已提交
33
// clang-format on
H
Hongze Cheng 已提交
34

H
Hongze Cheng 已提交
35 36 37 38 39 40 41 42 43 44
typedef struct TSDBROW       TSDBROW;
typedef struct TABLEID       TABLEID;
typedef struct TSDBKEY       TSDBKEY;
typedef struct SDelData      SDelData;
typedef struct SDelIdx       SDelIdx;
typedef struct STbData       STbData;
typedef struct SMemTable     SMemTable;
typedef struct STbDataIter   STbDataIter;
typedef struct SMapData      SMapData;
typedef struct SBlockIdx     SBlockIdx;
H
Hongze Cheng 已提交
45
typedef struct SDataBlk      SDataBlk;
H
Hongze Cheng 已提交
46
typedef struct SSttBlk       SSttBlk;
H
Hongze Cheng 已提交
47
typedef struct SDiskDataHdr  SDiskDataHdr;
H
Hongze Cheng 已提交
48 49 50 51
typedef struct SBlockData    SBlockData;
typedef struct SDelFile      SDelFile;
typedef struct SHeadFile     SHeadFile;
typedef struct SDataFile     SDataFile;
H
Hongze Cheng 已提交
52
typedef struct SSttFile      SSttFile;
H
Hongze Cheng 已提交
53 54 55 56 57 58 59 60 61
typedef struct SSmaFile      SSmaFile;
typedef struct SDFileSet     SDFileSet;
typedef struct SDataFWriter  SDataFWriter;
typedef struct SDataFReader  SDataFReader;
typedef struct SDelFWriter   SDelFWriter;
typedef struct SDelFReader   SDelFReader;
typedef struct SRowIter      SRowIter;
typedef struct STsdbFS       STsdbFS;
typedef struct SRowMerger    SRowMerger;
H
Hongze Cheng 已提交
62
typedef struct STsdbReadSnap STsdbReadSnap;
H
Hongze Cheng 已提交
63 64
typedef struct SBlockInfo    SBlockInfo;
typedef struct SSmaInfo      SSmaInfo;
H
Hongze Cheng 已提交
65
typedef struct SBlockCol     SBlockCol;
66
typedef struct SVersionRange SVersionRange;
H
Hongze Cheng 已提交
67
typedef struct SLDataIter    SLDataIter;
H
Hongze Cheng 已提交
68

H
Hongze Cheng 已提交
69 70 71
#define TSDB_FILE_DLMT     ((uint32_t)0xF00AFA0F)
#define TSDB_MAX_SUBBLOCKS 8
#define TSDB_FHDR_SIZE     512
H
refact  
Hongze Cheng 已提交
72

H
Hongze Cheng 已提交
73 74 75
#define VERSION_MIN 0
#define VERSION_MAX INT64_MAX

H
Hongze Cheng 已提交
76 77 78
#define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN})
#define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX})

H
Hongze Cheng 已提交
79 80
#define TABLE_SAME_SCHEMA(SUID1, UID1, SUID2, UID2) ((SUID1) ? (SUID1) == (SUID2) : (UID1) == (UID2))

H
Hongze Cheng 已提交
81 82 83 84 85 86
#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM))
#define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \
  ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE))
#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE))
#define PAGE_OFFSET(PGNO, PAGE)            (((PGNO)-1) * (PAGE))
#define OFFSET_PGNO(OFFSET, PAGE)          ((OFFSET) / (PAGE) + 1)
H
Hongze Cheng 已提交
87 88 89 90 91 92 93 94 95 96 97 98

static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
  int64_t fOffSet = LOGIC_TO_FILE_OFFSET(lSize, szPage);
  int64_t pgno = OFFSET_PGNO(fOffSet, szPage);
  int32_t szPageCont = PAGE_CONTENT_SIZE(szPage);

  if (fOffSet % szPageCont == 0) {
    pgno--;
  }

  return pgno * szPage;
}
H
Hongze Cheng 已提交
99

H
Hongze Cheng 已提交
100
// tsdbUtil.c ==============================================================================================
H
Hongze Cheng 已提交
101
// TSDBROW
H
Hongze Cheng 已提交
102 103
#define TSDBROW_TS(ROW)                       (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow])
#define TSDBROW_VERSION(ROW)                  (((ROW)->type == 0) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow])
H
Hongze Cheng 已提交
104
#define TSDBROW_SVERSION(ROW)                 TD_ROW_SVER((ROW)->pTSRow)
H
Hongze Cheng 已提交
105
#define TSDBROW_KEY(ROW)                      ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)})
H
Hongze Cheng 已提交
106 107
#define tsdbRowFromTSRow(VERSION, TSROW)      ((TSDBROW){.type = 0, .version = (VERSION), .pTSRow = (TSROW)})
#define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
H
Hongze Cheng 已提交
108 109 110
void    tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
H
Hongze Cheng 已提交
111
int32_t tsdbRowCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
112
// SRowIter
H
Hongze Cheng 已提交
113
void     tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);
H
Hongze Cheng 已提交
114
SColVal *tRowIterNext(SRowIter *pIter);
H
Hongze Cheng 已提交
115
// SRowMerger
116 117 118
int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema);
int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);

H
Hongze Cheng 已提交
119 120 121 122
int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
void    tRowMergerClear(SRowMerger *pMerger);
int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow);
int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow);
H
Hongze Cheng 已提交
123
// TABLEID
H
Hongze Cheng 已提交
124
int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
125
// TSDBKEY
H
Hongze Cheng 已提交
126 127
#define MIN_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) < 0) ? (KEY1) : (KEY2))
#define MAX_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) > 0) ? (KEY1) : (KEY2))
H
Hongze Cheng 已提交
128 129 130
// SBlockCol
int32_t tPutBlockCol(uint8_t *p, void *ph);
int32_t tGetBlockCol(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
131
int32_t tBlockColCmprFn(const void *p1, const void *p2);
H
Hongze Cheng 已提交
132
// SDataBlk
H
Hongze Cheng 已提交
133
void    tDataBlkReset(SDataBlk *pBlock);
H
Hongze Cheng 已提交
134 135
int32_t tPutDataBlk(uint8_t *p, void *ph);
int32_t tGetDataBlk(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
136 137
int32_t tDataBlkCmprFn(const void *p1, const void *p2);
bool    tDataBlkHasSma(SDataBlk *pDataBlk);
H
Hongze Cheng 已提交
138 139 140
// SSttBlk
int32_t tPutSttBlk(uint8_t *p, void *ph);
int32_t tGetSttBlk(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
141 142 143
// SBlockIdx
int32_t tPutBlockIdx(uint8_t *p, void *ph);
int32_t tGetBlockIdx(uint8_t *p, void *ph);
144
int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
145
int32_t tCmprBlockL(void const *lhs, void const *rhs);
H
Hongze Cheng 已提交
146
// SBlockData
H
Hongze Cheng 已提交
147 148
#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0)
#define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
H
Hongze Cheng 已提交
149 150
#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA))
#define tBlockDataLastKey(PBLOCKDATA)  TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA))
H
Hongze Cheng 已提交
151

H
Hongze Cheng 已提交
152 153 154
int32_t   tBlockDataCreate(SBlockData *pBlockData);
void      tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear);
int32_t   tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema);
H
Hongze Cheng 已提交
155
int32_t   tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom);
H
Hongze Cheng 已提交
156 157 158
void      tBlockDataReset(SBlockData *pBlockData);
int32_t   tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
void      tBlockDataClear(SBlockData *pBlockData);
H
Hongze Cheng 已提交
159 160
SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx);
void      tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
H
Hongze Cheng 已提交
161 162
int32_t   tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
H
Hongze Cheng 已提交
163
int32_t   tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
H
Hongze Cheng 已提交
164 165
int32_t   tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
                         int32_t aBufN[]);
H
Hongze Cheng 已提交
166
int32_t   tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
H
Hongze Cheng 已提交
167 168 169
// SDiskDataHdr
int32_t tPutDiskDataHdr(uint8_t *p, void *ph);
int32_t tGetDiskDataHdr(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
170
// SDelIdx
H
Hongze Cheng 已提交
171 172
int32_t tPutDelIdx(uint8_t *p, void *ph);
int32_t tGetDelIdx(uint8_t *p, void *ph);
173
int32_t tCmprDelIdx(void const *lhs, void const *rhs);
H
Hongze Cheng 已提交
174
// SDelData
H
Hongze Cheng 已提交
175 176
int32_t tPutDelData(uint8_t *p, void *ph);
int32_t tGetDelData(uint8_t *p, void *ph);
H
Hongze Cheng 已提交
177
// SMapData
H
Hongze Cheng 已提交
178
#define tMapDataInit() ((SMapData){0})
H
Hongze Cheng 已提交
179 180 181
void    tMapDataReset(SMapData *pMapData);
void    tMapDataClear(SMapData *pMapData);
int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *));
H
Hongze Cheng 已提交
182
int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo);
H
Hongze Cheng 已提交
183
void    tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *));
H
Hongze Cheng 已提交
184 185
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
                       int32_t (*tItemCmprFn)(const void *, const void *), void *pItem);
H
Hongze Cheng 已提交
186 187 188 189 190
int32_t tPutMapData(uint8_t *p, SMapData *pMapData);
int32_t tGetMapData(uint8_t *p, SMapData *pMapData);
// other
int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision);
void    tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey);
H
Hongze Cheng 已提交
191
int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now);
H
Hongze Cheng 已提交
192
int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline);
H
Hongze Cheng 已提交
193
void    tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg);
H
Hongze Cheng 已提交
194 195
int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg);
int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg);
H
Hongze Cheng 已提交
196 197
int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut,
                     int32_t *szOut, uint8_t **ppBuf);
H
Hongze Cheng 已提交
198 199
int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut,
                       uint8_t **ppBuf);
H
Hongze Cheng 已提交
200
int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int32_t nOut,
H
Hongze Cheng 已提交
201
                        uint8_t **ppBuf);
H
Hongze Cheng 已提交
202 203
int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData,
                          uint8_t **ppBuf);
H
Hongze Cheng 已提交
204 205
// tsdbMemTable ==============================================================================================
// SMemTable
H
Hongze Cheng 已提交
206 207 208 209 210 211
int32_t  tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
void     tsdbMemTableDestroy(SMemTable *pMemTable);
STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid);
void     tsdbRefMemTable(SMemTable *pMemTable);
void     tsdbUnrefMemTable(SMemTable *pMemTable);
SArray  *tsdbMemTableGetTbDataArray(SMemTable *pMemTable);
H
Hongze Cheng 已提交
212 213 214 215 216 217
// STbDataIter
int32_t  tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
void    *tsdbTbDataIterDestroy(STbDataIter *pIter);
void     tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter);
bool     tsdbTbDataIterNext(STbDataIter *pIter);
H
Hongze Cheng 已提交
218 219
// STbData
int32_t tsdbGetNRowsInTbData(STbData *pTbData);
H
Hongze Cheng 已提交
220
// tsdbFile.c ==============================================================================================
H
Hongze Cheng 已提交
221
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
H
Hongze Cheng 已提交
222

H
Hongze Cheng 已提交
223
bool    tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
H
Hongze Cheng 已提交
224
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
H
Hongze Cheng 已提交
225 226
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
H
Hongze Cheng 已提交
227
int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile);
H
Hongze Cheng 已提交
228
int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile);
H
Hongze Cheng 已提交
229 230 231 232
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile);
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet);
int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet);
H
Hongze Cheng 已提交
233 234 235

void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]);
void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]);
H
Hongze Cheng 已提交
236
void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]);
H
Hongze Cheng 已提交
237
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]);
H
Hongze Cheng 已提交
238
// SDelFile
H
Hongze Cheng 已提交
239
void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]);
H
Hongze Cheng 已提交
240
// tsdbFS.c ==============================================================================================
H
Hongze Cheng 已提交
241 242 243 244 245 246 247 248 249 250
int32_t tsdbFSOpen(STsdb *pTsdb);
int32_t tsdbFSClose(STsdb *pTsdb);
int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS);
void    tsdbFSDestroy(STsdbFS *pFS);
int32_t tDFileSetCmprFn(const void *p1, const void *p2);
int32_t tsdbFSCommit1(STsdb *pTsdb, STsdbFS *pFS);
int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFS);
int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS);
void    tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS);

H
Hongze Cheng 已提交
251
int32_t tsdbFSRollback(STsdbFS *pFS);
H
Hongze Cheng 已提交
252

H
Hongze Cheng 已提交
253 254
int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet);
int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile);
H
Hongze Cheng 已提交
255 256 257
// tsdbReaderWriter.c ==============================================================================================
// SDataFWriter
int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet);
H
Hongze Cheng 已提交
258
int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
H
Hongze Cheng 已提交
259
int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
H
Hongze Cheng 已提交
260
int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx);
H
Hongze Cheng 已提交
261
int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx);
H
Hongze Cheng 已提交
262
int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk);
H
Hongze Cheng 已提交
263
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
H
Hongze Cheng 已提交
264
                           int8_t cmprAlg, int8_t toLast);
H
Hongze Cheng 已提交
265 266

int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
H
Hongze Cheng 已提交
267 268
// SDataFReader
int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet);
H
Hongze Cheng 已提交
269
int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
H
Hongze Cheng 已提交
270
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx);
H
Hongze Cheng 已提交
271
int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk);
H
Hongze Cheng 已提交
272
int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk);
H
Hongze Cheng 已提交
273 274
int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
H
Hongze Cheng 已提交
275
int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
H
Hongze Cheng 已提交
276 277
// SDelFWriter
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
H
Hongze Cheng 已提交
278
int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
H
Hongze Cheng 已提交
279 280
int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx);
int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx);
H
Hongze Cheng 已提交
281
int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter);
H
Hongze Cheng 已提交
282
// SDelFReader
H
Hongze Cheng 已提交
283
int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb);
H
Hongze Cheng 已提交
284
int32_t tsdbDelFReaderClose(SDelFReader **ppReader);
H
Hongze Cheng 已提交
285 286
int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData);
int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx);
H
Hongze Cheng 已提交
287
// tsdbRead.c ==============================================================================================
H
Haojun Liao 已提交
288 289
int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char* id);
void    tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char* id);
H
Hongze Cheng 已提交
290 291
// tsdbMerge.c ==============================================================================================
int32_t tsdbMerge(STsdb *pTsdb);
H
Hongze Cheng 已提交
292

293 294 295 296
#define TSDB_CACHE_NO(c)       ((c).cacheLast == 0)
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
#define TSDB_CACHE_LAST(c)     (((c).cacheLast & 2) > 0)

297 298
// tsdbCache
int32_t tsdbOpenCache(STsdb *pTsdb);
299
void    tsdbCloseCache(STsdb *pTsdb);
300
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
301
int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup);
302 303
int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHandle **h);
M
Minglei Jin 已提交
304
int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);
305

306 307
int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
308
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
309

310 311 312
void   tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode);

313 314
int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);

H
Hongze Cheng 已提交
315
// structs =======================
H
Hongze Cheng 已提交
316 317 318 319 320
struct STsdbFS {
  SDelFile *pDelFile;
  SArray   *aDFileSet;  // SArray<SDFileSet>
};

H
Hongze Cheng 已提交
321
struct STsdb {
H
Hongze Cheng 已提交
322 323 324
  char          *path;
  SVnode        *pVnode;
  STsdbKeepCfg   keepCfg;
H
more  
Hongze Cheng 已提交
325
  TdThreadRwlock rwLock;
H
Hongze Cheng 已提交
326 327
  SMemTable     *mem;
  SMemTable     *imem;
H
Hongze Cheng 已提交
328
  STsdbFS        fs;
H
Hongze Cheng 已提交
329
  SLRUCache     *lruCache;
330
  TdThreadMutex  lruMutex;
H
Hongze Cheng 已提交
331 332
};

H
refact  
Hongze Cheng 已提交
333 334 335 336 337
struct TSDBKEY {
  int64_t version;
  TSKEY   ts;
};

338 339 340 341 342
struct SVersionRange {
  uint64_t minVer;
  uint64_t maxVer;
};

H
Hongze Cheng 已提交
343 344 345 346 347 348 349
typedef struct SMemSkipListNode SMemSkipListNode;
struct SMemSkipListNode {
  int8_t            level;
  SMemSkipListNode *forwards[0];
};
typedef struct SMemSkipList {
  int64_t           size;
H
Hongze Cheng 已提交
350
  uint32_t          seed;
H
Hongze Cheng 已提交
351 352 353 354 355 356
  int8_t            maxLevel;
  int8_t            level;
  SMemSkipListNode *pHead;
  SMemSkipListNode *pTail;
} SMemSkipList;

H
Hongze Cheng 已提交
357
struct STbData {
H
Hongze Cheng 已提交
358 359
  tb_uid_t     suid;
  tb_uid_t     uid;
H
Hongze Cheng 已提交
360 361
  TSKEY        minKey;
  TSKEY        maxKey;
H
Hongze Cheng 已提交
362 363
  SDelData    *pHead;
  SDelData    *pTail;
H
Hongze Cheng 已提交
364
  SMemSkipList sl;
H
Hongze Cheng 已提交
365
  STbData     *next;
H
Hongze Cheng 已提交
366
};
H
Hongze Cheng 已提交
367

H
refact  
Hongze Cheng 已提交
368
struct SMemTable {
H
Hongze Cheng 已提交
369 370 371 372 373 374 375 376
  SRWLatch         latch;
  STsdb           *pTsdb;
  SVBufPool       *pPool;
  volatile int32_t nRef;
  TSKEY            minKey;
  TSKEY            maxKey;
  int64_t          nRow;
  int64_t          nDel;
H
Hongze Cheng 已提交
377 378 379 380 381
  struct {
    int32_t   nTbData;
    int32_t   nBucket;
    STbData **aBucket;
  };
H
Hongze Cheng 已提交
382
};
H
Hongze Cheng 已提交
383

H
Hongze Cheng 已提交
384
struct TSDBROW {
H
Hongze Cheng 已提交
385
  int8_t type;  // 0 for row from tsRow, 1 for row from block data
H
Hongze Cheng 已提交
386 387 388 389 390 391
  union {
    struct {
      int64_t version;
      STSRow *pTSRow;
    };
    struct {
H
Hongze Cheng 已提交
392 393
      SBlockData *pBlockData;
      int32_t     iRow;
H
Hongze Cheng 已提交
394 395
    };
  };
H
Hongze Cheng 已提交
396 397
};

H
Hongze Cheng 已提交
398
struct SBlockIdx {
H
Hongze Cheng 已提交
399 400
  int64_t suid;
  int64_t uid;
H
Hongze Cheng 已提交
401 402 403 404
  int64_t offset;
  int64_t size;
};

H
Hongze Cheng 已提交
405 406
struct SMapData {
  int32_t  nItem;
H
Hongze Cheng 已提交
407
  int32_t  nData;
408
  int32_t *aOffset;
H
Hongze Cheng 已提交
409 410 411
  uint8_t *pData;
};

H
Hongze Cheng 已提交
412
struct SBlockCol {
H
Hongze Cheng 已提交
413 414 415 416 417 418 419 420 421
  int16_t cid;
  int8_t  type;
  int8_t  smaOn;
  int8_t  flag;      // HAS_NONE|HAS_NULL|HAS_VALUE
  int32_t szOrigin;  // original column value size (only save for variant data type)
  int32_t szBitmap;  // bitmap size, 0 only for flag == HAS_VAL
  int32_t szOffset;  // offset size, 0 only for non-variant-length type
  int32_t szValue;   // value size, 0 when flag == (HAS_NULL | HAS_NONE)
  int32_t offset;
H
Hongze Cheng 已提交
422
};
H
Hongze Cheng 已提交
423

H
Hongze Cheng 已提交
424
struct SBlockInfo {
H
Hongze Cheng 已提交
425 426 427
  int64_t offset;  // block data offset
  int32_t szBlock;
  int32_t szKey;
H
Hongze Cheng 已提交
428
};
H
Hongze Cheng 已提交
429

H
Hongze Cheng 已提交
430
struct SSmaInfo {
H
Hongze Cheng 已提交
431 432
  int64_t offset;
  int32_t size;
H
Hongze Cheng 已提交
433
};
H
Hongze Cheng 已提交
434

H
Hongze Cheng 已提交
435
struct SDataBlk {
H
Hongze Cheng 已提交
436 437 438 439 440 441 442 443 444
  TSDBKEY    minKey;
  TSDBKEY    maxKey;
  int64_t    minVer;
  int64_t    maxVer;
  int32_t    nRow;
  int8_t     hasDup;
  int8_t     nSubBlock;
  SBlockInfo aSubBlock[TSDB_MAX_SUBBLOCKS];
  SSmaInfo   smaInfo;
H
refact  
Hongze Cheng 已提交
445
};
H
Hongze Cheng 已提交
446

H
Hongze Cheng 已提交
447
struct SSttBlk {
H
Hongze Cheng 已提交
448 449 450
  int64_t    suid;
  int64_t    minUid;
  int64_t    maxUid;
H
Hongze Cheng 已提交
451 452
  TSKEY      minKey;
  TSKEY      maxKey;
H
Hongze Cheng 已提交
453 454 455 456
  int64_t    minVer;
  int64_t    maxVer;
  int32_t    nRow;
  SBlockInfo bInfo;
H
Hongze Cheng 已提交
457 458
};

H
Hongze Cheng 已提交
459 460 461 462
// (SBlockData){.suid = 0, .uid = 0}: block data not initialized
// (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file
// (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file
// (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file
H
Hongze Cheng 已提交
463
struct SBlockData {
H
Hongze Cheng 已提交
464 465 466 467 468 469
  int64_t  suid;      // 0 means normal table block data, otherwise child table block data
  int64_t  uid;       // 0 means block data in .last file, otherwise in .data file
  int32_t  nRow;      // number of rows
  int64_t *aUid;      // uids of each row, only exist in block data in .last file (uid == 0)
  int64_t *aVersion;  // versions of each row
  TSKEY   *aTSKEY;    // timestamp of each row
H
Hongze Cheng 已提交
470 471
  SArray  *aIdx;      // SArray<int32_t>
  SArray  *aColData;  // SArray<SColData>
H
Hongze Cheng 已提交
472
};
H
Hongze Cheng 已提交
473

H
Hongze Cheng 已提交
474 475 476 477 478
struct TABLEID {
  tb_uid_t suid;
  tb_uid_t uid;
};

H
Hongze Cheng 已提交
479 480 481 482
struct STbDataIter {
  STbData          *pTbData;
  int8_t            backward;
  SMemSkipListNode *pNode;
H
Hongze Cheng 已提交
483 484
  TSDBROW          *pRow;
  TSDBROW           row;
H
Hongze Cheng 已提交
485 486
};

H
Hongze Cheng 已提交
487
struct SDelData {
H
Hongze Cheng 已提交
488 489 490 491
  int64_t   version;
  TSKEY     sKey;
  TSKEY     eKey;
  SDelData *pNext;
H
Hongze Cheng 已提交
492
};
H
Hongze Cheng 已提交
493

H
Hongze Cheng 已提交
494
struct SDelIdx {
H
Hongze Cheng 已提交
495 496 497 498
  tb_uid_t suid;
  tb_uid_t uid;
  int64_t  offset;
  int64_t  size;
H
Hongze Cheng 已提交
499
};
H
Hongze Cheng 已提交
500

H
Hongze Cheng 已提交
501
struct SDiskDataHdr {
H
more  
Hongze Cheng 已提交
502
  uint32_t delimiter;
H
Hongze Cheng 已提交
503
  uint32_t fmtVer;
H
more  
Hongze Cheng 已提交
504 505
  int64_t  suid;
  int64_t  uid;
H
Hongze Cheng 已提交
506 507 508 509
  int32_t  szUid;
  int32_t  szVer;
  int32_t  szKey;
  int32_t  szBlkCol;
H
Hongze Cheng 已提交
510
  int32_t  nRow;
H
Hongze Cheng 已提交
511
  int8_t   cmprAlg;
H
Hongze Cheng 已提交
512 513
};

H
Hongze Cheng 已提交
514 515 516 517 518 519 520 521
struct SDelFile {
  volatile int32_t nRef;

  int64_t commitID;
  int64_t size;
  int64_t offset;
};

H
Hongze Cheng 已提交
522
struct SHeadFile {
H
Hongze Cheng 已提交
523 524
  volatile int32_t nRef;

H
Hongze Cheng 已提交
525
  int64_t commitID;
H
Hongze Cheng 已提交
526 527 528 529 530
  int64_t size;
  int64_t offset;
};

struct SDataFile {
H
Hongze Cheng 已提交
531 532
  volatile int32_t nRef;

H
Hongze Cheng 已提交
533
  int64_t commitID;
H
Hongze Cheng 已提交
534 535 536
  int64_t size;
};

H
Hongze Cheng 已提交
537
struct SSttFile {
H
Hongze Cheng 已提交
538 539
  volatile int32_t nRef;

H
Hongze Cheng 已提交
540
  int64_t commitID;
H
Hongze Cheng 已提交
541
  int64_t size;
H
Hongze Cheng 已提交
542
  int64_t offset;
H
Hongze Cheng 已提交
543 544 545
};

struct SSmaFile {
H
Hongze Cheng 已提交
546 547
  volatile int32_t nRef;

H
Hongze Cheng 已提交
548
  int64_t commitID;
H
Hongze Cheng 已提交
549 550 551 552
  int64_t size;
};

struct SDFileSet {
H
Hongze Cheng 已提交
553 554 555 556 557
  SDiskID    diskId;
  int32_t    fid;
  SHeadFile *pHeadF;
  SDataFile *pDataF;
  SSmaFile  *pSmaF;
H
Hongze Cheng 已提交
558
  uint8_t    nSttF;
H
Hongze Cheng 已提交
559
  SSttFile  *aSttF[TSDB_MAX_STT_TRIGGER];
H
Hongze Cheng 已提交
560 561 562 563
};

struct SRowIter {
  TSDBROW  *pRow;
H
Hongze Cheng 已提交
564
  STSchema *pTSchema;
H
Hongze Cheng 已提交
565 566
  SColVal   colVal;
  int32_t   i;
H
Hongze Cheng 已提交
567
};
H
Hongze Cheng 已提交
568 569 570 571 572
struct SRowMerger {
  STSchema *pTSchema;
  int64_t   version;
  SArray   *pArray;  // SArray<SColVal>
};
H
Hongze Cheng 已提交
573

H
more  
Hongze Cheng 已提交
574
typedef struct {
H
Hongze Cheng 已提交
575
  char     *path;
H
more  
Hongze Cheng 已提交
576
  int32_t   szPage;
H
Hongze Cheng 已提交
577 578 579
  int32_t   flag;
  TdFilePtr pFD;
  int64_t   pgno;
H
more  
Hongze Cheng 已提交
580
  uint8_t  *pBuf;
H
Hongze Cheng 已提交
581
  int64_t   szFile;
H
more  
Hongze Cheng 已提交
582
} STsdbFD;
H
Hongze Cheng 已提交
583

H
Hongze Cheng 已提交
584 585 586 587
struct SDelFWriter {
  STsdb   *pTsdb;
  SDelFile fDel;
  STsdbFD *pWriteH;
H
Hongze Cheng 已提交
588
  uint8_t *aBuf[1];
H
Hongze Cheng 已提交
589 590
};

H
Hongze Cheng 已提交
591 592 593 594 595 596
struct STsdbReadSnap {
  SMemTable *pMem;
  SMemTable *pIMem;
  STsdbFS    fs;
};

H
Hongze Cheng 已提交
597 598 599 600
struct SDataFWriter {
  STsdb    *pTsdb;
  SDFileSet wSet;

H
Hongze Cheng 已提交
601 602 603
  STsdbFD *pHeadFD;
  STsdbFD *pDataFD;
  STsdbFD *pSmaFD;
H
Hongze Cheng 已提交
604
  STsdbFD *pSttFD;
H
Hongze Cheng 已提交
605 606 607 608

  SHeadFile fHead;
  SDataFile fData;
  SSmaFile  fSma;
H
Hongze Cheng 已提交
609
  SSttFile  fStt[TSDB_MAX_STT_TRIGGER];
H
Hongze Cheng 已提交
610

H
Hongze Cheng 已提交
611
  uint8_t *aBuf[4];
H
Hongze Cheng 已提交
612 613
};

H
Hongze Cheng 已提交
614 615 616
struct SDataFReader {
  STsdb     *pTsdb;
  SDFileSet *pSet;
H
Hongze Cheng 已提交
617 618 619
  STsdbFD   *pHeadFD;
  STsdbFD   *pDataFD;
  STsdbFD   *pSmaFD;
H
Hongze Cheng 已提交
620
  STsdbFD   *aSttFD[TSDB_MAX_STT_TRIGGER];
H
Hongze Cheng 已提交
621
  uint8_t   *aBuf[3];
H
Hongze Cheng 已提交
622 623
};

H
Hongze Cheng 已提交
624 625 626 627 628 629
typedef struct {
  int64_t suid;
  int64_t uid;
  TSDBROW row;
} SRowInfo;

630
typedef struct SSttBlockLoadInfo {
H
Hongze Cheng 已提交
631 632 633 634 635 636
  SBlockData blockData[2];
  SArray    *aSttBlk;
  int32_t    blockIndex[2];  // to denote the loaded block in the corresponding position.
  int32_t    currentLoadBlockIndex;
  int32_t    loadBlocks;
  double     elapsedTime;
637 638
} SSttBlockLoadInfo;

639
typedef struct SMergeTree {
H
Hongze Cheng 已提交
640 641 642 643 644 645 646
  int8_t             backward;
  SRBTree            rbt;
  SArray            *pIterList;
  SLDataIter        *pIter;
  bool               destroyLoadInfo;
  SSttBlockLoadInfo *pLoadInfo;
  const char        *idStr;
647 648
} SMergeTree;

H
Hongze Cheng 已提交
649 650 651 652 653 654
typedef struct {
  int64_t   suid;
  int64_t   uid;
  STSchema *pTSchema;
} SSkmInfo;

H
Haojun Liao 已提交
655
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
H
Hongze Cheng 已提交
656
                       STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr);
H
Hongze Cheng 已提交
657
void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
H
Hongze Cheng 已提交
658
bool    tMergeTreeNext(SMergeTree *pMTree);
H
refact  
Hongze Cheng 已提交
659 660
TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
void    tMergeTreeClose(SMergeTree *pMTree);
661

H
Hongze Cheng 已提交
662 663 664 665
SSttBlockLoadInfo *tCreateLastBlockLoadInfo();
void               resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void               getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el);
void              *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
666

667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
// ========== inline functions ==========
static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
  TSDBKEY *pKey1 = (TSDBKEY *)p1;
  TSDBKEY *pKey2 = (TSDBKEY *)p2;

  if (pKey1->ts < pKey2->ts) {
    return -1;
  } else if (pKey1->ts > pKey2->ts) {
    return 1;
  }

  if (pKey1->version < pKey2->version) {
    return -1;
  } else if (pKey1->version > pKey2->version) {
    return 1;
  }

  return 0;
}

H
Hongze Cheng 已提交
687 688 689 690
#ifdef __cplusplus
}
#endif

691
#endif /*_TD_VNODE_TSDB_H_*/