tsdb.h 26.5 KB
Newer Older
L
Liu Jicong 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
Hongze Cheng 已提交
15 16 17 18

#ifndef _TD_VNODE_TSDB_H_
#define _TD_VNODE_TSDB_H_

H
Hongze Cheng 已提交
19 20
#include "vnodeInt.h"

H
Hongze Cheng 已提交
21 22 23 24
#ifdef __cplusplus
extern "C" {
#endif

H
Hongze Cheng 已提交
25 26 27 28 29 30 31 32 33 34
// tsdbDebug ================
// clang-format off
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on

H
more  
Hongze Cheng 已提交
35
typedef struct TSDBROW TSDBROW;
H
Hongze Cheng 已提交
36
typedef struct TSDBKEY TSDBKEY;
H
Hongze Cheng 已提交
37
typedef struct TABLEID TABLEID;
H
more  
Hongze Cheng 已提交
38
typedef struct SDelOp  SDelOp;
H
Hongze Cheng 已提交
39

H
Hongze Cheng 已提交
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
// tsdbMemTable ==============================================================================================
typedef struct STbData     STbData;
typedef struct SMemTable   SMemTable;
typedef struct STbDataIter STbDataIter;
typedef struct SMergeInfo  SMergeInfo;
typedef struct STable      STable;

// SMemTable
int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
void    tsdbMemTableDestroy(SMemTable *pMemTable);
void    tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);

// STbDataIter
int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
void   *tsdbTbDataIterDestroy(STbDataIter *pIter);
void    tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
bool    tsdbTbDataIterNext(STbDataIter *pIter);
bool    tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow);

H
refact  
Hongze Cheng 已提交
59
// tsdbFile.c ==============================================================================================
H
Hongze Cheng 已提交
60 61 62 63 64 65 66 67
typedef struct SDelFile       SDelFile;
typedef struct STsdbCacheFile STsdbCacheFile;
typedef struct STsdbIndexFile STsdbIndexFile;
typedef struct STsdbDataFile  STsdbDataFile;
typedef struct STsdbLastFile  STsdbLastFile;
typedef struct STsdbSmaFile   STsdbSmaFile;
typedef struct STsdbSmalFile  STsdbSmalFile;
typedef struct SDFileSet      SDFileSet;
H
Hongze Cheng 已提交
68 69

// tsdbFS.c ==============================================================================================
H
Hongze Cheng 已提交
70 71 72 73 74 75 76
typedef struct STsdbFS STsdbFS;

int32_t tsdbFSOpen(STsdb *pTsdb, STsdbFS **ppFS);
int32_t tsdbFSClose(STsdbFS *pFS);
int32_t tsdbFSStart(STsdbFS *pFS);
int32_t tsdbFSEnd(STsdbFS *pFS, int8_t rollback);

H
Hongze Cheng 已提交
77 78 79 80
// tsdbReaderWriter.c ==============================================================================================
typedef struct SDelData SDelData;
typedef struct SDelIdx  SDelIdx;

H
Hongze Cheng 已提交
81 82 83 84 85 86
// SDataFWriter
typedef struct SDataFWriter SDataFWriter;

// SDataFReader
typedef struct SDataFReader SDataFReader;

H
Hongze Cheng 已提交
87 88 89 90 91 92 93 94 95 96 97
// SDelFWriter
typedef struct SDelFWriter SDelFWriter;

int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile);
int32_t tsdbDelFWriterClose(SDelFWriter *pWriter);
int32_t tsdbWriteDelData(SDelFWriter *pWriter, SDelData *pDelData, uint8_t **ppBuf);
int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SDelIdx *pDelIdx, uint8_t **ppBuf);

// SDelFReader
typedef struct SDelFReader SDelFReader;

H
Hongze Cheng 已提交
98
int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile);
H
Hongze Cheng 已提交
99 100 101
int32_t tsdbDelFReaderClose(SDelFReader *pReader);
int32_t tsdbReadDelData(SDelFReader *pReader, SDelData *pDelData, uint8_t **ppBuf);
int32_t tsdbReadDelIdx(SDelFReader *pReader, SDelIdx *pDelIdx, uint8_t **ppBuf);
H
Hongze Cheng 已提交
102

H
Hongze Cheng 已提交
103 104 105 106 107 108
// SCacheFWriter
typedef struct SCacheFWriter SCacheFWriter;

// SCacheFReader
typedef struct SCacheFReader SCacheFReader;

H
Hongze Cheng 已提交
109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
// tsdbCommit.c ==============================================================================================

// old
// typedef int32_t          TSDB_FILE_T;
// typedef struct SDFInfo   SDFInfo;
// typedef struct SDFile    SDFile;
// typedef struct SDFileSet SDFileSet;

// // SDFile
// void    tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
// void    tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile);
// int     tsdbOpenDFile(SDFile *pDFile, int flags);
// void    tsdbCloseDFile(SDFile *pDFile);
// int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence);
// int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte);
// void    tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm);
// int     tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset);
// int     tsdbRemoveDFile(SDFile *pDFile);
// int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte);
// int     tsdbCopyDFile(SDFile *pSrc, SDFile *pDest);
// int     tsdbEncodeSDFile(void **buf, SDFile *pDFile);
// void   *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile);
// int     tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType);
// int     tsdbUpdateDFileHeader(SDFile *pDFile);
// int     tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo);
// int     tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version);

// // SDFileSet
// void  tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver);
// void  tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet);
// int   tsdbEncodeDFileSet(void **buf, SDFileSet *pSet);
// void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet);
// int   tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet);
// void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet);
// int   tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to);
// int   tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader);
// int   tsdbUpdateDFileSetHeader(SDFileSet *pSet);
// int   tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet);
// void  tsdbCloseDFileSet(SDFileSet *pSet);
// int   tsdbOpenDFileSet(SDFileSet *pSet, int flags);
// void  tsdbRemoveDFileSet(SDFileSet *pSet);
// int   tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest);
// void  tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey);

// typedef struct SFSIter     SFSIter;
// typedef struct STsdbFSMeta STsdbFSMeta;

// STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg);
// void    *tsdbFreeFS(STsdbFS *pfs);
// int      tsdbOpenFS(STsdb *pRepo);
// void     tsdbCloseFS(STsdb *pRepo);
// void     tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
// int      tsdbEndFSTxn(STsdb *pRepo);
// int      tsdbEndFSTxnWithError(STsdbFS *pfs);
// int      tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);

// void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
// void       tsdbFSIterSeek(SFSIter *pIter, int fid);
// SDFileSet *tsdbFSIterNext(SFSIter *pIter);
// int        tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
// int        tsdbRLockFS(STsdbFS *pFs);
// int        tsdbWLockFS(STsdbFS *pFs);
// int        tsdbUnLockFS(STsdbFS *pFs);
H
Hongze Cheng 已提交
172

H
refact  
Hongze Cheng 已提交
173 174 175 176 177 178 179 180 181 182 183
// tsdbReadImpl.c ==============================================================================================
typedef struct SBlockIdx    SBlockIdx;
typedef struct SBlockInfo   SBlockInfo;
typedef struct SBlock       SBlock;
typedef struct SBlockCol    SBlockCol;
typedef struct SBlockStatis SBlockStatis;
typedef struct SAggrBlkCol  SAggrBlkCol;
typedef struct SBlockData   SBlockData;
typedef struct SReadH       SReadH;

// SReadH
H
Hongze Cheng 已提交
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
// int  tsdbInitReadH(SReadH *pReadh, STsdb *pRepo);
// void tsdbDestroyReadH(SReadH *pReadh);
// int  tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
// void tsdbCloseAndUnsetFSet(SReadH *pReadh);
// int  tsdbLoadBlockIdx(SReadH *pReadh);
// int  tsdbSetReadTable(SReadH *pReadh, STable *pTable);
// int  tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
// int  tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
// int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int
// numOfColsIds,
//                           bool mergeBitmap);
// int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
// int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
// void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
// void  tsdbGetBlockStatis(SReadH *pReadh, SColumnDataAgg *pStatis, int numOfCols, SBlock *pBlock);
H
refact  
Hongze Cheng 已提交
199

H
Hongze Cheng 已提交
200 201
typedef struct SDFileSetReader SDFileSetReader;
typedef struct SDFileSetWriter SDFileSetWriter;
H
refact  
Hongze Cheng 已提交
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216

// SDFileSetWriter
int32_t tsdbDFileSetWriterOpen(SDFileSetWriter *pWriter, STsdb *pTsdb, SDFileSet *pSet);
int32_t tsdbDFileSetWriterClose(SDFileSetWriter *pWriter);
int32_t tsdbWriteBlockData(SDFileSetWriter *pWriter, SDataCols *pDataCols, SBlock *pBlock);
int32_t tsdbWriteSBlockInfo(SDFileSetWriter *pWriter, SBlockInfo *pBlockInfo, SBlockIdx *pBlockIdx);
int32_t tsdbWriteSBlockIdx(SDFileSetWriter *pWriter, SBlockIdx *pBlockIdx);

// SDFileSetReader
int32_t tsdbDFileSetReaderOpen(SDFileSetReader *pReader, STsdb *pTsdb, SDFileSet *pSet);
int32_t tsdbDFileSetReaderClose(SDFileSetReader *pReader);
int32_t tsdbLoadSBlockIdx(SDFileSetReader *pReader, SArray *pArray);
int32_t tsdbLoadSBlockInfo(SDFileSetReader *pReader, SBlockIdx *pBlockIdx, SBlockInfo *pBlockInfo);
int32_t tsdbLoadSBlockStatis(SDFileSetReader *pReader, SBlock *pBlock, SBlockStatis *pBlockStatis);

H
Hongze Cheng 已提交
217
// SDelFWriter
H
refact  
Hongze Cheng 已提交
218

H
Hongze Cheng 已提交
219
// SDelFReader
H
refact  
Hongze Cheng 已提交
220

H
Hongze Cheng 已提交
221
// tsdbUtil.c ==============================================================================================
H
Hongze Cheng 已提交
222 223 224 225 226 227 228 229 230
int32_t tsdbRealloc(uint8_t **ppBuf, int64_t size);
void    tsdbFree(uint8_t *pBuf);

// STMap
typedef struct STMap STMap;

int32_t tPutTMap(uint8_t *p, STMap *pMap);
int32_t tGetTMap(uint8_t *p, STMap *pMap);

H
Hongze Cheng 已提交
231 232 233
int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
int32_t tsdbKeyCmprFn(const void *p1, const void *p2);

H
Hongze Cheng 已提交
234 235 236 237 238 239 240 241
// structs
typedef struct {
  int   minFid;
  int   midFid;
  int   maxFid;
  TSKEY minKey;
} SRtn;

C
Cary Xu 已提交
242
#define TSDB_DATA_DIR_LEN 6  // adapt accordingly
H
Hongze Cheng 已提交
243
struct STsdb {
H
refact  
Hongze Cheng 已提交
244 245 246 247 248 249 250 251 252 253
  char         *path;
  SVnode       *pVnode;
  TdThreadMutex mutex;
  char          dir[TSDB_DATA_DIR_LEN];
  bool          repoLocked;
  STsdbKeepCfg  keepCfg;
  SMemTable    *mem;
  SMemTable    *imem;
  SRtn          rtn;
  STsdbFS      *fs;
H
Hongze Cheng 已提交
254 255
};

H
Hongze Cheng 已提交
256
struct STable {
H
Hongze Cheng 已提交
257
  uint64_t  suid;
H
Hongze Cheng 已提交
258
  uint64_t  uid;
259 260
  STSchema *pSchema;       // latest schema
  STSchema *pCacheSchema;  // cached cache
H
Hongze Cheng 已提交
261
};
H
Hongze Cheng 已提交
262

H
refact  
Hongze Cheng 已提交
263
// int tsdbPrepareCommit(STsdb *pTsdb);
H
Hongze Cheng 已提交
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
// typedef enum {
//   TSDB_FILE_HEAD = 0,  // .head
//   TSDB_FILE_DATA,      // .data
//   TSDB_FILE_LAST,      // .last
//   TSDB_FILE_SMAD,      // .smad(Block-wise SMA)
//   TSDB_FILE_SMAL,      // .smal(Block-wise SMA)
//   TSDB_FILE_MAX,       //
//   TSDB_FILE_META,      // meta
// } E_TSDB_FILE_T;

// struct SDFInfo {
//   uint32_t magic;
//   uint32_t fver;
//   uint32_t len;
//   uint32_t totalBlocks;
//   uint32_t totalSubBlocks;
//   uint32_t offset;
//   uint64_t size;
//   uint64_t tombSize;
// };

// struct SDFile {
//   SDFInfo   info;
//   STfsFile  f;
//   TdFilePtr pFile;
//   uint8_t   state;
// };

// struct SDFileSet {
//   int      fid;
//   int8_t   state;  // -128~127
//   uint8_t  ver;    // 0~255, DFileSet version
//   uint16_t reserve;
//   SDFile   files[TSDB_FILE_MAX];
// };
H
Hongze Cheng 已提交
299

H
refact  
Hongze Cheng 已提交
300 301 302 303 304
struct TSDBKEY {
  int64_t version;
  TSKEY   ts;
};

H
Hongze Cheng 已提交
305 306 307 308 309 310 311 312 313 314 315 316 317 318
typedef struct SMemSkipListNode SMemSkipListNode;
struct SMemSkipListNode {
  int8_t            level;
  SMemSkipListNode *forwards[0];
};
typedef struct SMemSkipList {
  uint32_t          seed;
  int64_t           size;
  int8_t            maxLevel;
  int8_t            level;
  SMemSkipListNode *pHead;
  SMemSkipListNode *pTail;
} SMemSkipList;

H
Hongze Cheng 已提交
319
struct STbData {
H
Hongze Cheng 已提交
320 321 322 323 324 325 326
  tb_uid_t     suid;
  tb_uid_t     uid;
  TSDBKEY      minKey;
  TSDBKEY      maxKey;
  SDelOp      *pHead;
  SDelOp      *pTail;
  SMemSkipList sl;
H
Hongze Cheng 已提交
327
};
H
Hongze Cheng 已提交
328

H
refact  
Hongze Cheng 已提交
329
struct SMemTable {
H
Hongze Cheng 已提交
330 331 332 333 334 335
  SRWLatch latch;
  STsdb   *pTsdb;
  int32_t  nRef;
  TSDBKEY  minKey;
  TSDBKEY  maxKey;
  int64_t  nRow;
H
Hongze Cheng 已提交
336
  int64_t  nDel;
H
Hongze Cheng 已提交
337
  SArray  *aTbData;  // SArray<STbData*>
H
Hongze Cheng 已提交
338
};
H
Hongze Cheng 已提交
339

H
Hongze Cheng 已提交
340 341 342 343 344
// struct STsdbFSMeta {
//   uint32_t version;       // Commit version from 0 to increase
//   int64_t  totalPoints;   // total points
//   int64_t  totalStorage;  // Uncompressed total storage
// };
H
Hongze Cheng 已提交
345 346

// ==================
H
Hongze Cheng 已提交
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
// typedef struct {
//   STsdbFSMeta meta;       // FS meta
//   SDFile      cacheFile;  // cache file
//   SDFile      tombstone;  // tomestome file
//   SArray     *df;         // data file array
//   SArray     *sf;         // sma data file array    v2f1900.index_name_1
// } SFSStatus;

// struct STsdbFS {
//   TdThreadRwlock lock;

//   SFSStatus *cstatus;  // current status
//   bool       intxn;
//   SFSStatus *nstatus;  // new status
// };

// #define REPO_ID(r)        TD_VID((r)->pVnode)
// #define REPO_CFG(r)       (&(r)->pVnode->config.tsdbCfg)
// #define REPO_KEEP_CFG(r)  (&(r)->keepCfg)
// #define REPO_FS(r)        ((r)->fs)
// #define REPO_META(r)      ((r)->pVnode->pMeta)
// #define REPO_TFS(r)       ((r)->pVnode->pTfs)
// #define IS_REPO_LOCKED(r) ((r)->repoLocked)
H
Hongze Cheng 已提交
370 371 372 373

int tsdbLockRepo(STsdb *pTsdb);
int tsdbUnlockRepo(STsdb *pTsdb);

H
Hongze Cheng 已提交
374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
// static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STsdb *pTsdb, STable *pTable, bool lock, bool copy,
//                                                      int32_t version) {
//   if ((version < 0) || (schemaVersion(pTable->pSchema) == version)) {
//     return pTable->pSchema;
//   }

//   if (!pTable->pCacheSchema || (schemaVersion(pTable->pCacheSchema) != version)) {
//     taosMemoryFreeClear(pTable->pCacheSchema);
//     pTable->pCacheSchema = metaGetTbTSchema(REPO_META(pTsdb), pTable->uid, version);
//   }
//   return pTable->pCacheSchema;
// }

// // tsdbMemTable.h
// struct SMergeInfo {
//   int   rowsInserted;
//   int   rowsUpdated;
//   int   rowsDeleteSucceed;
//   int   rowsDeleteFailed;
//   int   nOperations;
//   TSKEY keyFirst;
//   TSKEY keyLast;
// };

// static void  *taosTMalloc(size_t size);
// static void  *taosTCalloc(size_t nmemb, size_t size);
// static void  *taosTRealloc(void *ptr, size_t size);
// static void  *taosTZfree(void *ptr);
// static size_t taosTSizeof(void *ptr);
// static void   taosTMemset(void *ptr, int c);
H
Hongze Cheng 已提交
404

H
Hongze Cheng 已提交
405 406 407 408 409
struct TSDBROW {
  int64_t version;
  STSRow *pTSRow;
};

H
Hongze Cheng 已提交
410 411
// static FORCE_INLINE STSRow *tsdbNextIterRow(STbDataIter *pIter) {
//   TSDBROW row;
H
Hongze Cheng 已提交
412

H
Hongze Cheng 已提交
413
//   if (pIter == NULL) return NULL;
H
Hongze Cheng 已提交
414

H
Hongze Cheng 已提交
415 416 417
//   if (tsdbTbDataIterGet(pIter, &row)) {
//     return row.pTSRow;
//   }
H
Hongze Cheng 已提交
418

H
Hongze Cheng 已提交
419 420
//   return NULL;
// }
H
Hongze Cheng 已提交
421

H
Hongze Cheng 已提交
422 423 424
// static FORCE_INLINE TSKEY tsdbNextIterKey(STbDataIter *pIter) {
//   STSRow *row = tsdbNextIterRow(pIter);
//   if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
H
Hongze Cheng 已提交
425

H
Hongze Cheng 已提交
426 427
//   return TD_ROW_KEY(row);
// }
H
Hongze Cheng 已提交
428 429 430

// tsdbReadImpl

H
refact  
Hongze Cheng 已提交
431
struct SBlockIdx {
H
Hongze Cheng 已提交
432 433 434 435 436 437 438 439 440 441 442
  int64_t suid;
  int64_t uid;
  int64_t maxVersion;
  int64_t minVersion;
  int64_t offset;
  int64_t size;
  // uint32_t len;
  // uint32_t offset;
  // uint32_t hasLast : 2;
  // uint32_t numOfBlocks : 30;
  // TSDBKEY  maxKey;
H
refact  
Hongze Cheng 已提交
443
};
H
Hongze Cheng 已提交
444 445 446 447 448 449 450 451

typedef enum {
  TSDB_SBLK_VER_0 = 0,
  TSDB_SBLK_VER_MAX,
} ESBlockVer;

#define SBlockVerLatest TSDB_SBLK_VER_0

H
refact  
Hongze Cheng 已提交
452
struct SBlock {
H
Hongze Cheng 已提交
453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
  TSDBKEY minKey;
  TSDBKEY maxKey;
  int64_t maxVersion;
  int64_t minVersion;
  uint8_t flags;  // last, algorithm
  // uint8_t  last : 1;
  // uint8_t  hasDupKey : 1;  // 0: no dup TS key, 1: has dup TS key(since supporting Multi-Version)
  // uint8_t  blkVer : 6;
  // uint8_t  numOfSubBlocks;
  // col_id_t numOfCols;    // not including timestamp column
  // uint32_t len;          // data block length
  // uint32_t keyLen : 20;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
  // uint32_t algorithm : 4;
  // uint32_t reserve : 8;
  // col_id_t numOfBSma;
  // uint16_t numOfRows;
  // int64_t  offset;
  // uint64_t aggrStat : 1;
  // uint64_t aggrOffset : 63;
H
refact  
Hongze Cheng 已提交
472
};
H
Hongze Cheng 已提交
473

H
refact  
Hongze Cheng 已提交
474
struct SBlockInfo {
H
Hongze Cheng 已提交
475
  int32_t  delimiter;  // For recovery usage
H
Hongze Cheng 已提交
476
  uint64_t suid;
H
Hongze Cheng 已提交
477 478
  uint64_t uid;
  SBlock   blocks[];
H
refact  
Hongze Cheng 已提交
479
};
H
Hongze Cheng 已提交
480

H
refact  
Hongze Cheng 已提交
481
struct SBlockCol {
H
Hongze Cheng 已提交
482 483
  int16_t  colId;
  uint16_t type : 6;
C
Cary Xu 已提交
484 485
  uint16_t blen : 10;  // 0 no bitmap if all rows are NORM, > 0 bitmap length
  uint32_t len;        // data length + bitmap length
H
Hongze Cheng 已提交
486
  uint32_t offset;
H
refact  
Hongze Cheng 已提交
487
};
H
Hongze Cheng 已提交
488

H
refact  
Hongze Cheng 已提交
489
struct SAggrBlkCol {
H
Hongze Cheng 已提交
490 491 492 493 494 495 496
  int16_t colId;
  int16_t maxIndex;
  int16_t minIndex;
  int16_t numOfNull;
  int64_t sum;
  int64_t max;
  int64_t min;
H
refact  
Hongze Cheng 已提交
497
};
H
Hongze Cheng 已提交
498

H
refact  
Hongze Cheng 已提交
499
struct SBlockData {
H
Hongze Cheng 已提交
500 501 502 503
  int32_t   delimiter;  // For recovery usage
  int32_t   numOfCols;  // For recovery usage
  uint64_t  uid;        // For recovery usage
  SBlockCol cols[];
H
refact  
Hongze Cheng 已提交
504
};
H
Hongze Cheng 已提交
505 506 507

typedef void SAggrBlkData;  // SBlockCol cols[];

H
Hongze Cheng 已提交
508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
// struct SReadH {
//   STsdb        *pRepo;
//   SDFileSet     rSet;     // FSET to read
//   SArray       *aBlkIdx;  // SBlockIdx array
//   STable       *pTable;   // table to read
//   SBlockIdx    *pBlkIdx;  // current reading table SBlockIdx
//   int           cidx;
//   SBlockInfo   *pBlkInfo;
//   SBlockData   *pBlkData;      // Block info
//   SAggrBlkData *pAggrBlkData;  // Aggregate Block info
//   SDataCols    *pDCols[2];
//   void         *pBuf;    // buffer
//   void         *pCBuf;   // compression buffer
//   void         *pExBuf;  // extra buffer
// };

// #define TSDB_READ_REPO(rh)      ((rh)->pRepo)
// #define TSDB_READ_REPO_ID(rh)   REPO_ID(TSDB_READ_REPO(rh))
// #define TSDB_READ_FSET(rh)      (&((rh)->rSet))
// #define TSDB_READ_TABLE(rh)     ((rh)->pTable)
// #define TSDB_READ_TABLE_UID(rh) ((rh)->pTable->uid)
// #define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
// #define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
// #define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
// #define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD)
// #define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL)
// #define TSDB_READ_BUF(rh)       ((rh)->pBuf)
// #define TSDB_READ_COMP_BUF(rh)  ((rh)->pCBuf)
// #define TSDB_READ_EXBUF(rh)     ((rh)->pExBuf)

// #define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM))

// static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
//   switch (blkVer) {
//     case TSDB_SBLK_VER_0:
//     default:
//       return TSDB_BLOCK_STATIS_SIZE(nCols, 0);
//   }
// }

// #define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkCol) * (ncols) + sizeof(TSCKSUM))

// static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
//   switch (blkVer) {
//     case TSDB_SBLK_VER_0:
//     default:
//       return TSDB_BLOCK_AGGR_SIZE(nCols, 0);
//   }
// }

// static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
//   void  *pBuf = *ppBuf;
//   size_t tsize = taosTSizeof(pBuf);

//   if (tsize < size) {
//     if (tsize == 0) tsize = 1024;

//     while (tsize < size) {
//       tsize *= 2;
//     }

//     *ppBuf = taosTRealloc(pBuf, tsize);
//     if (*ppBuf == NULL) {
//       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
//       return -1;
//     }
//   }

//   return 0;
// }

// // tsdbMemory
// static FORCE_INLINE void *taosTMalloc(size_t size) {
//   if (size <= 0) return NULL;

//   void *ret = taosMemoryMalloc(size + sizeof(size_t));
//   if (ret == NULL) return NULL;

//   *(size_t *)ret = size;

//   return (void *)((char *)ret + sizeof(size_t));
// }

// static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) {
//   size_t tsize = nmemb * size;
//   void  *ret = taosTMalloc(tsize);
//   if (ret == NULL) return NULL;

//   taosTMemset(ret, 0);
//   return ret;
// }

// static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }

// static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); }

// static FORCE_INLINE void *taosTRealloc(void *ptr, size_t size) {
//   if (ptr == NULL) return taosTMalloc(size);

//   if (size <= taosTSizeof(ptr)) return ptr;

//   void  *tptr = (void *)((char *)ptr - sizeof(size_t));
//   size_t tsize = size + sizeof(size_t);
//   void  *tptr1 = taosMemoryRealloc(tptr, tsize);
//   if (tptr1 == NULL) return NULL;
//   tptr = tptr1;

//   *(size_t *)tptr = size;

//   return (void *)((char *)tptr + sizeof(size_t));
// }

// static FORCE_INLINE void *taosTZfree(void *ptr) {
//   if (ptr) {
//     taosMemoryFree((void *)((char *)ptr - sizeof(size_t)));
//   }
//   return NULL;
// }
H
Hongze Cheng 已提交
626 627 628

// tsdbCommit

H
Hongze Cheng 已提交
629
// void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
H
Hongze Cheng 已提交
630

H
Hongze Cheng 已提交
631
static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t minutes, int8_t precision) {
H
Hongze Cheng 已提交
632
  if (key < 0) {
H
Hongze Cheng 已提交
633
    return (int)((key + 1) / tsTickPerMin[precision] / minutes - 1);
H
Hongze Cheng 已提交
634
  } else {
H
Hongze Cheng 已提交
635
    return (int)((key / tsTickPerMin[precision] / minutes));
H
Hongze Cheng 已提交
636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
  }
}

static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
  if (fid >= pRtn->maxFid) {
    return 0;
  } else if (fid >= pRtn->midFid) {
    return 1;
  } else if (fid >= pRtn->minFid) {
    return 2;
  } else {
    return -1;
  }
}

// tsdbFile
H
Hongze Cheng 已提交
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
// #define TSDB_FILE_HEAD_SIZE  512
// #define TSDB_FILE_DELIMITER  0xF00AFA0F
// #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
// #define TSDB_IVLD_FID        INT_MIN
// #define TSDB_FILE_STATE_OK   0
// #define TSDB_FILE_STATE_BAD  1

// #define TSDB_FILE_F(tf)            (&((tf)->f))
// #define TSDB_FILE_PFILE(tf)        ((tf)->pFile)
// #define TSDB_FILE_FULL_NAME(tf)    (TSDB_FILE_F(tf)->aname)
// #define TSDB_FILE_OPENED(tf)       (TSDB_FILE_PFILE(tf) != NULL)
// #define TSDB_FILE_CLOSED(tf)       (!TSDB_FILE_OPENED(tf))
// #define TSDB_FILE_SET_CLOSED(f)    (TSDB_FILE_PFILE(f) = NULL)
// #define TSDB_FILE_LEVEL(tf)        (TSDB_FILE_F(tf)->did.level)
// #define TSDB_FILE_ID(tf)           (TSDB_FILE_F(tf)->did.id)
// #define TSDB_FILE_DID(tf)          (TSDB_FILE_F(tf)->did)
// #define TSDB_FILE_REL_NAME(tf)     (TSDB_FILE_F(tf)->rname)
// #define TSDB_FILE_ABS_NAME(tf)     (TSDB_FILE_F(tf)->aname)
// #define TSDB_FILE_FSYNC(tf)        taosFsyncFile(TSDB_FILE_PFILE(tf))
// #define TSDB_FILE_STATE(tf)        ((tf)->state)
// #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
// #define TSDB_FILE_IS_OK(tf)        (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
// #define TSDB_FILE_IS_BAD(tf)       (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)

// typedef enum {
//   TSDB_FS_VER_0 = 0,
//   TSDB_FS_VER_MAX,
// } ETsdbFsVer;

// #define TSDB_LATEST_FVER    TSDB_FS_VER_0  // latest version for DFile
// #define TSDB_LATEST_SFS_VER TSDB_FS_VER_0  // latest version for 'current' file

// static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) {  // latest version for DFile
//   switch (fType) {
//     case TSDB_FILE_HEAD:  // .head
//     case TSDB_FILE_DATA:  // .data
//     case TSDB_FILE_LAST:  // .last
//     case TSDB_FILE_SMAD:  // .smad(Block-wise SMA)
//     case TSDB_FILE_SMAL:  // .smal(Block-wise SMA)
//     default:
//       return TSDB_LATEST_FVER;
//   }
// }
H
Hongze Cheng 已提交
695 696 697

// =============== SDFileSet

H
Hongze Cheng 已提交
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
// #define TSDB_LATEST_FSET_VER    0
// #define TSDB_FSET_FID(s)        ((s)->fid)
// #define TSDB_FSET_STATE(s)      ((s)->state)
// #define TSDB_FSET_VER(s)        ((s)->ver)
// #define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
// #define TSDB_FSET_LEVEL(s)      TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
// #define TSDB_FSET_ID(s)         TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
// #define TSDB_FSET_SET_CLOSED(s)                                                \
//   do {                                                                         \
//     for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
//       TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
//     }                                                                          \
//   } while (0);
// #define TSDB_FSET_FSYNC(s)                                                     \
//   do {                                                                         \
//     for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
//       TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                            \
//     }                                                                          \
//   } while (0);

// static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet *pSet) {
//   for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
//     if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
//       return false;
//     }
//   }

//   return true;
// }
H
Hongze Cheng 已提交
727 728 729 730 731

// ================== TSDB global config
extern bool tsdbForceKeepFile;

// ================== CURRENT file header info
H
Hongze Cheng 已提交
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
// typedef struct {
//   uint32_t version;  // Current file system version (relating to code)
//   uint32_t len;      // Encode content length (including checksum)
// } SFSHeader;

// // ================== TSDB File System Meta
// #define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
// #define FS_NEW_STATUS(pfs)     ((pfs)->nstatus)
// #define FS_IN_TXN(pfs)         (pfs)->intxn
// #define FS_VERSION(pfs)        ((pfs)->cstatus->meta.version)
// #define FS_TXN_VERSION(pfs)    ((pfs)->nstatus->meta.version)

// struct SFSIter {
//   int        direction;
//   uint64_t   version;  // current FS version
//   STsdbFS   *pfs;
//   int        index;  // used to position next fset when version the same
//   int        fid;    // used to seek when version is changed
//   SDFileSet *pSet;
// };
H
Hongze Cheng 已提交
752

H
Hongze Cheng 已提交
753
#define TSDB_FS_ITER_FORWARD  TSDB_ORDER_ASC
H
Hongze Cheng 已提交
754 755
#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC

H
Hongze Cheng 已提交
756 757 758 759 760
struct TABLEID {
  tb_uid_t suid;
  tb_uid_t uid;
};

H
Hongze Cheng 已提交
761 762 763 764 765 766
struct STbDataIter {
  STbData          *pTbData;
  int8_t            backward;
  SMemSkipListNode *pNode;
};

H
more  
Hongze Cheng 已提交
767 768 769 770
struct SDelOp {
  int64_t version;
  TSKEY   sKey;  // included
  TSKEY   eKey;  // included
H
Hongze Cheng 已提交
771
  SDelOp *pNext;
H
more  
Hongze Cheng 已提交
772 773
};

H
Hongze Cheng 已提交
774
typedef struct {
H
Hongze Cheng 已提交
775 776 777 778 779 780
  int64_t version;
  TSKEY   sKey;
  TSKEY   eKey;
} SDelDataItem;

struct SDelData {
H
Hongze Cheng 已提交
781 782 783
  int64_t version;
  TSKEY   sKey;
  TSKEY   eKey;
H
Hongze Cheng 已提交
784
};
H
Hongze Cheng 已提交
785

H
Hongze Cheng 已提交
786
struct SDelIdx {
H
Hongze Cheng 已提交
787 788 789 790 791
  tb_uid_t suid;
  tb_uid_t uid;
  TSKEY    minKey;
  TSKEY    maxKey;
  int64_t  minVersion;
H
Hongze Cheng 已提交
792
  int64_t  maxVersion;
H
Hongze Cheng 已提交
793 794
  int64_t  offset;
  int64_t  size;
H
Hongze Cheng 已提交
795
};
H
Hongze Cheng 已提交
796

H
Hongze Cheng 已提交
797
struct STMap {
H
Hongze Cheng 已提交
798 799
  uint8_t  flags;
  uint32_t nOffset;
H
Hongze Cheng 已提交
800
  uint8_t *pOffset;
H
Hongze Cheng 已提交
801 802
  uint32_t nData;
  uint8_t *pData;
H
Hongze Cheng 已提交
803 804
};

H
Hongze Cheng 已提交
805 806 807 808 809 810 811 812
struct SDelFile {
  TSKEY   minKey;
  TSKEY   maxKey;
  int64_t minVersion;
  int64_t maxVersion;
  int64_t size;
  int64_t offset;
};
H
Hongze Cheng 已提交
813

H
Hongze Cheng 已提交
814 815 816 817 818
#ifdef __cplusplus
}
#endif

#endif /*_TD_VNODE_TSDB_H_*/