tsdbMain.h 13.9 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
TD-34  
hzcheng 已提交
15 16
#ifndef _TD_TSDB_MAIN_H_
#define _TD_TSDB_MAIN_H_
H
more  
Hongze Cheng 已提交
17

18
#include "tsdb.h"
H
TD-34  
hzcheng 已提交
19 20 21 22
#include "tlist.h"
#include "tglobalcfg.h"
#include "tskiplist.h"
#include "tutil.h"
H
more  
Hongze Cheng 已提交
23

H
more  
hzcheng 已提交
24 25 26 27
#ifdef __cplusplus
extern "C" {
#endif

H
TD-34  
hzcheng 已提交
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
// ------------------------------ TSDB META FILE INTERFACES ------------------------------
#define TSDB_META_FILE_NAME "META"
#define TSDB_META_HASH_FRACTION 1.1

typedef int (*iterFunc)(void *, void *cont, int contLen);
typedef void (*afterFunc)(void *);

typedef struct {
  int       fd;        // File descriptor
  int       nDel;      // number of deletions
  int       tombSize;  // deleted size
  int64_t   size;      // Total file size
  void *    map;       // Map from uid ==> position
  iterFunc  iFunc;
  afterFunc aFunc;
  void *    appH;
} SMetaFile;

SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, afterFunc aFunc, void *appH);
int32_t    tsdbInsertMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen);
int32_t    tsdbDeleteMetaRecord(SMetaFile *mfh, int64_t uid);
int32_t    tsdbUpdateMetaRecord(SMetaFile *mfh, int64_t uid, void *cont, int32_t contLen);
void       tsdbCloseMetaFile(SMetaFile *mfh);

// ------------------------------ TSDB META INTERFACES ------------------------------
#define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL)

typedef struct {
  TSKEY   keyFirst;
  TSKEY   keyLast;
  int32_t numOfPoints;
  void *  pData;
} SMemTable;

// ---------- TSDB TABLE DEFINITION
typedef struct STable {
  int8_t         type;
  STableId       tableId;
  int64_t        superUid;  // Super table UID
  int32_t        sversion;
  STSchema *     schema;
  STSchema *     tagSchema;
  SDataRow       tagVal;
  SMemTable *    mem;
  SMemTable *    imem;
  void *         pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
  void *         eventHandler;   // TODO
  void *         streamHandler;  // TODO
  struct STable *next;           // TODO: remove the next
} STable;

void *  tsdbEncodeTable(STable *pTable, int *contLen);
STable *tsdbDecodeTable(void *cont, int contLen);
H
hzcheng 已提交
81
void    tsdbFreeEncode(void *cont);
H
TD-34  
hzcheng 已提交
82 83 84 85 86 87 88 89 90 91 92

// ---------- TSDB META HANDLE DEFINITION
typedef struct {
  int32_t maxTables;  // Max number of tables

  int32_t nTables;  // Tables created

  STable **tables;  // table array

  STable *superList;  // super table list TODO: change  it to list container

H
hzcheng 已提交
93
  void *map;  // table map of (uid ===> table)
H
TD-34  
hzcheng 已提交
94

H
hzcheng 已提交
95
  SMetaFile *mfh;  // meta file handle
H
TD-34  
hzcheng 已提交
96 97 98 99
  int        maxRowBytes;
  int        maxCols;
} STsdbMeta;

H
hzcheng 已提交
100
STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables);
H
TD-34  
hzcheng 已提交
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
int32_t    tsdbFreeMeta(STsdbMeta *pMeta);
STSchema * tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable);

// ---- Operation on STable
#define TSDB_TABLE_ID(pTable) ((pTable)->tableId)
#define TSDB_TABLE_UID(pTable) ((pTable)->uid)
#define TSDB_TABLE_NAME(pTable) ((pTable)->tableName)
#define TSDB_TABLE_TYPE(pTable) ((pTable)->type)
#define TSDB_TABLE_SUPER_TABLE_UID(pTable) ((pTable)->stableUid)
#define TSDB_TABLE_IS_SUPER_TABLE(pTable) (TSDB_TABLE_TYPE(pTable) == TSDB_SUPER_TABLE)
#define TSDB_TABLE_TAG_VALUE(pTable) ((pTable)->pTagVal)
#define TSDB_TABLE_CACHE_DATA(pTable) ((pTable)->content.pData)
#define TSDB_SUPER_TABLE_INDEX(pTable) ((pTable)->content.pIndex)

// ---- Operation on SMetaHandle
#define TSDB_NUM_OF_TABLES(pHandle) ((pHandle)->numOfTables)
#define TSDB_NUM_OF_SUPER_TABLES(pHandle) ((pHandle)->numOfSuperTables)
#define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id]
#define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */

H
hzcheng 已提交
121
STsdbMeta *tsdbGetMeta(tsdb_repo_t *pRepo);
H
TD-34  
hzcheng 已提交
122 123 124 125 126 127

int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg);
int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId);
STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId);
// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable);
STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid);
H
hzcheng 已提交
128
char *  getTupleKey(const void *data);
H
TD-34  
hzcheng 已提交
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168

// ------------------------------ TSDB CACHE INTERFACES ------------------------------
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024 /* 16M */

typedef struct {
  int  blockId;
  int  offset;
  int  remain;
  int  padding;
  char data[];
} STsdbCacheBlock;

typedef struct {
  int64_t index;
  SList * memPool;
} STsdbCachePool;

typedef struct {
  TSKEY   keyFirst;
  TSKEY   keyLast;
  int64_t numOfPoints;
  SList * list;
} SCacheMem;

typedef struct {
  int              maxBytes;
  int              cacheBlockSize;
  int              totalCacheBlocks;
  STsdbCachePool   pool;
  STsdbCacheBlock *curBlock;
  SCacheMem *      mem;
  SCacheMem *      imem;
  tsdb_repo_t *    pRepo;
} STsdbCache;

STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, tsdb_repo_t *pRepo);
void        tsdbFreeCache(STsdbCache *pCache);
void *      tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key);

// ------------------------------ TSDB FILE INTERFACES ------------------------------
H
TD-34  
hzcheng 已提交
169
#define TSDB_FILE_HEAD_SIZE 512
H
TD-34  
hzcheng 已提交
170
#define TSDB_FILE_DELIMITER 0xF00AFA0F
H
TD-34  
hzcheng 已提交
171

H
TD-34  
hzcheng 已提交
172 173 174
#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3)

H
Hongze Cheng 已提交
175
typedef enum {
H
TD-34  
hzcheng 已提交
176 177 178 179
  TSDB_FILE_TYPE_HEAD = 0,  // .head file type
  TSDB_FILE_TYPE_DATA,      // .data file type
  TSDB_FILE_TYPE_LAST,      // .last file type
  TSDB_FILE_TYPE_MAX
H
more  
Hongze Cheng 已提交
180 181
} TSDB_FILE_TYPE;

H
TD-34  
hzcheng 已提交
182 183
#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX)

H
more  
hzcheng 已提交
184
extern const char *tsdbFileSuffix[];
H
Hongze Cheng 已提交
185

H
more  
Hongze Cheng 已提交
186
typedef struct {
H
TD-34  
hzcheng 已提交
187 188
  int64_t size;      // total size of the file
  int64_t tombSize;  // unused file size
H
TD-34  
hzcheng 已提交
189 190
  int32_t totalBlocks;
  int32_t totalSubBlocks;
H
TD-34  
hzcheng 已提交
191 192 193
} SFileInfo;

typedef struct {
H
hzcheng 已提交
194 195
  int       fd;
  char      fname[128];
H
TD-34  
hzcheng 已提交
196
  SFileInfo info;
H
hzcheng 已提交
197
} SFile;
H
more  
Hongze Cheng 已提交
198

H
TD-34  
hzcheng 已提交
199 200
#define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1)

H
hzcheng 已提交
201 202
typedef struct {
  int32_t fileId;
H
TD-34  
hzcheng 已提交
203
  SFile   files[TSDB_FILE_TYPE_MAX];
H
hzcheng 已提交
204 205 206 207
} SFileGroup;

// TSDB file handle
typedef struct {
H
TD-34  
hzcheng 已提交
208 209 210
  int maxFGroups;
  int numOfFGroups;

H
hzcheng 已提交
211 212
  SFileGroup fGroup[];
} STsdbFileH;
H
more  
Hongze Cheng 已提交
213

H
TD-34  
hzcheng 已提交
214 215 216 217 218
#define TSDB_MIN_FILE_ID(fh) (fh)->fGroup[0].fileId
#define TSDB_MAX_FILE_ID(fh) (fh)->fGroup[(fh)->numOfFGroups - 1].fileId

STsdbFileH *tsdbInitFileH(char *dataDir, int maxFiles);
void        tsdbCloseFileH(STsdbFileH *pFileH);
H
hzcheng 已提交
219 220 221
int         tsdbCreateFile(char *dataDir, int fileId, const char *suffix, int maxTables, SFile *pFile, int writeHeader,
                           int toClose);
SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables);
H
TD-34  
hzcheng 已提交
222
int         tsdbOpenFile(SFile *pFile, int oflag);
H
hzcheng 已提交
223 224
int         tsdbCloseFile(SFile *pFile);
SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid);
H
TD-34  
hzcheng 已提交
225
int         tsdbRemoveFileGroup(STsdbFileH *pFile, int fid);
H
hzcheng 已提交
226
int         tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname);
H
TD-34  
hzcheng 已提交
227

228 229 230
#define TSDB_FGROUP_ITER_FORWARD  TSDB_ORDER_ASC
#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC

H
TD-34  
hzcheng 已提交
231 232 233 234 235 236 237 238 239 240 241
typedef struct {
  int         numOfFGroups;
  SFileGroup *base;
  SFileGroup *pFileGroup;
  int         direction;
} SFileGroupIter;

void        tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction);
void        tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid);
SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter);

H
TD-34  
hzcheng 已提交
242 243
typedef struct {
  int32_t len;
H
TD-34  
hzcheng 已提交
244 245 246 247 248 249
  int32_t offset;
  int32_t hasLast : 1;
  int32_t numOfSuperBlocks : 31;
  int32_t checksum;
  TSKEY   maxKey;
} SCompIdx; /* sizeof(SCompIdx) = 24 */
H
TD-34  
hzcheng 已提交
250

251
/**
H
TD-34  
hzcheng 已提交
252 253 254 255 256 257
 * if numOfSubBlocks == 0, then the SCompBlock is a sub-block
 * if numOfSubBlocks >= 1, then the SCompBlock is a super-block
 *    - if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to
 *      the data block offset and length
 *    - if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the
 *      binary
258 259 260 261 262 263 264 265 266 267 268 269 270 271
 */
typedef struct {
  int64_t last : 1;          // If the block in data file or last file
  int64_t offset : 63;       // Offset of data block or sub-block index depending on numOfSubBlocks
  int32_t algorithm : 8;     // Compression algorithm
  int32_t numOfPoints : 24;  // Number of total points
  int32_t sversion;          // Schema version
  int32_t len;               // Data block length or nothing
  int16_t numOfSubBlocks;    // Number of sub-blocks;
  int16_t numOfCols;
  TSKEY   keyFirst;
  TSKEY   keyLast;
} SCompBlock;

H
hzcheng 已提交
272 273
// Maximum number of sub-blocks a super-block can have
#define TSDB_MAX_SUBBLOCKS 8
H
TD-34  
hzcheng 已提交
274 275 276
#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1)
#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)

H
TD-34  
hzcheng 已提交
277 278 279 280 281 282
typedef struct {
  int32_t    delimiter;  // For recovery usage
  int32_t    checksum;   // TODO: decide if checksum logic in this file or make it one API
  int64_t    uid;
  SCompBlock blocks[];
} SCompInfo;
H
hzcheng 已提交
283

H
TD-34  
hzcheng 已提交
284
#define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx))
H
hzcheng 已提交
285 286 287 288 289 290 291 292 293
#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size) \
  do {                                                                 \
    if (pCompBlock->numOfSubBlocks > 1) {                              \
      pCompBlock = pCompInfo->blocks + pCompBlock->offset;             \
      size = pCompBlock->numOfSubBlocks;                               \
    } else {                                                           \
      size = 1;                                                        \
    }                                                                  \
  } while (0)
H
TD-34  
hzcheng 已提交
294

H
TD-34  
hzcheng 已提交
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
// TODO: take pre-calculation into account
typedef struct {
  int16_t colId;  // Column ID
  int16_t len;    // Column length
  int32_t type : 8;
  int32_t offset : 24;
} SCompCol;

// TODO: Take recover into account
typedef struct {
  int32_t  delimiter;  // For recovery usage
  int32_t  numOfCols;  // For recovery usage
  int64_t  uid;        // For recovery usage
  SCompCol cols[];
} SCompData;
H
TD-34  
hzcheng 已提交
310

H
hzcheng 已提交
311
STsdbFileH *tsdbGetFile(tsdb_repo_t *pRepo);
H
TD-34  
hzcheng 已提交
312

H
hzcheng 已提交
313 314
int         tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast,
                                    SDataCols *pCols);
H
TD-34  
hzcheng 已提交
315
SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid);
H
TD-34  
hzcheng 已提交
316
void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey);
H
TD-34  
hzcheng 已提交
317 318 319 320 321 322 323

// TSDB repository definition
typedef struct _tsdb_repo {
  char *rootDir;
  // TSDB configuration
  STsdbCfg config;

H
hzcheng 已提交
324 325
  STsdbAppH appH;

H
TD-34  
hzcheng 已提交
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
  // The meter meta handle of this TSDB repository
  STsdbMeta *tsdbMeta;

  // The cache Handle
  STsdbCache *tsdbCache;

  // The TSDB file handle
  STsdbFileH *tsdbFileH;

  // Disk tier handle for multi-tier storage
  void *diskTier;

  pthread_mutex_t mutex;

  int       commit;
  pthread_t commitThread;

  // A limiter to monitor the resources used by tsdb
  void *limiter;

  int8_t state;

} STsdbRepo;

H
hzcheng 已提交
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;

typedef struct {
  tsdb_rw_helper_t type;  // helper type
  int              maxTables;
  int              maxRowSize;
  int              maxRows;
  int              maxCols;
  int              minRowsPerFileBlock;
  int              maxRowsPerFileBlock;
  int8_t           compress;
} SHelperCfg;

typedef struct {
  int fid;
  TSKEY minKey;
  TSKEY maxKey;
  // For read/write purpose
  SFile headF;
  SFile dataF;
  SFile lastF;
  // For write purpose only
  SFile nHeadF;
  SFile nLastF;
} SHelperFile;

typedef struct {
  int64_t uid;
  int32_t tid;
  int32_t sversion;
} SHelperTable;

typedef struct {
  // Global configuration
  SHelperCfg config;

H
TD-100  
hzcheng 已提交
386 387 388
  int8_t state;

  // For file set usage
H
hzcheng 已提交
389
  SHelperFile files;
H
TD-100  
hzcheng 已提交
390 391
  SCompIdx *  pCompIdx;
  size_t      compIdxSize;
H
hzcheng 已提交
392

H
TD-100  
hzcheng 已提交
393
  // For table set usage
H
hzcheng 已提交
394
  SHelperTable tableInfo;
H
TD-100  
hzcheng 已提交
395 396 397
  SCompInfo *  pCompInfo;
  size_t       compInfoSize;
  bool         hasOldLastBlock;
H
hzcheng 已提交
398

H
TD-100  
hzcheng 已提交
399
  // For block set usage
H
hzcheng 已提交
400 401 402 403 404 405 406
  SCompData *pCompData;
  size_t     compDataSize;
  SDataCols *pDataCols[2];

} SRWHelper;

// --------- Helper state
H
TD-100  
hzcheng 已提交
407 408 409 410 411 412
#define TSDB_HELPER_CLEAR_STATE 0x0        // Clear state
#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1  // File is set
#define TSDB_HELPER_IDX_LOAD 0x2           // SCompIdx part is loaded
#define TSDB_HELPER_TABLE_SET 0x4          // Table is set
#define TSDB_HELPER_INFO_LOAD 0x8          // SCompInfo part is loaded
#define TSDB_HELPER_FILE_DATA_LOAD 0x10    // SCompData part is loaded
H
hzcheng 已提交
413 414 415 416 417 418 419 420 421

#define TSDB_HELPER_TYPE(h) ((h)->config.type)

#define helperSetState(h, s) (((h)->state) |= (s))
#define helperClearState(h, s) ((h)->state &= (~(s)))
#define helperHasState(h, s) ((((h)->state) & (s)) == (s))

int  tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg);
void tsdbDestroyHelper(SRWHelper *pHelper);
H
TD-100  
hzcheng 已提交
422
void tsdbResetHelper(SRWHelper *pHelper);
H
hzcheng 已提交
423 424

// --------- For set operations
H
TD-100  
hzcheng 已提交
425
int  tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup);
H
hzcheng 已提交
426 427 428 429 430 431
void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema);
int  tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);

// --------- For read operations
int tsdbLoadCompIdx(SRWHelper *pHelper, void *target);
int tsdbLoadCompInfo(SRWHelper *pHelper, void *target);
H
TD-100  
hzcheng 已提交
432 433
int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target);
int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds);
H
TD-100  
hzcheng 已提交
434
int tsdbLoadBlockData(SRWHelper *pHelper, int blkIdx, SDataCols *target);
H
hzcheng 已提交
435 436 437 438 439 440

// --------- For write operations
int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols);
int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper);
int tsdbWriteCompInfo(SRWHelper *pHelper);
int tsdbWriteCompIdx(SRWHelper *pHelper);
H
TD-34  
hzcheng 已提交
441

H
more  
hzcheng 已提交
442 443 444 445
#ifdef __cplusplus
}
#endif

H
TD-34  
hzcheng 已提交
446
#endif