tsdbMain.h 22.0 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
TD-34  
hzcheng 已提交
15 16
#ifndef _TD_TSDB_MAIN_H_
#define _TD_TSDB_MAIN_H_
H
more  
Hongze Cheng 已提交
17

S
TD-1057  
Shengliang Guan 已提交
18
#include "os.h"
H
TD-353  
Hongze Cheng 已提交
19 20
#include "hash.h"
#include "tcoding.h"
S
slguan 已提交
21
#include "tglobal.h"
H
TD-353  
Hongze Cheng 已提交
22
#include "tkvstore.h"
H
hzcheng 已提交
23
#include "tlist.h"
H
TD-353  
Hongze Cheng 已提交
24
#include "tlog.h"
B
Bomin Zhang 已提交
25
#include "tlockfree.h"
H
hzcheng 已提交
26
#include "tsdb.h"
H
TD-34  
hzcheng 已提交
27 28
#include "tskiplist.h"
#include "tutil.h"
H
Hongze Cheng 已提交
29
#include "tfs.h"
H
more  
Hongze Cheng 已提交
30

H
more  
hzcheng 已提交
31 32 33 34
#ifdef __cplusplus
extern "C" {
#endif

H
refact  
Hongze Cheng 已提交
35 36 37
typedef struct STsdbRepo STsdbRepo;

// ================= tsdbLog.h
S
Shengliang Guan 已提交
38
extern int32_t tsdbDebugFlag;
H
hzcheng 已提交
39

H
Haojun Liao 已提交
40 41 42 43 44 45
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
H
hzcheng 已提交
46

H
refact  
Hongze Cheng 已提交
47
// ================= OTHERS
H
TD-353  
Hongze Cheng 已提交
48

H
Hongze Cheng 已提交
49 50
#define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax)))

51 52 53 54
// NOTE: Any file format change must increase this version number by 1
//       Also, implement the convert function
#define TSDB_FILE_VERSION ((uint32_t)0)

H
TD-353  
Hongze Cheng 已提交
55
// Definitions
H
refact  
Hongze Cheng 已提交
56
// ================= tsdbMeta.c
H
Hongze Cheng 已提交
57 58
#define TSDB_MAX_TABLE_SCHEMAS 16

H
TD-34  
hzcheng 已提交
59
typedef struct STable {
H
Haojun Liao 已提交
60
  STableId       tableId;
H
TD-353  
Hongze Cheng 已提交
61 62 63 64 65
  ETableType     type;
  tstr*          name;  // NOTE: there a flexible string here
  uint64_t       suid;
  struct STable* pSuper;  // super table pointer
  uint8_t        numOfSchemas;
H
TD-353  
Hongze Cheng 已提交
66
  STSchema*      schema[TSDB_MAX_TABLE_SCHEMAS];
H
TD-353  
Hongze Cheng 已提交
67 68
  STSchema*      tagSchema;
  SKVRow         tagVal;
H
TD-353  
Hongze Cheng 已提交
69
  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
H
TD-353  
Hongze Cheng 已提交
70 71
  void*          eventHandler;   // TODO
  void*          streamHandler;  // TODO
H
Hongze Cheng 已提交
72 73
  TSKEY          lastKey;
  SDataRow       lastRow;
H
TD-353  
Hongze Cheng 已提交
74 75
  char*          sql;
  void*          cqhandle;
H
Hongze Cheng 已提交
76
  SRWLatch       latch;  // TODO: implementa latch functions
S
TD-1057  
Shengliang Guan 已提交
77
  T_REF_DECLARE()
H
TD-34  
hzcheng 已提交
78 79 80
} STable;

typedef struct {
H
TD-353  
Hongze Cheng 已提交
81 82 83
  pthread_rwlock_t rwLock;

  int32_t   nTables;
H
TD-987  
Hongze Cheng 已提交
84
  int32_t   maxTables;
H
TD-353  
Hongze Cheng 已提交
85 86 87 88
  STable**  tables;
  SList*    superList;
  SHashObj* uidMap;
  SKVStore* pStore;
H
TD-353  
Hongze Cheng 已提交
89 90
  int       maxRowBytes;
  int       maxCols;
H
TD-34  
hzcheng 已提交
91 92
} STsdbMeta;

H
refact  
Hongze Cheng 已提交
93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
#define TSDB_INIT_NTABLES 1024
#define TABLE_TYPE(t) (t)->type
#define TABLE_NAME(t) (t)->name
#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
#define TABLE_UID(t) (t)->tableId.uid
#define TABLE_TID(t) (t)->tableId.tid
#define TABLE_SUID(t) (t)->suid
#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch))
#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch))
#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch))
#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch))

STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
void       tsdbFreeMeta(STsdbMeta* pMeta);
int        tsdbOpenMeta(STsdbRepo* pRepo);
int        tsdbCloseMeta(STsdbRepo* pRepo);
STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
void       tsdbRefTable(STable* pTable);
void       tsdbUnRefTable(STable* pTable);
void       tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);

static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
  if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
    return -1;
  } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
    return 1;
  } else {
    return 0;
  }
}

static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) {
  STable*   pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable;
  STSchema* pSchema = NULL;
  STSchema* pTSchema = NULL;

  if (lock) TSDB_RLOCK_TABLE(pDTable);
  if (version < 0) {  // get the latest version of schema
    pTSchema = pDTable->schema[pDTable->numOfSchemas - 1];
  } else {  // get the schema with version
    void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*),
                            tsdbCompareSchemaVersion, TD_EQ);
    if (ptr == NULL) {
      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
      goto _exit;
    }
    pTSchema = *(STSchema**)ptr;
  }

  ASSERT(pTSchema != NULL);

  if (copy) {
    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
  } else {
    pSchema = pTSchema;
  }

_exit:
  if (lock) TSDB_RUNLOCK_TABLE(pDTable);
  return pSchema;
}

static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
  return tsdbGetTableSchemaImpl(pTable, false, false, -1);
}

static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
    STable *pSuper = pTable->pSuper;
    if (pSuper == NULL) return NULL;
    return pSuper->tagSchema;
  } else if (pTable->type == TSDB_SUPER_TABLE) {
    return pTable->tagSchema;
  } else {
    return NULL;
  }
}

static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
  ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow));
  return pTable->lastKey;
}

H
refact  
Hongze Cheng 已提交
181
// ================= tsdbBuffer.c
H
TD-34  
hzcheng 已提交
182
typedef struct {
H
TD-353  
Hongze Cheng 已提交
183 184 185 186 187
  int64_t blockId;
  int     offset;
  int     remain;
  char    data[];
} STsdbBufBlock;
H
TD-34  
hzcheng 已提交
188 189

typedef struct {
H
TD-353  
Hongze Cheng 已提交
190 191 192 193 194 195 196 197
  pthread_cond_t poolNotEmpty;
  int            bufBlockSize;
  int            tBufBlocks;
  int            nBufBlocks;
  int64_t        index;
  SList*         bufBlockList;
} STsdbBufPool;

H
refact  
Hongze Cheng 已提交
198 199 200 201 202 203 204 205
#define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold

STsdbBufPool* tsdbNewBufPool();
void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
int           tsdbOpenBufPool(STsdbRepo* pRepo);
void          tsdbCloseBufPool(STsdbRepo* pRepo);
SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);

H
TD-353  
Hongze Cheng 已提交
206
// ------------------ tsdbMemTable.c
H
refact  
Hongze Cheng 已提交
207 208 209 210 211 212 213 214 215 216
typedef struct {
  int   rowsInserted;
  int   rowsUpdated;
  int   rowsDeleteSucceed;
  int   rowsDeleteFailed;
  int   nOperations;
  TSKEY keyFirst;
  TSKEY keyLast;
} SMergeInfo;

H
Hongze Cheng 已提交
217 218 219 220 221
typedef struct {
  STable *           pTable;
  SSkipListIterator *pIter;
} SCommitIter;

H
TD-34  
hzcheng 已提交
222
typedef struct {
H
TD-353  
Hongze Cheng 已提交
223 224 225 226 227 228
  uint64_t   uid;
  TSKEY      keyFirst;
  TSKEY      keyLast;
  int64_t    numOfRows;
  SSkipList* pData;
} STableData;
H
TD-34  
hzcheng 已提交
229 230

typedef struct {
S
TD-1057  
Shengliang Guan 已提交
231
  T_REF_DECLARE()
H
TD-987  
Hongze Cheng 已提交
232
  SRWLatch     latch;
H
TD-353  
Hongze Cheng 已提交
233 234 235
  TSKEY        keyFirst;
  TSKEY        keyLast;
  int64_t      numOfRows;
H
TD-987  
Hongze Cheng 已提交
236
  int32_t      maxTables;
H
TD-353  
Hongze Cheng 已提交
237 238
  STableData** tData;
  SList*       actList;
H
Hongze Cheng 已提交
239
  SList*       extraBuffList;
H
TD-353  
Hongze Cheng 已提交
240 241
  SList*       bufBlockList;
} SMemTable;
H
TD-34  
hzcheng 已提交
242

H
TD-353  
Hongze Cheng 已提交
243
enum { TSDB_UPDATE_META, TSDB_DROP_META };
S
TD-1057  
Shengliang Guan 已提交
244 245 246 247 248

#ifdef WINDOWS
#pragma pack(push ,1) 
typedef struct {
#else
H
TD-353  
Hongze Cheng 已提交
249
typedef struct __attribute__((packed)){
S
TD-1057  
Shengliang Guan 已提交
250
#endif
H
TD-353  
Hongze Cheng 已提交
251 252 253
  char     act;
  uint64_t uid;
} SActObj;
S
TD-1057  
Shengliang Guan 已提交
254 255 256
#ifdef WINDOWS
#pragma pack(pop) 
#endif
H
TD-353  
Hongze Cheng 已提交
257 258 259 260 261 262

typedef struct {
  int  len;
  char cont[];
} SActCont;

H
refact  
Hongze Cheng 已提交
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem);
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
int   tsdbAsyncCommit(STsdbRepo* pRepo);
int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
void* tsdbCommitData(STsdbRepo* pRepo);

static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
  if (pIter == NULL) return NULL;

  SSkipListNode* node = tSkipListIterGet(pIter);
  if (node == NULL) return NULL;

  return (SDataRow)SL_GET_NODE_DATA(node);
}

static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
  SDataRow row = tsdbNextIterRow(pIter);
  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;

  return dataRowKey(row);
}

static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
  SDataRow row = tsdbNextIterRow(pIter);
  if (row == NULL) return TKEY_NULL;

  return dataRowTKey(row);
}

H
Hongze Cheng 已提交
296 297 298 299
// ================= tsdbFS.c
#define TSDB_FILE_HEAD_SIZE 512
#define TSDB_FILE_DELIMITER 0xF00AFA0F
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
H
Hongze Cheng 已提交
300

H
Hongze Cheng 已提交
301 302 303
enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX };

// For meta file
H
Hongze Cheng 已提交
304
typedef struct {
H
Hongze Cheng 已提交
305 306 307 308 309 310 311 312 313 314 315 316
  int64_t  size;
  int64_t  tombSize;
  int64_t  nRecords;
  int64_t  nDels;
  uint32_t magic;
} SMFInfo;

typedef struct {
  SMFInfo info;
  TFILE   f;
  int     fd;
} SMFile;
H
Hongze Cheng 已提交
317

H
Hongze Cheng 已提交
318
// For .head/.data/.last file
H
more  
Hongze Cheng 已提交
319
typedef struct {
H
Hongze Cheng 已提交
320
  uint32_t magic;
H
TD-353  
Hongze Cheng 已提交
321 322 323
  uint32_t len;
  uint32_t totalBlocks;
  uint32_t totalSubBlocks;
H
Hongze Cheng 已提交
324
  uint32_t offset;
H
Hongze Cheng 已提交
325 326 327
  uint64_t size;
  uint64_t tombSize;
} SDFInfo;
H
TD-34  
hzcheng 已提交
328 329

typedef struct {
H
Hongze Cheng 已提交
330 331 332 333
  SDFInfo info;
  TFILE   f;
  int     fd;
} SDFile;
H
more  
Hongze Cheng 已提交
334

H
hzcheng 已提交
335
typedef struct {
H
Hongze Cheng 已提交
336 337 338 339
  int    id;
  int    state;
  SDFile files[TSDB_FILE_MAX];
} SDFileSet;
H
hzcheng 已提交
340

H
Hongze Cheng 已提交
341 342
/* Statistic information of the TSDB file system.
 */
H
hzcheng 已提交
343
typedef struct {
H
Hongze Cheng 已提交
344 345 346 347 348
  int64_t fsversion; // file system version, related to program
  int64_t version;
  int64_t totalPoints;
  int64_t totalStorage;
} STsdbFSMeta;
H
TD-353  
Hongze Cheng 已提交
349

H
Hongze Cheng 已提交
350 351 352 353 354 355
typedef struct {
  int64_t     version;
  STsdbFSMeta meta;
  SMFile      mf;  // meta file
  SArray *    df;  // data file array
} SFSSnapshot;
H
more  
Hongze Cheng 已提交
356

H
TD-34  
hzcheng 已提交
357
typedef struct {
H
Hongze Cheng 已提交
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
  pthread_rwlock_t lock;

  SFSSnapshot *curr;
  SFSSnapshot *new;
} STsdbFS;

#define TSDB_FILE_INFO(tf) (&((tf)->info))
#define TSDB_FILE_F(tf) (&((tf)->f)))
#define TSDB_FILE_FD(tf) ((tf)->fd)

int  tsdbOpenFS(STsdbRepo *pRepo);
void tsdbCloseFS(STsdbRepo *pRepo);
int  tsdbFSNewTxn(STsdbRepo *pRepo);
int  tsdbFSEndTxn(STsdbRepo *pRepo, bool hasError);
int  tsdbUpdateMFile(STsdbRepo *pRepo, SMFile *pMFile);
int  tsdbUpdateDFileSet(STsdbRepo *pRepo, SDFileSet *pSet);
void tsdbRemoveExpiredDFileSet(STsdbRepo *pRepo, int mfid);
int  tsdbRemoveDFileSet(SDFileSet *pSet);

static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) {
  int code = pthread_rwlock_rdlock(&(pFs->lock));
  if (code != 0) {
    terrno = TAOS_SYSTEM_ERROR(code);
    return -1;
  }
  return 0;
}

static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) {
  int code = pthread_rwlock_wrlock(&(pFs->lock));
  if (code != 0) {
    terrno = TAOS_SYSTEM_ERROR(code);
    return -1;
  }
  return 0;
}

static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
  int code = pthread_rwlock_unlock(&(pFs->lock));
  if (code != 0) {
    terrno = TAOS_SYSTEM_ERROR(code);
    return -1;
  }
  return 0;
}


// ================= tsdbFile.c
// extern const char* tsdbFileSuffix[];

// minFid <= midFid <= maxFid
// typedef struct {
//   int minFid;  // >= minFid && < midFid, at level 2
//   int midFid;  // >= midFid && < maxFid, at level 1
//   int maxFid;  // >= maxFid, at level 0
// } SFidGroup;

// typedef enum {
//   TSDB_FILE_TYPE_HEAD = 0,
//   TSDB_FILE_TYPE_DATA,
//   TSDB_FILE_TYPE_LAST,
//   TSDB_FILE_TYPE_STAT,
//   TSDB_FILE_TYPE_NHEAD,
//   TSDB_FILE_TYPE_NDATA,
//   TSDB_FILE_TYPE_NLAST,
//   TSDB_FILE_TYPE_NSTAT
// } TSDB_FILE_TYPE;

// #ifndef TDINTERNAL
// #define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1)
// #else
// #define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1)
// #endif

// typedef struct {
//   uint32_t magic;
//   uint32_t len;
//   uint32_t totalBlocks;
//   uint32_t totalSubBlocks;
//   uint32_t offset;
//   uint64_t size;      // total size of the file
//   uint64_t tombSize;  // unused file size
// } STsdbFileInfo;

// typedef struct {
//   TFILE         file;
//   STsdbFileInfo info;
//   int           fd;
// } SFile;

// typedef struct {
//   int   fileId;
//   int   state; // 0 for health, 1 for problem
//   SFile files[TSDB_FILE_TYPE_MAX];
// } SFileGroup;

// typedef struct {
//   pthread_rwlock_t fhlock;

//   int         maxFGroups;
//   int         nFGroups;
//   SFileGroup* pFGroup;
// } STsdbFileH;

// typedef struct {
//   int         direction;
//   STsdbFileH* pFileH;
//   int         fileId;
//   int         index;
// } SFileGroupIter;

// #define TSDB_FILE_NAME(pFile) ((pFile)->file.aname)
// #define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
// #define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
// #define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId
// #define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId
// #define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0)
// #define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
// #define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC

// STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg);
// void        tsdbFreeFileH(STsdbFileH* pFileH);
// int         tsdbOpenFileH(STsdbRepo* pRepo);
// void        tsdbCloseFileH(STsdbRepo* pRepo, bool isRestart);
// SFileGroup *tsdbCreateFGroup(STsdbRepo *pRepo, int fid, int level);
// void        tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction);
// void        tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid);
// SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter);
// int         tsdbOpenFile(SFile* pFile, int oflag);
// void        tsdbCloseFile(SFile* pFile);
// int         tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type);
// SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags);
// int         tsdbGetFidLevel(int fid, SFidGroup fidg);
// void        tsdbRemoveFilesBeyondRetention(STsdbRepo* pRepo, SFidGroup* pFidGroup);
// int         tsdbUpdateFileHeader(SFile* pFile);
// int         tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo);
// void*       tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo);
// void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
// int         tsdbLoadFileHeader(SFile* pFile, uint32_t* version);
// void        tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int64_t* size);
// void        tsdbGetFidGroup(STsdbCfg* pCfg, SFidGroup* pFidGroup);
// void        tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
// int         tsdbApplyRetention(STsdbRepo* pRepo, SFidGroup *pFidGroup);
H
Hongze Cheng 已提交
501

H
refact  
Hongze Cheng 已提交
502
// ================= tsdbMain.c
H
Hongze Cheng 已提交
503 504 505 506 507 508 509 510 511 512 513 514
typedef struct {
  int32_t  totalLen;
  int32_t  len;
  SDataRow row;
} SSubmitBlkIter;

typedef struct {
  int32_t totalLen;
  int32_t len;
  void *  pMsg;
} SSubmitMsgIter;

H
refact  
Hongze Cheng 已提交
515
struct STsdbRepo {
H
TD-353  
Hongze Cheng 已提交
516 517 518 519 520 521 522 523 524 525
  int8_t state;

  char*           rootDir;
  STsdbCfg        config;
  STsdbAppH       appH;
  STsdbStat       stat;
  STsdbMeta*      tsdbMeta;
  STsdbBufPool*   pPool;
  SMemTable*      mem;
  SMemTable*      imem;
H
Hongze Cheng 已提交
526
  STsdbFS*        fs;
H
Hongze Cheng 已提交
527
  sem_t           readyToCommit;
H
TD-353  
Hongze Cheng 已提交
528 529
  pthread_mutex_t mutex;
  bool            repoLocked;
H
Hongze Cheng 已提交
530
  int32_t         code; // Commit code
H
refact  
Hongze Cheng 已提交
531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
};

#define REPO_ID(r) (r)->config.tsdbId
#define IS_REPO_LOCKED(r) (r)->repoLocked
#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)

char*       tsdbGetMetaFileName(char* rootDir);
void        tsdbGetDataFileName(char* rootDir, int vid, int fid, int type, char* fname);
int         tsdbLockRepo(STsdbRepo* pRepo);
int         tsdbUnlockRepo(STsdbRepo* pRepo);
char*       tsdbGetDataDirName(char* rootDir);
int         tsdbGetNextMaxTables(int tid);
STsdbMeta*  tsdbGetMeta(TSDB_REPO_T* pRepo);
STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo);
int         tsdbCheckCommit(STsdbRepo* pRepo);

static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
  ASSERT(pRepo != NULL);
  if (pRepo->mem == NULL) return NULL;

  SListNode* pNode = listTail(pRepo->mem->bufBlockList);
  if (pNode == NULL) return NULL;

  STsdbBufBlock* pBufBlock = NULL;
  tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock));

  return pBufBlock;
}
H
TD-353  
Hongze Cheng 已提交
559

H
refact  
Hongze Cheng 已提交
560
// ================= tsdbRWHelper.c
H
TD-34  
hzcheng 已提交
561
typedef struct {
H
Hongze Cheng 已提交
562
  int32_t  tid;
H
TD-185  
Hongze Cheng 已提交
563 564 565 566
  uint32_t len;
  uint32_t offset;
  uint32_t hasLast : 2;
  uint32_t numOfBlocks : 30;
567 568
  uint64_t uid;
  TSKEY    maxKey;
H
refact  
Hongze Cheng 已提交
569
} SBlockIdx;
H
TD-353  
Hongze Cheng 已提交
570

571
typedef struct {
H
TD-353  
Hongze Cheng 已提交
572 573 574 575 576
  int64_t last : 1;
  int64_t offset : 63;
  int32_t algorithm : 8;
  int32_t numOfRows : 24;
  int32_t len;
H
refact  
Hongze Cheng 已提交
577
  int32_t keyLen;     // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
H
TD-353  
Hongze Cheng 已提交
578
  int16_t numOfSubBlocks;
H
Hongze Cheng 已提交
579
  int16_t numOfCols; // not including timestamp column
580 581
  TSKEY   keyFirst;
  TSKEY   keyLast;
H
refact  
Hongze Cheng 已提交
582
} SBlock;
583

H
TD-34  
hzcheng 已提交
584 585
typedef struct {
  int32_t    delimiter;  // For recovery usage
586
  int32_t    tid;
587
  uint64_t   uid;
H
refact  
Hongze Cheng 已提交
588 589
  SBlock blocks[];
} SBlockInfo;
H
hzcheng 已提交
590

H
TD-34  
hzcheng 已提交
591
typedef struct {
H
TD-353  
Hongze Cheng 已提交
592
  int16_t colId;
H
TD-541  
Hongze Cheng 已提交
593
  int32_t len;
H
TD-34  
hzcheng 已提交
594 595
  int32_t type : 8;
  int32_t offset : 24;
H
TD-321  
Hongze Cheng 已提交
596 597 598 599 600 601 602
  int64_t sum;
  int64_t max;
  int64_t min;
  int16_t maxIndex;
  int16_t minIndex;
  int16_t numOfNull;
  char    padding[2];
H
refact  
Hongze Cheng 已提交
603
} SBlockCol;
H
TD-34  
hzcheng 已提交
604 605 606 607

typedef struct {
  int32_t  delimiter;  // For recovery usage
  int32_t  numOfCols;  // For recovery usage
608
  uint64_t uid;        // For recovery usage
H
refact  
Hongze Cheng 已提交
609 610
  SBlockCol cols[];
} SBlockData;
H
TD-34  
hzcheng 已提交
611

H
hzcheng 已提交
612 613 614
typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;

typedef struct {
H
Hongze Cheng 已提交
615 616 617 618 619
  TSKEY      minKey;
  TSKEY      maxKey;
  SFileGroup fGroup;
  SFile      nHeadF;
  SFile      nLastF;
H
hzcheng 已提交
620 621 622
} SHelperFile;

typedef struct {
623 624
  uint64_t uid;
  int32_t  tid;
H
hzcheng 已提交
625 626
} SHelperTable;

H
Hongze Cheng 已提交
627
typedef struct {
H
refact  
Hongze Cheng 已提交
628
  SBlockIdx* pIdxArray;
H
Hongze Cheng 已提交
629 630 631 632
  int       numOfIdx;
  int       curIdx;
} SIdxH;

H
hzcheng 已提交
633
typedef struct {
H
TD-353  
Hongze Cheng 已提交
634
  tsdb_rw_helper_t type;
H
TD-100  
hzcheng 已提交
635

H
TD-353  
Hongze Cheng 已提交
636 637
  STsdbRepo* pRepo;
  int8_t     state;
H
TD-100  
hzcheng 已提交
638
  // For file set usage
H
hzcheng 已提交
639
  SHelperFile files;
H
Hongze Cheng 已提交
640
  SIdxH       idxH;
H
refact  
Hongze Cheng 已提交
641
  SBlockIdx    curCompIdx;
H
Hongze Cheng 已提交
642
  void*       pWIdx;
H
TD-100  
hzcheng 已提交
643
  // For table set usage
H
hzcheng 已提交
644
  SHelperTable tableInfo;
H
refact  
Hongze Cheng 已提交
645
  SBlockInfo*   pCompInfo;
H
TD-100  
hzcheng 已提交
646 647
  bool         hasOldLastBlock;
  // For block set usage
H
refact  
Hongze Cheng 已提交
648
  SBlockData* pCompData;
H
TD-353  
Hongze Cheng 已提交
649 650 651
  SDataCols* pDataCols[2];
  void*      pBuffer;     // Buffer to hold the whole data block
  void*      compBuffer;  // Buffer for temperary compress/decompress purpose
H
hzcheng 已提交
652 653
} SRWHelper;

H
TD-353  
Hongze Cheng 已提交
654 655
#define TSDB_HELPER_CLEAR_STATE 0x0        // Clear state
#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1  // File is set
H
refact  
Hongze Cheng 已提交
656
#define TSDB_HELPER_IDX_LOAD 0x2           // SBlockIdx part is loaded
H
TD-353  
Hongze Cheng 已提交
657
#define TSDB_HELPER_TABLE_SET 0x4          // Table is set
H
refact  
Hongze Cheng 已提交
658 659
#define TSDB_HELPER_INFO_LOAD 0x8          // SBlockInfo part is loaded
#define TSDB_HELPER_FILE_DATA_LOAD 0x10    // SBlockData part is loaded
H
TD-353  
Hongze Cheng 已提交
660 661 662 663
#define helperSetState(h, s) (((h)->state) |= (s))
#define helperClearState(h, s) ((h)->state &= (~(s)))
#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
H
TD-353  
Hongze Cheng 已提交
664 665
#define TSDB_MAX_SUBBLOCKS 8
#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
H
TD-353  
Hongze Cheng 已提交
666 667 668
#define helperType(h) (h)->type
#define helperRepo(h) (h)->pRepo
#define helperState(h) (h)->state
H
Hongze Cheng 已提交
669
#define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0)
H
Hongze Cheng 已提交
670 671 672 673 674 675
#define helperFileId(h) ((h)->files.fGroup.fileId)
#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD]))
#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA]))
#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST]))
#define helperNewHeadF(h) (&((h)->files.nHeadF))
#define helperNewLastF(h) (&((h)->files.nLastF))
H
Hongze Cheng 已提交
676 677 678 679 680 681

int  tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
int  tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
void tsdbDestroyHelper(SRWHelper* pHelper);
void tsdbResetHelper(SRWHelper* pHelper);
int  tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
H
Hongze Cheng 已提交
682
int  tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup);
H
Hongze Cheng 已提交
683
int  tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
H
Hongze Cheng 已提交
684 685 686 687
int  tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey);
int  tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
int  tsdbWriteCompInfo(SRWHelper* pHelper);
int  tsdbWriteCompIdx(SRWHelper* pHelper);
H
Hongze Cheng 已提交
688
int  tsdbLoadCompIdxImpl(SFile* pFile, uint32_t offset, uint32_t len, void* buffer);
H
refact  
Hongze Cheng 已提交
689
int  tsdbDecodeSBlockIdxImpl(void* buffer, uint32_t len, SBlockIdx** ppCompIdx, int* numOfIdx);
H
Hongze Cheng 已提交
690
int  tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
H
refact  
Hongze Cheng 已提交
691
int  tsdbLoadCompInfoImpl(SFile* pFile, SBlockIdx* pIdx, SBlockInfo** ppCompInfo);
H
Hongze Cheng 已提交
692
int  tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
H
refact  
Hongze Cheng 已提交
693
int  tsdbLoadCompData(SRWHelper* phelper, SBlock* pcompblock, void* target);
H
Hongze Cheng 已提交
694
void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
H
refact  
Hongze Cheng 已提交
695
int  tsdbLoadBlockDataCols(SRWHelper* pHelper, SBlock* pCompBlock, SBlockInfo* pCompInfo, int16_t* colIds,
H
Hongze Cheng 已提交
696
                           int numOfColIds);
H
refact  
Hongze Cheng 已提交
697
int  tsdbLoadBlockData(SRWHelper* pHelper, SBlock* pCompBlock, SBlockInfo* pCompInfo);
H
Hongze Cheng 已提交
698 699 700 701 702 703 704 705 706 707

static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) {
  if (*(TSKEY*)key1 > *(TSKEY*)key2) {
    return 1;
  } else if (*(TSKEY*)key1 == *(TSKEY*)key2) {
    return 0;
  } else {
    return -1;
  }
}
H
TD-353  
Hongze Cheng 已提交
708

H
refact  
Hongze Cheng 已提交
709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
// ================= tsdbScan.c
typedef struct {
  SFileGroup  fGroup;
  int         numOfIdx;
  SBlockIdx*  pCompIdx;
  SBlockInfo* pCompInfo;
  void*       pBuf;
  FILE*       tLogStream;
} STsdbScanHandle;

int              tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid);
STsdbScanHandle* tsdbNewScanHandle();
void             tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream);
int              tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid);
int              tsdbScanSBlockIdx(STsdbScanHandle* pScanHandle);
int              tsdbScanSBlock(STsdbScanHandle* pScanHandle, int idx);
int              tsdbCloseScanFile(STsdbScanHandle* pScanHandle);
void             tsdbFreeScanHandle(STsdbScanHandle* pScanHandle);

H
Hongze Cheng 已提交
728 729 730
// ------------------ tsdbCommitQueue.c
int tsdbScheduleCommit(STsdbRepo *pRepo);

H
more  
hzcheng 已提交
731 732 733 734
#ifdef __cplusplus
}
#endif

H
TD-34  
hzcheng 已提交
735
#endif