tsdbMain.h 16.2 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
TD-34  
hzcheng 已提交
15 16
#ifndef _TD_TSDB_MAIN_H_
#define _TD_TSDB_MAIN_H_
H
more  
Hongze Cheng 已提交
17

H
TD-353  
Hongze Cheng 已提交
18 19
#include "hash.h"
#include "tcoding.h"
S
slguan 已提交
20
#include "tglobal.h"
H
TD-353  
Hongze Cheng 已提交
21
#include "tkvstore.h"
H
hzcheng 已提交
22
#include "tlist.h"
H
TD-353  
Hongze Cheng 已提交
23
#include "tlog.h"
B
Bomin Zhang 已提交
24
#include "tlockfree.h"
H
hzcheng 已提交
25
#include "tsdb.h"
H
TD-34  
hzcheng 已提交
26 27
#include "tskiplist.h"
#include "tutil.h"
H
more  
Hongze Cheng 已提交
28

H
more  
hzcheng 已提交
29 30 31 32
#ifdef __cplusplus
extern "C" {
#endif

H
hzcheng 已提交
33 34
extern int tsdbDebugFlag;

S
Shengliang Guan 已提交
35 36
#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}
#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}
S
Shengliang Guan 已提交
37 38 39 40
#define tsdbWarn(...)  { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}
#define tsdbInfo(...)  { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}
#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
H
hzcheng 已提交
41

H
TD-90  
Hongze Cheng 已提交
42
#define TSDB_MAX_TABLE_SCHEMAS 16
H
TD-353  
Hongze Cheng 已提交
43 44
#define TSDB_FILE_HEAD_SIZE 512
#define TSDB_FILE_DELIMITER 0xF00AFA0F
45
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
H
TD-353  
Hongze Cheng 已提交
46 47 48

// Definitions
// ------------------ tsdbMeta.c
H
TD-34  
hzcheng 已提交
49
typedef struct STable {
H
Haojun Liao 已提交
50
  STableId       tableId;
H
TD-353  
Hongze Cheng 已提交
51 52 53 54 55
  ETableType     type;
  tstr*          name;  // NOTE: there a flexible string here
  uint64_t       suid;
  struct STable* pSuper;  // super table pointer
  uint8_t        numOfSchemas;
H
TD-353  
Hongze Cheng 已提交
56
  STSchema*      schema[TSDB_MAX_TABLE_SCHEMAS];
H
TD-353  
Hongze Cheng 已提交
57 58
  STSchema*      tagSchema;
  SKVRow         tagVal;
H
TD-353  
Hongze Cheng 已提交
59
  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
H
TD-353  
Hongze Cheng 已提交
60 61 62 63 64
  void*          eventHandler;   // TODO
  void*          streamHandler;  // TODO
  TSKEY          lastKey;        // lastkey inserted in this table, initialized as 0, TODO: make a structure
  char*          sql;
  void*          cqhandle;
H
Hongze Cheng 已提交
65
  SRWLatch       latch;  // TODO: implementa latch functions
H
TD-353  
Hongze Cheng 已提交
66
  T_REF_DECLARE();
H
TD-34  
hzcheng 已提交
67 68 69
} STable;

typedef struct {
H
TD-353  
Hongze Cheng 已提交
70 71 72
  pthread_rwlock_t rwLock;

  int32_t   nTables;
H
TD-987  
Hongze Cheng 已提交
73
  int32_t   maxTables;
H
TD-353  
Hongze Cheng 已提交
74 75 76 77
  STable**  tables;
  SList*    superList;
  SHashObj* uidMap;
  SKVStore* pStore;
H
TD-353  
Hongze Cheng 已提交
78 79
  int       maxRowBytes;
  int       maxCols;
H
TD-34  
hzcheng 已提交
80 81
} STsdbMeta;

H
TD-353  
Hongze Cheng 已提交
82
// ------------------ tsdbBuffer.c
H
TD-34  
hzcheng 已提交
83
typedef struct {
H
TD-353  
Hongze Cheng 已提交
84 85 86 87 88
  int64_t blockId;
  int     offset;
  int     remain;
  char    data[];
} STsdbBufBlock;
H
TD-34  
hzcheng 已提交
89 90

typedef struct {
H
TD-353  
Hongze Cheng 已提交
91 92 93 94 95 96 97 98 99
  pthread_cond_t poolNotEmpty;
  int            bufBlockSize;
  int            tBufBlocks;
  int            nBufBlocks;
  int64_t        index;
  SList*         bufBlockList;
} STsdbBufPool;

// ------------------ tsdbMemTable.c
H
Hongze Cheng 已提交
100 101 102 103 104
typedef struct {
  STable *           pTable;
  SSkipListIterator *pIter;
} SCommitIter;

H
TD-34  
hzcheng 已提交
105
typedef struct {
H
TD-353  
Hongze Cheng 已提交
106 107 108 109 110 111
  uint64_t   uid;
  TSKEY      keyFirst;
  TSKEY      keyLast;
  int64_t    numOfRows;
  SSkipList* pData;
} STableData;
H
TD-34  
hzcheng 已提交
112 113

typedef struct {
H
TD-353  
Hongze Cheng 已提交
114
  T_REF_DECLARE();
H
TD-987  
Hongze Cheng 已提交
115
  SRWLatch     latch;
H
TD-353  
Hongze Cheng 已提交
116 117 118
  TSKEY        keyFirst;
  TSKEY        keyLast;
  int64_t      numOfRows;
H
TD-987  
Hongze Cheng 已提交
119
  int32_t      maxTables;
H
TD-353  
Hongze Cheng 已提交
120 121 122 123
  STableData** tData;
  SList*       actList;
  SList*       bufBlockList;
} SMemTable;
H
TD-34  
hzcheng 已提交
124

H
TD-353  
Hongze Cheng 已提交
125 126 127 128 129 130 131 132 133 134 135
enum { TSDB_UPDATE_META, TSDB_DROP_META };
typedef struct __attribute__((packed)){
  char     act;
  uint64_t uid;
} SActObj;

typedef struct {
  int  len;
  char cont[];
} SActCont;

H
TD-353  
Hongze Cheng 已提交
136
// ------------------ tsdbFile.c
H
TD-353  
Hongze Cheng 已提交
137
extern const char* tsdbFileSuffix[];
H
TD-353  
Hongze Cheng 已提交
138
typedef enum {
H
Hongze Cheng 已提交
139
#ifdef TSDB_IDX
H
Hongze Cheng 已提交
140 141
  TSDB_FILE_TYPE_IDX = 0,
  TSDB_FILE_TYPE_HEAD,
H
Hongze Cheng 已提交
142 143 144
#else
  TSDB_FILE_TYPE_HEAD = 0,
#endif
H
TD-353  
Hongze Cheng 已提交
145 146
  TSDB_FILE_TYPE_DATA,
  TSDB_FILE_TYPE_LAST,
H
TD-353  
Hongze Cheng 已提交
147
  TSDB_FILE_TYPE_MAX,
H
Hongze Cheng 已提交
148
#ifdef TSDB_IDX
H
Hongze Cheng 已提交
149
  TSDB_FILE_TYPE_NIDX,
H
Hongze Cheng 已提交
150
#endif
H
TD-353  
Hongze Cheng 已提交
151 152
  TSDB_FILE_TYPE_NHEAD,
  TSDB_FILE_TYPE_NLAST
H
TD-353  
Hongze Cheng 已提交
153
} TSDB_FILE_TYPE;
H
Hongze Cheng 已提交
154

H
more  
Hongze Cheng 已提交
155
typedef struct {
H
Hongze Cheng 已提交
156
  uint32_t magic;
H
TD-353  
Hongze Cheng 已提交
157 158 159
  uint32_t len;
  uint32_t totalBlocks;
  uint32_t totalSubBlocks;
H
Hongze Cheng 已提交
160
  uint32_t offset;
H
Hongze Cheng 已提交
161 162
  uint64_t size;      // total size of the file
  uint64_t tombSize;  // unused file size
163
} STsdbFileInfo;
H
TD-34  
hzcheng 已提交
164 165

typedef struct {
H
TD-353  
Hongze Cheng 已提交
166
  char  fname[TSDB_FILENAME_LEN];
H
TD-353  
Hongze Cheng 已提交
167 168
  int   fd;

H
TD-353  
Hongze Cheng 已提交
169
  STsdbFileInfo info;
H
hzcheng 已提交
170
} SFile;
H
more  
Hongze Cheng 已提交
171

H
hzcheng 已提交
172
typedef struct {
H
TD-353  
Hongze Cheng 已提交
173 174
  int   fileId;
  SFile files[TSDB_FILE_TYPE_MAX];
H
hzcheng 已提交
175 176 177
} SFileGroup;

typedef struct {
H
TD-353  
Hongze Cheng 已提交
178 179
  pthread_rwlock_t fhlock;

H
TD-353  
Hongze Cheng 已提交
180 181 182
  int         maxFGroups;
  int         nFGroups;
  SFileGroup* pFGroup;
H
hzcheng 已提交
183
} STsdbFileH;
H
more  
Hongze Cheng 已提交
184

H
TD-34  
hzcheng 已提交
185 186 187 188 189 190 191
typedef struct {
  int         numOfFGroups;
  SFileGroup *base;
  SFileGroup *pFileGroup;
  int         direction;
} SFileGroupIter;

H
TD-353  
Hongze Cheng 已提交
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
// ------------------ tsdbMain.c
typedef struct {
  int8_t state;

  char*           rootDir;
  STsdbCfg        config;
  STsdbAppH       appH;
  STsdbStat       stat;
  STsdbMeta*      tsdbMeta;
  STsdbBufPool*   pPool;
  SMemTable*      mem;
  SMemTable*      imem;
  STsdbFileH*     tsdbFileH;
  int             commit;
  pthread_t       commitThread;
  pthread_mutex_t mutex;
  bool            repoLocked;
} STsdbRepo;

H
TD-353  
Hongze Cheng 已提交
211
// ------------------ tsdbRWHelper.c
H
TD-34  
hzcheng 已提交
212
typedef struct {
H
Hongze Cheng 已提交
213
  int32_t  tid;
H
TD-185  
Hongze Cheng 已提交
214 215 216 217
  uint32_t len;
  uint32_t offset;
  uint32_t hasLast : 2;
  uint32_t numOfBlocks : 30;
218 219
  uint64_t uid;
  TSKEY    maxKey;
H
TD-353  
Hongze Cheng 已提交
220 221
} SCompIdx;

222
typedef struct {
H
TD-353  
Hongze Cheng 已提交
223 224 225 226 227
  int64_t last : 1;
  int64_t offset : 63;
  int32_t algorithm : 8;
  int32_t numOfRows : 24;
  int32_t len;
H
Hongze Cheng 已提交
228
  int32_t keyLen;     // key column length, keyOffset = offset+sizeof(SCompData)+sizeof(SCompCol)*numOfCols
H
TD-353  
Hongze Cheng 已提交
229
  int16_t numOfSubBlocks;
H
Hongze Cheng 已提交
230
  int16_t numOfCols; // not including timestamp column
231 232 233 234
  TSKEY   keyFirst;
  TSKEY   keyLast;
} SCompBlock;

H
TD-34  
hzcheng 已提交
235 236
typedef struct {
  int32_t    delimiter;  // For recovery usage
237
  int32_t    tid;
238
  uint64_t   uid;
H
TD-34  
hzcheng 已提交
239 240
  SCompBlock blocks[];
} SCompInfo;
H
hzcheng 已提交
241

H
TD-34  
hzcheng 已提交
242
typedef struct {
H
TD-353  
Hongze Cheng 已提交
243
  int16_t colId;
H
TD-541  
Hongze Cheng 已提交
244
  int32_t len;
H
TD-34  
hzcheng 已提交
245 246
  int32_t type : 8;
  int32_t offset : 24;
H
TD-321  
Hongze Cheng 已提交
247 248 249 250 251 252 253
  int64_t sum;
  int64_t max;
  int64_t min;
  int16_t maxIndex;
  int16_t minIndex;
  int16_t numOfNull;
  char    padding[2];
H
TD-34  
hzcheng 已提交
254 255 256 257 258
} SCompCol;

typedef struct {
  int32_t  delimiter;  // For recovery usage
  int32_t  numOfCols;  // For recovery usage
259
  uint64_t uid;        // For recovery usage
H
TD-34  
hzcheng 已提交
260 261
  SCompCol cols[];
} SCompData;
H
TD-34  
hzcheng 已提交
262

H
hzcheng 已提交
263 264 265
typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;

typedef struct {
H
Hongze Cheng 已提交
266 267 268
  TSKEY      minKey;
  TSKEY      maxKey;
  SFileGroup fGroup;
H
Hongze Cheng 已提交
269 270 271
#ifdef TSDB_IDX
  SFile nIdxF;
#endif
H
Hongze Cheng 已提交
272 273
  SFile      nHeadF;
  SFile      nLastF;
H
hzcheng 已提交
274 275 276
} SHelperFile;

typedef struct {
277 278
  uint64_t uid;
  int32_t  tid;
H
hzcheng 已提交
279 280
} SHelperTable;

H
Hongze Cheng 已提交
281 282 283 284 285 286
typedef struct {
  SCompIdx* pIdxArray;
  int       numOfIdx;
  int       curIdx;
} SIdxH;

H
hzcheng 已提交
287
typedef struct {
H
TD-353  
Hongze Cheng 已提交
288
  tsdb_rw_helper_t type;
H
TD-100  
hzcheng 已提交
289

H
TD-353  
Hongze Cheng 已提交
290 291
  STsdbRepo* pRepo;
  int8_t     state;
H
TD-100  
hzcheng 已提交
292
  // For file set usage
H
hzcheng 已提交
293
  SHelperFile files;
H
Hongze Cheng 已提交
294 295 296
  SIdxH       idxH;
  SCompIdx    curCompIdx;
  void*       pWIdx;
H
TD-100  
hzcheng 已提交
297
  // For table set usage
H
hzcheng 已提交
298
  SHelperTable tableInfo;
H
TD-353  
Hongze Cheng 已提交
299
  SCompInfo*   pCompInfo;
H
TD-100  
hzcheng 已提交
300 301
  bool         hasOldLastBlock;
  // For block set usage
H
TD-353  
Hongze Cheng 已提交
302
  SCompData* pCompData;
H
TD-353  
Hongze Cheng 已提交
303 304 305
  SDataCols* pDataCols[2];
  void*      pBuffer;     // Buffer to hold the whole data block
  void*      compBuffer;  // Buffer for temperary compress/decompress purpose
H
hzcheng 已提交
306 307
} SRWHelper;

H
TD-353  
Hongze Cheng 已提交
308 309
// Operations
// ------------------ tsdbMeta.c
H
TD-987  
Hongze Cheng 已提交
310
#define TSDB_INIT_NTABLES 1024
H
TD-353  
Hongze Cheng 已提交
311 312 313
#define TABLE_TYPE(t) (t)->type
#define TABLE_NAME(t) (t)->name
#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
H
TD-353  
Hongze Cheng 已提交
314
#define TABLE_UID(t) (t)->tableId.uid
H
TD-353  
Hongze Cheng 已提交
315
#define TABLE_TID(t) (t)->tableId.tid
H
TD-353  
Hongze Cheng 已提交
316
#define TABLE_SUID(t) (t)->suid
H
TD-353  
Hongze Cheng 已提交
317
#define TABLE_LASTKEY(t) (t)->lastKey
318
#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
H
TD-353  
Hongze Cheng 已提交
319 320 321

STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
void       tsdbFreeMeta(STsdbMeta* pMeta);
H
TD-353  
Hongze Cheng 已提交
322 323 324 325 326 327 328
int        tsdbOpenMeta(STsdbRepo* pRepo);
int        tsdbCloseMeta(STsdbRepo* pRepo);
STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
329 330
void       tsdbRefTable(STable* pTable);
void       tsdbUnRefTable(STable* pTable);
H
Hongze Cheng 已提交
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
void       tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);

static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
  if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
    return -1;
  } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
    return 1;
  } else {
    return 0;
  }
}

static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) {
  STable*   pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable;
  STSchema* pSchema = NULL;
  STSchema* pTSchema = NULL;

  if (lock) taosRLockLatch(&(pDTable->latch));
  if (version < 0) {  // get the latest version of schema
    pTSchema = pDTable->schema[pDTable->numOfSchemas - 1];
  } else {  // get the schema with version
    void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*),
                            tsdbCompareSchemaVersion, TD_EQ);
    if (ptr == NULL) {
      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
      goto _exit;
    }
    pTSchema = *(STSchema**)ptr;
  }

  ASSERT(pTSchema != NULL);

  if (copy) {
    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
  } else {
    pSchema = pTSchema;
  }

_exit:
  if (lock) taosRUnLockLatch(&(pDTable->latch));
  return pSchema;
}
H
TD-353  
Hongze Cheng 已提交
373

H
Hongze Cheng 已提交
374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
  return tsdbGetTableSchemaImpl(pTable, false, false, -1);
}

static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
    STable *pSuper = pTable->pSuper;
    if (pSuper == NULL) return NULL;
    return pSuper->tagSchema;
  } else if (pTable->type == TSDB_SUPER_TABLE) {
    return pTable->tagSchema;
  } else {
    return NULL;
  }
}

H
TD-353  
Hongze Cheng 已提交
390 391 392 393
// ------------------ tsdbBuffer.c
STsdbBufPool* tsdbNewBufPool();
void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
int           tsdbOpenBufPool(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
394
void          tsdbCloseBufPool(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
395 396 397
SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);

// ------------------ tsdbMemTable.c
H
TD-353  
Hongze Cheng 已提交
398 399 400 401
int   tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
H
TD-987  
Hongze Cheng 已提交
402
void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem);
H
TD-353  
Hongze Cheng 已提交
403
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
H
TD-353  
Hongze Cheng 已提交
404
int   tsdbAsyncCommit(STsdbRepo* pRepo);
H
Hongze Cheng 已提交
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
                            TSKEY* filterKeys, int nFilterKeys);

static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
  if (pIter == NULL) return NULL;

  SSkipListNode* node = tSkipListIterGet(pIter);
  if (node == NULL) return NULL;

  return SL_GET_NODE_DATA(node);
}

static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
  SDataRow row = tsdbNextIterRow(pIter);
  if (row == NULL) return -1;

  return dataRowKey(row);
}
H
TD-353  
Hongze Cheng 已提交
423 424 425 426 427 428

// ------------------ tsdbFile.c
#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId
#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId
H
TD-353  
Hongze Cheng 已提交
429
#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0)
H
TD-353  
Hongze Cheng 已提交
430 431 432
#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC

H
TD-353  
Hongze Cheng 已提交
433 434
STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg);
void        tsdbFreeFileH(STsdbFileH* pFileH);
H
TD-353  
Hongze Cheng 已提交
435
int         tsdbOpenFileH(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
436
void        tsdbCloseFileH(STsdbRepo* pRepo);
H
TD-987  
Hongze Cheng 已提交
437
SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid);
H
TD-353  
Hongze Cheng 已提交
438 439 440 441 442 443 444 445
void        tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction);
void        tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid);
SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter);
int         tsdbOpenFile(SFile* pFile, int oflag);
void        tsdbCloseFile(SFile* pFile);
int         tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type);
SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags);
void        tsdbFitRetention(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
446
int         tsdbUpdateFileHeader(SFile* pFile, uint32_t version);
H
TD-353  
Hongze Cheng 已提交
447
int         tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo);
H
TD-353  
Hongze Cheng 已提交
448
void*       tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo);
H
TD-353  
Hongze Cheng 已提交
449
void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
450
void        tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
H
TD-353  
Hongze Cheng 已提交
451 452

// ------------------ tsdbRWHelper.c
H
TD-353  
Hongze Cheng 已提交
453 454 455 456 457 458
#define TSDB_HELPER_CLEAR_STATE 0x0        // Clear state
#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1  // File is set
#define TSDB_HELPER_IDX_LOAD 0x2           // SCompIdx part is loaded
#define TSDB_HELPER_TABLE_SET 0x4          // Table is set
#define TSDB_HELPER_INFO_LOAD 0x8          // SCompInfo part is loaded
#define TSDB_HELPER_FILE_DATA_LOAD 0x10    // SCompData part is loaded
H
TD-353  
Hongze Cheng 已提交
459 460 461 462
#define helperSetState(h, s) (((h)->state) |= (s))
#define helperClearState(h, s) ((h)->state &= (~(s)))
#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
H
TD-353  
Hongze Cheng 已提交
463 464
#define TSDB_MAX_SUBBLOCKS 8
#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
H
TD-353  
Hongze Cheng 已提交
465 466 467
#define helperType(h) (h)->type
#define helperRepo(h) (h)->pRepo
#define helperState(h) (h)->state
H
Hongze Cheng 已提交
468
#define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0)
H
Hongze Cheng 已提交
469
#define helperFileId(h) ((h)->files.fGroup.fileId)
H
Hongze Cheng 已提交
470
#ifdef TSDB_IDX
H
Hongze Cheng 已提交
471
#define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX]))
H
Hongze Cheng 已提交
472 473
#define helperNewIdxF(h) (&((h)->files.nIdxF))
#endif
H
Hongze Cheng 已提交
474 475 476 477 478
#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD]))
#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA]))
#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST]))
#define helperNewHeadF(h) (&((h)->files.nHeadF))
#define helperNewLastF(h) (&((h)->files.nLastF))
H
Hongze Cheng 已提交
479 480 481 482 483 484 485

int  tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
int  tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
void tsdbDestroyHelper(SRWHelper* pHelper);
void tsdbResetHelper(SRWHelper* pHelper);
int  tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
int  tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError);
H
Hongze Cheng 已提交
486
int  tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
H
Hongze Cheng 已提交
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
int  tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey);
int  tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
int  tsdbWriteCompInfo(SRWHelper* pHelper);
int  tsdbWriteCompIdx(SRWHelper* pHelper);
int  tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
int  tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
int  tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target);
void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
int  tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds,
                           int numOfColIds);
int  tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo);

static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) {
  if (*(TSKEY*)key1 > *(TSKEY*)key2) {
    return 1;
  } else if (*(TSKEY*)key1 == *(TSKEY*)key2) {
    return 0;
  } else {
    return -1;
  }
}
H
TD-353  
Hongze Cheng 已提交
508

H
TD-353  
Hongze Cheng 已提交
509 510
// ------------------ tsdbMain.c
#define REPO_ID(r) (r)->config.tsdbId
H
TD-353  
Hongze Cheng 已提交
511
#define IS_REPO_LOCKED(r) (r)->repoLocked
H
TD-353  
Hongze Cheng 已提交
512
#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
H
TD-353  
Hongze Cheng 已提交
513

H
TD-353  
Hongze Cheng 已提交
514
char*       tsdbGetMetaFileName(char* rootDir);
H
TD-353  
Hongze Cheng 已提交
515
void        tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type, char* fname);
H
TD-353  
Hongze Cheng 已提交
516 517 518
int         tsdbLockRepo(STsdbRepo* pRepo);
int         tsdbUnlockRepo(STsdbRepo* pRepo);
char*       tsdbGetDataDirName(char* rootDir);
H
TD-987  
Hongze Cheng 已提交
519
int         tsdbGetNextMaxTables(int tid);
H
TD-353  
Hongze Cheng 已提交
520 521
STsdbMeta*  tsdbGetMeta(TSDB_REPO_T* pRepo);
STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo);
H
TD-353  
Hongze Cheng 已提交
522

H
more  
hzcheng 已提交
523 524 525 526
#ifdef __cplusplus
}
#endif

H
TD-34  
hzcheng 已提交
527
#endif