tsdbFile.h 11.8 KB
Newer Older
H
refact  
Hongze Cheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef _TS_TSDB_FILE_H_
#define _TS_TSDB_FILE_H_

#define TSDB_FILE_HEAD_SIZE 512
#define TSDB_FILE_DELIMITER 0xF00AFA0F
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
H
Hongze Cheng 已提交
22
#define TSDB_IVLD_FID INT_MIN
H
Hongze Cheng 已提交
23 24
#define TSDB_FILE_STATE_OK 0
#define TSDB_FILE_STATE_BAD 1
H
refact  
Hongze Cheng 已提交
25

H
refact  
Hongze Cheng 已提交
26 27 28
#define TSDB_FILE_INFO(tf) (&((tf)->info))
#define TSDB_FILE_F(tf) (&((tf)->f))
#define TSDB_FILE_FD(tf) ((tf)->fd)
H
Hongze Cheng 已提交
29 30
#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf))
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
H
Hongze Cheng 已提交
31
#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
H
refact  
Hongze Cheng 已提交
32
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
H
Hongze Cheng 已提交
33 34
#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
S
TD-4088  
Shengliang Guan 已提交
35
#define TSDB_FILE_FSYNC(tf) taosFsync(TSDB_FILE_FD(tf))
H
Hongze Cheng 已提交
36 37 38 39
#define TSDB_FILE_STATE(tf) ((tf)->state)
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
C
Cary Xu 已提交
40
#define TSDB_FSET_NFILES_VALID(s) (((s)->nFiles >= TSDB_FILE_MIN) && ((s)->nFiles <= TSDB_FILE_MAX))
41
typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_SMA, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T;
H
refact  
Hongze Cheng 已提交
42

C
Cary Xu 已提交
43 44
#define TSDB_FILE_MIN 3U  // min number of files in one DFileSet

H
refact  
Hongze Cheng 已提交
45
// =============== SMFile
H
refact  
Hongze Cheng 已提交
46 47 48 49 50 51 52 53 54 55 56 57
typedef struct {
  int64_t  size;
  int64_t  tombSize;
  int64_t  nRecords;
  int64_t  nDels;
  uint32_t magic;
} SMFInfo;

typedef struct {
  SMFInfo info;
  TFILE   f;
  int     fd;
H
Hongze Cheng 已提交
58
  uint8_t state;
H
refact  
Hongze Cheng 已提交
59 60
} SMFile;

H
Hongze Cheng 已提交
61
void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
H
Hongze Cheng 已提交
62
void  tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile);
H
refact  
Hongze Cheng 已提交
63 64
int   tsdbEncodeSMFile(void** buf, SMFile* pMFile);
void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
H
Hongze Cheng 已提交
65 66
int   tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile);
H
Hongze Cheng 已提交
67
int   tsdbApplyMFileChange(SMFile* from, SMFile* to);
H
Hongze Cheng 已提交
68
int   tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
H
Hongze Cheng 已提交
69
int   tsdbUpdateMFileHeader(SMFile* pMFile);
H
Hongze Cheng 已提交
70
int   tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
H
Hongze Cheng 已提交
71
int   tsdbScanAndTryFixMFile(STsdbRepo* pRepo);
H
Hongze Cheng 已提交
72 73
int   tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
H
Hongze Cheng 已提交
74

H
Hongze Cheng 已提交
75
static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; }
H
refact  
Hongze Cheng 已提交
76

H
refact  
Hongze Cheng 已提交
77
static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) {
H
Hongze Cheng 已提交
78
  ASSERT(TSDB_FILE_CLOSED(pMFile));
H
refact  
Hongze Cheng 已提交
79 80 81 82 83 84 85 86 87 88

  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags);
  if (pMFile->fd < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  return 0;
}

H
refact  
Hongze Cheng 已提交
89
static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
H
refact  
Hongze Cheng 已提交
90 91 92 93 94 95
  if (TSDB_FILE_OPENED(pMFile)) {
    close(pMFile->fd);
    TSDB_FILE_SET_CLOSED(pMFile);
  }
}

H
refact  
Hongze Cheng 已提交
96
static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
H
refact  
Hongze Cheng 已提交
97 98 99 100 101 102 103 104 105 106
  ASSERT(TSDB_FILE_OPENED(pMFile));

  int64_t loffset = taosLSeek(TSDB_FILE_FD(pMFile), offset, whence);
  if (loffset < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  return loffset;
}
H
refact  
Hongze Cheng 已提交
107

H
refact  
Hongze Cheng 已提交
108
static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
H
refact  
Hongze Cheng 已提交
109 110 111 112 113 114 115 116 117 118 119
  ASSERT(TSDB_FILE_OPENED(pMFile));

  int64_t nwrite = taosWrite(pMFile->fd, buf, nbyte);
  if (nwrite < nbyte) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  return nwrite;
}

H
refact  
Hongze Cheng 已提交
120
static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) {
H
refact  
Hongze Cheng 已提交
121 122 123
  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM));
}

H
Hongze Cheng 已提交
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) {
  ASSERT(TSDB_FILE_OPENED(pMFile));

  int64_t toffset;

  if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) {
    return -1;
  }

  ASSERT(pMFile->info.size == toffset);

  if (offset) {
    *offset = toffset;
  }

  if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) {
    return -1;
  }

  pMFile->info.size += nbyte;

S
TD-1207  
Shengliang Guan 已提交
145
  return (int)nbyte;
H
Hongze Cheng 已提交
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
}

static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); }

static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
  ASSERT(TSDB_FILE_OPENED(pMFile));

  int64_t nread = taosRead(pMFile->fd, buf, nbyte);
  if (nread < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  return nread;
}
H
refact  
Hongze Cheng 已提交
161 162

// =============== SDFile
H
refact  
Hongze Cheng 已提交
163 164 165 166 167 168 169 170
typedef struct {
  uint32_t magic;
  uint32_t len;
  uint32_t totalBlocks;
  uint32_t totalSubBlocks;
  uint32_t offset;
  uint64_t size;
  uint64_t tombSize;
C
Cary Xu 已提交
171
  uint32_t fver;
H
refact  
Hongze Cheng 已提交
172 173 174 175 176 177
} SDFInfo;

typedef struct {
  SDFInfo info;
  TFILE   f;
  int     fd;
H
Hongze Cheng 已提交
178
  uint8_t state;
H
refact  
Hongze Cheng 已提交
179 180
} SDFile;

H
Hongze Cheng 已提交
181 182
void  tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype);
void  tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
H
refact  
Hongze Cheng 已提交
183 184
int   tsdbEncodeSDFile(void** buf, SDFile* pDFile);
void* tsdbDecodeSDFile(void* buf, SDFile* pDFile);
C
Cary Xu 已提交
185
int   tsdbCreateDFile(SDFile* pDFile, bool updateHeader, TSDB_FILE_T ftype);
H
Hongze Cheng 已提交
186
int   tsdbUpdateDFileHeader(SDFile* pDFile);
C
Cary Xu 已提交
187
int   tsdbUpdateDFileHeaderEx(SDFile* pDFile, uint32_t fver);
H
Hongze Cheng 已提交
188 189
int   tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
int   tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
H
Hongze Cheng 已提交
190 191

static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
H
refact  
Hongze Cheng 已提交
192

H
Hongze Cheng 已提交
193
static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
H
refact  
Hongze Cheng 已提交
194 195
  ASSERT(!TSDB_FILE_OPENED(pDFile));

H
Hongze Cheng 已提交
196
  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags);
H
refact  
Hongze Cheng 已提交
197 198 199 200 201 202 203 204
  if (pDFile->fd < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  return 0;
}

H
refact  
Hongze Cheng 已提交
205
static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
H
refact  
Hongze Cheng 已提交
206 207 208 209 210 211
  if (TSDB_FILE_OPENED(pDFile)) {
    close(pDFile->fd);
    TSDB_FILE_SET_CLOSED(pDFile);
  }
}

H
Hongze Cheng 已提交
212
static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
H
refact  
Hongze Cheng 已提交
213 214
  ASSERT(TSDB_FILE_OPENED(pDFile));

H
Hongze Cheng 已提交
215
  int64_t loffset = taosLSeek(TSDB_FILE_FD(pDFile), offset, whence);
H
refact  
Hongze Cheng 已提交
216 217 218 219 220 221 222 223
  if (loffset < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  return loffset;
}

H
refact  
Hongze Cheng 已提交
224
static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
H
refact  
Hongze Cheng 已提交
225 226 227 228 229 230 231 232 233 234 235
  ASSERT(TSDB_FILE_OPENED(pDFile));

  int64_t nwrite = taosWrite(pDFile->fd, buf, nbyte);
  if (nwrite < nbyte) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  return nwrite;
}

H
Hongze Cheng 已提交
236 237 238 239 240
static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
}

static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
H
refact  
Hongze Cheng 已提交
241 242
  ASSERT(TSDB_FILE_OPENED(pDFile));

H
Hongze Cheng 已提交
243
  int64_t toffset;
H
refact  
Hongze Cheng 已提交
244

H
Hongze Cheng 已提交
245 246 247
  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
    return -1;
  }
H
refact  
Hongze Cheng 已提交
248

H
Hongze Cheng 已提交
249 250 251 252 253 254 255 256 257 258 259 260
  ASSERT(pDFile->info.size == toffset);

  if (offset) {
    *offset = toffset;
  }

  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
    return -1;
  }

  pDFile->info.size += nbyte;

S
TD-1207  
Shengliang Guan 已提交
261
  return (int)nbyte;
H
refact  
Hongze Cheng 已提交
262 263
}

H
Hongze Cheng 已提交
264 265
static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); }

H
refact  
Hongze Cheng 已提交
266
static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
H
refact  
Hongze Cheng 已提交
267 268 269 270 271 272 273 274 275 276 277
  ASSERT(TSDB_FILE_OPENED(pDFile));

  int64_t nread = taosRead(pDFile->fd, buf, nbyte);
  if (nread < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

  return nread;
}

H
Hongze Cheng 已提交
278 279 280
static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
  if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
H
Hongze Cheng 已提交
281 282 283
    return -1;
  }

H
Hongze Cheng 已提交
284
  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
H
Hongze Cheng 已提交
285 286 287
  return 0;
}

H
refact  
Hongze Cheng 已提交
288
// =============== SDFileSet
H
refact  
Hongze Cheng 已提交
289
typedef struct {
C
Cary Xu 已提交
290 291 292 293
  int     fid;
  int     state;
  uint8_t nFiles;
  SDFile  files[TSDB_FILE_MAX];
H
refact  
Hongze Cheng 已提交
294 295
} SDFileSet;

H
refact  
Hongze Cheng 已提交
296
#define TSDB_FSET_FID(s) ((s)->fid)
H
refact  
Hongze Cheng 已提交
297
#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
H
Hongze Cheng 已提交
298 299
#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
C
Cary Xu 已提交
300
#define TSDB_FSET_SET_INIT(s)                                                  \
H
Hongze Cheng 已提交
301 302 303 304 305
  do {                                                                         \
    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
    }                                                                          \
  } while (0);
C
Cary Xu 已提交
306 307 308 309 310 311 312 313 314 315 316
#define TSDB_FSET_SET_CLOSED(s)                                              \
  do {                                                                       \
    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < (s)->nFiles; ftype++) { \
      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                     \
    }                                                                        \
  } while (0);
#define TSDB_FSET_FSYNC(s)                                                   \
  do {                                                                       \
    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < (s)->nFiles; ftype++) { \
      TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                          \
    }                                                                        \
H
Hongze Cheng 已提交
317
  } while (0);
H
refact  
Hongze Cheng 已提交
318

H
Hongze Cheng 已提交
319
void  tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver);
H
Hongze Cheng 已提交
320 321
void  tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
int   tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
C
Cary Xu 已提交
322
void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet, bool containNFiles);
H
Hongze Cheng 已提交
323 324
int   tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
H
Hongze Cheng 已提交
325
int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
H
Hongze Cheng 已提交
326
int   tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader);
H
Hongze Cheng 已提交
327
int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
H
Hongze Cheng 已提交
328
int   tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet* pSet);
H
Hongze Cheng 已提交
329 330

static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
C
Cary Xu 已提交
331 332
  ASSERT(pSet->nFiles <= TSDB_FILE_MAX);
  for (TSDB_FILE_T ftype = 0; ftype < pSet->nFiles; ftype++) {
H
Hongze Cheng 已提交
333 334 335 336 337
    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
  }
}

static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
C
Cary Xu 已提交
338 339
  ASSERT(TSDB_FSET_NFILES_VALID(pSet));
  for (TSDB_FILE_T ftype = 0; ftype < pSet->nFiles; ftype++) {
H
Hongze Cheng 已提交
340 341 342 343 344 345 346 347 348
    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
      tsdbCloseDFileSet(pSet);
      return -1;
    }
  }
  return 0;
}

static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
C
Cary Xu 已提交
349 350
  ASSERT(TSDB_FSET_NFILES_VALID(pSet));
  for (TSDB_FILE_T ftype = 0; ftype < pSet->nFiles; ftype++) {
351
    (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
H
Hongze Cheng 已提交
352 353 354 355
  }
}

static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
C
Cary Xu 已提交
356 357
  ASSERT(TSDB_FSET_NFILES_VALID(pSrc));
  for (TSDB_FILE_T ftype = 0; ftype < pSrc->nFiles; ftype++) {
H
Hongze Cheng 已提交
358 359 360 361 362 363 364 365
    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
      tsdbRemoveDFileSet(pDest);
      return -1;
    }
  }

  return 0;
}
H
refact  
Hongze Cheng 已提交
366

H
refact  
Hongze Cheng 已提交
367
static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
368 369
  *minKey = fid * days * tsTickPerDay[precision];
  *maxKey = *minKey + days * tsTickPerDay[precision] - 1;
H
refact  
Hongze Cheng 已提交
370 371
}

H
Hongze Cheng 已提交
372 373 374 375 376 377 378 379 380 381
static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
      return false;
    }
  }

  return true;
}

H
refact  
Hongze Cheng 已提交
382
#endif /* _TS_TSDB_FILE_H_ */