tsdbFile.c 19.5 KB
Newer Older
H
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
TD-353  
Hongze Cheng 已提交
15

H
Hongze Cheng 已提交
16
#include "tsdb.h"
H
TD-353  
Hongze Cheng 已提交
17

H
refact  
Hongze Cheng 已提交
18
static const char *TSDB_FNAME_SUFFIX[] = {
H
Hongze Cheng 已提交
19 20 21
    "head",  // TSDB_FILE_HEAD
    "data",  // TSDB_FILE_DATA
    "last",  // TSDB_FILE_LAST
22 23
    "smad",  // TSDB_FILE_SMAD
    "smal",  // TSDB_FILE_SMAL
H
Hongze Cheng 已提交
24 25
    "",      // TSDB_FILE_MAX
    "meta",  // TSDB_FILE_META
C
Cary Xu 已提交
26 27
    "tsma",  // TSDB_FILE_TSMA
    "rsma",  // TSDB_FILE_RSMA
H
refact  
Hongze Cheng 已提交
28
};
H
TD-353  
Hongze Cheng 已提交
29

H
Hongze Cheng 已提交
30 31
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);
// static int   tsdbRollBackMFile(SMFile *pMFile);
H
Hongze Cheng 已提交
32 33 34 35
static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
static int   tsdbRollBackDFile(SDFile *pDFile);

H
Hongze Cheng 已提交
36
#if 0
H
refact  
Hongze Cheng 已提交
37
// ============== SMFile
38
void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
H
refact  
Hongze Cheng 已提交
39 40
  char fname[TSDB_FILENAME_LEN];

H
Hongze Cheng 已提交
41
  TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK);
H
refact  
Hongze Cheng 已提交
42

H
Hongze Cheng 已提交
43 44
  memset(&(pMFile->info), 0, sizeof(pMFile->info));
  pMFile->info.magic = TSDB_FILE_INIT_MAGIC;
H
Hongze Cheng 已提交
45

46
  tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname);
H
Hongze Cheng 已提交
47 48 49
  tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
}

H
Hongze Cheng 已提交
50
void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) {
H
Hongze Cheng 已提交
51 52
  *pMFile = *pOMFile;
  TSDB_FILE_SET_CLOSED(pMFile);
H
Hongze Cheng 已提交
53
}
H
Hongze Cheng 已提交
54

H
refact  
Hongze Cheng 已提交
55
int tsdbEncodeSMFile(void **buf, SMFile *pMFile) {
H
Hongze Cheng 已提交
56
  int tlen = 0;
H
hzcheng 已提交
57

H
Hongze Cheng 已提交
58 59
  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
  tlen += tfsEncodeFile(buf, &(pMFile->f));
H
hzcheng 已提交
60

H
Hongze Cheng 已提交
61
  return tlen;
H
Hongze Cheng 已提交
62 63
}

H
refact  
Hongze Cheng 已提交
64
void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) {
H
Hongze Cheng 已提交
65 66
  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
  buf = tfsDecodeFile(buf, &(pMFile->f));
H
Hongze Cheng 已提交
67
  TSDB_FILE_SET_CLOSED(pMFile);
H
Hongze Cheng 已提交
68

H
Hongze Cheng 已提交
69 70
  return buf;
}
H
Hongze Cheng 已提交
71

H
Hongze Cheng 已提交
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) {
  int tlen = 0;

  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile));

  return tlen;
}

void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) {
  char *aname;
  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
  buf = taosDecodeString(buf, &aname);
  strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN);
  TSDB_FILE_SET_CLOSED(pMFile);

wafwerar's avatar
wafwerar 已提交
88
  taosMemoryFreeClear(aname);
H
Hongze Cheng 已提交
89 90 91 92

  return buf;
}

H
Hongze Cheng 已提交
93
int tsdbApplyMFileChange(SMFile *from, SMFile *to) {
H
Hongze Cheng 已提交
94
  if (from == NULL && to == NULL) return 0;
H
Hongze Cheng 已提交
95 96 97

  if (from != NULL) {
    if (to == NULL) {
H
Hongze Cheng 已提交
98
      return tsdbRemoveMFile(from);
H
Hongze Cheng 已提交
99 100 101
    } else {
      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
        if (from->info.size > to->info.size) {
H
Hongze Cheng 已提交
102
          tsdbRollBackMFile(to);
H
Hongze Cheng 已提交
103 104
        }
      } else {
H
Hongze Cheng 已提交
105
        return tsdbRemoveMFile(from);
H
Hongze Cheng 已提交
106 107 108 109 110 111 112
      }
    }
  }

  return 0;
}

H
Hongze Cheng 已提交
113
int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) {
H
Hongze Cheng 已提交
114 115
  ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC);

S
Shengliang Guan 已提交
116
  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
H
Hongze Cheng 已提交
117
  if (pMFile->fd < 0) {
H
Hongze Cheng 已提交
118 119 120 121
    if (errno == ENOENT) {
      // Try to create directory recursively
      char *s = strdup(TFILE_REL_NAME(&(pMFile->f)));
      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) {
wafwerar's avatar
wafwerar 已提交
122
        taosMemoryFreeClear(s);
H
Hongze Cheng 已提交
123 124
        return -1;
      }
wafwerar's avatar
wafwerar 已提交
125
      taosMemoryFreeClear(s);
H
Hongze Cheng 已提交
126 127 128 129 130 131 132 133 134 135

      pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
      if (pMFile->fd < 0) {
        terrno = TAOS_SYSTEM_ERROR(errno);
        return -1;
      }
    } else {
      terrno = TAOS_SYSTEM_ERROR(errno);
      return -1;
    }
H
Hongze Cheng 已提交
136 137
  }

H
Hongze Cheng 已提交
138 139 140 141
  if (!updateHeader) {
    return 0;
  }

H
Hongze Cheng 已提交
142 143
  pMFile->info.size += TSDB_FILE_HEAD_SIZE;

H
Hongze Cheng 已提交
144
  if (tsdbUpdateMFileHeader(pMFile) < 0) {
H
Hongze Cheng 已提交
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
    tsdbCloseMFile(pMFile);
    tsdbRemoveMFile(pMFile);
    return -1;
  }

  return 0;
}

int tsdbUpdateMFileHeader(SMFile *pMFile) {
  char buf[TSDB_FILE_HEAD_SIZE] = "\0";

  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
    return -1;
  }

  void *ptr = buf;
H
Hongze Cheng 已提交
161
  tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile));
H
Hongze Cheng 已提交
162

H
refact  
Hongze Cheng 已提交
163
  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
H
Hongze Cheng 已提交
164 165 166 167 168 169 170
  if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
    return -1;
  }

  return 0;
}

H
Hongze Cheng 已提交
171 172 173 174 175 176 177 178 179 180 181 182 183
int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) {
  char buf[TSDB_FILE_HEAD_SIZE] = "\0";

  ASSERT(TSDB_FILE_OPENED(pMFile));

  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
    return -1;
  }

  if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
    return -1;
  }

H
refact  
Hongze Cheng 已提交
184 185 186 187 188
  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
    return -1;
  }

H
Hongze Cheng 已提交
189 190 191 192
  tsdbDecodeMFInfo(buf, pInfo);
  return 0;
}

H
Hongze Cheng 已提交
193
int tsdbScanAndTryFixMFile(STsdb *pRepo) {
H
Hongze Cheng 已提交
194
  SMFile *    pMFile = pRepo->fs->cstatus->pmf;
H
Hongze Cheng 已提交
195
  struct stat mfstat;
H
Hongze Cheng 已提交
196 197 198
  SMFile      mf;

  if (pMFile == NULL) {
H
refact  
Hongze Cheng 已提交
199
    // No meta file, no need to scan
H
Hongze Cheng 已提交
200 201 202
    return 0;
  }

H
refact  
Hongze Cheng 已提交
203
  tsdbInitMFileEx(&mf, pMFile);
H
Hongze Cheng 已提交
204 205 206 207 208

  if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) {
    tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
              TSDB_FILE_FULL_NAME(pMFile));
    pRepo->state |= TSDB_STATE_BAD_META;
H
Hongze Cheng 已提交
209
    TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
H
Hongze Cheng 已提交
210 211
    return 0;
  }
H
Hongze Cheng 已提交
212 213 214 215 216 217

  if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

H
Hongze Cheng 已提交
218
  if (pMFile->info.size < mfstat.st_size) {
H
Hongze Cheng 已提交
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
    if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
      return -1;
    }

    if (taosFtruncate(mf.fd, mf.info.size) < 0) {
      terrno = TAOS_SYSTEM_ERROR(errno);
      tsdbCloseMFile(&mf);
      return -1;
    }

    if (tsdbUpdateMFileHeader(&mf) < 0) {
      tsdbCloseMFile(&mf);
      return -1;
    }

    tsdbCloseMFile(&mf);
H
Hongze Cheng 已提交
235 236
    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
             mfstat.st_size, pMFile->info.size);
H
Hongze Cheng 已提交
237
  } else if (pMFile->info.size > mfstat.st_size) {
H
Hongze Cheng 已提交
238 239 240
    tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size);
    pRepo->state |= TSDB_STATE_BAD_META;
H
Hongze Cheng 已提交
241
    TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
H
Hongze Cheng 已提交
242
    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
H
Hongze Cheng 已提交
243
    return 0;
H
refact  
Hongze Cheng 已提交
244 245
  } else {
    tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile));
H
Hongze Cheng 已提交
246 247 248 249 250
  }

  return 0;
}

H
Hongze Cheng 已提交
251
int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) {
H
Hongze Cheng 已提交
252
  int tlen = 0;
H
Hongze Cheng 已提交
253

H
Hongze Cheng 已提交
254 255 256 257 258
  tlen += taosEncodeVariantI64(buf, pInfo->size);
  tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
  tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
  tlen += taosEncodeVariantI64(buf, pInfo->nDels);
  tlen += taosEncodeFixedU32(buf, pInfo->magic);
H
Hongze Cheng 已提交
259

H
Hongze Cheng 已提交
260
  return tlen;
H
Hongze Cheng 已提交
261 262
}

H
Hongze Cheng 已提交
263
void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) {
H
Hongze Cheng 已提交
264 265 266 267 268 269 270
  buf = taosDecodeVariantI64(buf, &(pInfo->size));
  buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
  buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
  buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
  buf = taosDecodeFixedU32(buf, &(pInfo->magic));

  return buf;
H
Hongze Cheng 已提交
271 272
}

H
Hongze Cheng 已提交
273
static int tsdbRollBackMFile(SMFile *pMFile) {
H
refact  
Hongze Cheng 已提交
274 275 276
  SMFile mf;

  tsdbInitMFileEx(&mf, pMFile);
H
Hongze Cheng 已提交
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298

  if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
    return -1;
  }

  if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    tsdbCloseMFile(&mf);
    return -1;
  }

  if (tsdbUpdateMFileHeader(&mf) < 0) {
    tsdbCloseMFile(&mf);
    return -1;
  }

  TSDB_FILE_FSYNC(&mf);

  tsdbCloseMFile(&mf);
  return 0;
}

H
Hongze Cheng 已提交
299 300
#endif

H
Hongze Cheng 已提交
301
// ============== Operations on SDFile
S
Shengliang Guan 已提交
302
void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype) {
H
refact  
Hongze Cheng 已提交
303 304
  char fname[TSDB_FILENAME_LEN];

H
Hongze Cheng 已提交
305 306
  TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK);

H
Hongze Cheng 已提交
307
  TSDB_FILE_SET_CLOSED(pDFile);
H
refact  
Hongze Cheng 已提交
308

H
Hongze Cheng 已提交
309 310
  memset(&(pDFile->info), 0, sizeof(pDFile->info));
  pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
311
  pDFile->info.fver = tsdbGetDFSVersion(ftype);
H
refact  
Hongze Cheng 已提交
312

S
Shengliang Guan 已提交
313
  tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname);
H
Hongze Cheng 已提交
314
  tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
H
Hongze Cheng 已提交
315 316
}

H
Hongze Cheng 已提交
317 318
void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) {
  *pDFile = *pODFile;
H
Hongze Cheng 已提交
319 320 321
  TSDB_FILE_SET_CLOSED(pDFile);
}

H
refact  
Hongze Cheng 已提交
322
int tsdbEncodeSDFile(void **buf, SDFile *pDFile) {
H
Hongze Cheng 已提交
323
  int tlen = 0;
324

H
Hongze Cheng 已提交
325 326
  tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
  tlen += tfsEncodeFile(buf, &(pDFile->f));
H
TD-34  
hzcheng 已提交
327

H
Hongze Cheng 已提交
328
  return tlen;
H
TD-34  
hzcheng 已提交
329 330
}

S
Shengliang Guan 已提交
331
void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile) {
H
Hongze Cheng 已提交
332
  buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
H
Hongze Cheng 已提交
333
  buf = tfsDecodeFile(REPO_TFS(pRepo), buf, &(pDFile->f));
H
Hongze Cheng 已提交
334
  TSDB_FILE_SET_CLOSED(pDFile);
H
TD-353  
Hongze Cheng 已提交
335

H
Hongze Cheng 已提交
336
  return buf;
H
TD-353  
Hongze Cheng 已提交
337
}
H
TD-353  
Hongze Cheng 已提交
338

H
Hongze Cheng 已提交
339 340 341 342 343 344 345 346 347 348
static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) {
  int tlen = 0;

  tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pDFile));

  return tlen;
}

static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
349
  char *aname = NULL;
H
Hongze Cheng 已提交
350 351 352 353 354

  buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
  buf = taosDecodeString(buf, &aname);
  strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN);
  TSDB_FILE_SET_CLOSED(pDFile);
wafwerar's avatar
wafwerar 已提交
355
  taosMemoryFreeClear(aname);
H
Hongze Cheng 已提交
356 357 358 359

  return buf;
}

360
int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType) {
H
Hongze Cheng 已提交
361 362
  ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);

363
  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
364
  if (pDFile->pFile == NULL) {
H
Hongze Cheng 已提交
365 366
    if (errno == ENOENT) {
      // Try to create directory recursively
S
Shengliang Guan 已提交
367
      char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
H
Hongze Cheng 已提交
368
      if (tfsMkdirRecurAt(REPO_TFS(pRepo), taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) {
wafwerar's avatar
wafwerar 已提交
369
        taosMemoryFreeClear(s);
H
Hongze Cheng 已提交
370 371
        return -1;
      }
wafwerar's avatar
wafwerar 已提交
372
      taosMemoryFreeClear(s);
H
Hongze Cheng 已提交
373

374
      pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
375
      if (pDFile->pFile == NULL) {
H
Hongze Cheng 已提交
376 377 378 379 380 381 382
        terrno = TAOS_SYSTEM_ERROR(errno);
        return -1;
      }
    } else {
      terrno = TAOS_SYSTEM_ERROR(errno);
      return -1;
    }
H
Hongze Cheng 已提交
383 384
  }

H
Hongze Cheng 已提交
385 386 387 388
  if (!updateHeader) {
    return 0;
  }

H
Hongze Cheng 已提交
389
  pDFile->info.size += TSDB_FILE_HEAD_SIZE;
390
  pDFile->info.fver = tsdbGetDFSVersion(fType);
H
Hongze Cheng 已提交
391

H
Hongze Cheng 已提交
392
  if (tsdbUpdateDFileHeader(pDFile) < 0) {
H
Hongze Cheng 已提交
393 394 395 396 397
    tsdbCloseDFile(pDFile);
    tsdbRemoveDFile(pDFile);
    return -1;
  }

H
Hongze Cheng 已提交
398 399 400
  return 0;
}

H
Hongze Cheng 已提交
401 402 403 404 405 406
int tsdbUpdateDFileHeader(SDFile *pDFile) {
  char buf[TSDB_FILE_HEAD_SIZE] = "\0";

  if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
    return -1;
  }
H
Hongze Cheng 已提交
407

H
Hongze Cheng 已提交
408
  void *ptr = buf;
C
Cary Xu 已提交
409
  // taosEncodeFixedU32(&ptr, 0); // fver moved to SDFInfo and saved to current
H
Hongze Cheng 已提交
410
  tsdbEncodeDFInfo(&ptr, &(pDFile->info));
H
Hongze Cheng 已提交
411

H
refact  
Hongze Cheng 已提交
412
  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
H
Hongze Cheng 已提交
413
  if (tsdbWriteDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
H
Hongze Cheng 已提交
414 415
    return -1;
  }
H
Hongze Cheng 已提交
416 417

  return 0;
H
Hongze Cheng 已提交
418 419
}

H
Hongze Cheng 已提交
420
int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
H
Hongze Cheng 已提交
421
  char     buf[TSDB_FILE_HEAD_SIZE] = "\0";
422
  uint32_t _version;
H
Hongze Cheng 已提交
423 424 425 426 427 428 429 430 431 432 433

  ASSERT(TSDB_FILE_OPENED(pDFile));

  if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
    return -1;
  }

  if (tsdbReadDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
    return -1;
  }

H
refact  
Hongze Cheng 已提交
434 435 436 437 438
  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
    return -1;
  }

H
Hongze Cheng 已提交
439
  void *pBuf = buf;
C
Cary Xu 已提交
440
  // pBuf = taosDecodeFixedU32(pBuf, &_version);
H
Hongze Cheng 已提交
441
  pBuf = tsdbDecodeDFInfo(pBuf, pInfo);
H
Hongze Cheng 已提交
442 443 444
  return 0;
}

H
Hongze Cheng 已提交
445
static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
H
Hongze Cheng 已提交
446
  SDFile df;
H
refact  
Hongze Cheng 已提交
447 448

  tsdbInitDFileEx(&df, pDFile);
H
Hongze Cheng 已提交
449

450
  if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pDFile))) {
H
Hongze Cheng 已提交
451 452
    tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
              TSDB_FILE_FULL_NAME(pDFile));
H
Hongze Cheng 已提交
453
    // pRepo->state |= TSDB_STATE_BAD_DATA;
H
Hongze Cheng 已提交
454
    TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
H
Hongze Cheng 已提交
455 456
    return 0;
  }
457 458
  int64_t file_size = 0;
  if (taosStatFile(TSDB_FILE_FULL_NAME(&df), &file_size, NULL) < 0) {
H
Hongze Cheng 已提交
459 460 461 462
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

463
  if (pDFile->info.size < file_size) {
464 465
    // if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
    if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) {
H
Hongze Cheng 已提交
466 467 468
      return -1;
    }

469
    if (taosFtruncateFile(df.pFile, df.info.size) < 0) {
H
Hongze Cheng 已提交
470 471 472 473 474 475 476 477 478 479 480
      terrno = TAOS_SYSTEM_ERROR(errno);
      tsdbCloseDFile(&df);
      return -1;
    }

    if (tsdbUpdateDFileHeader(&df) < 0) {
      tsdbCloseDFile(&df);
      return -1;
    }

    tsdbCloseDFile(&df);
H
Hongze Cheng 已提交
481
    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
482 483
             file_size, pDFile->info.size);
  } else if (pDFile->info.size > file_size) {
H
Hongze Cheng 已提交
484
    tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
485
              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size);
H
Hongze Cheng 已提交
486
    // pRepo->state |= TSDB_STATE_BAD_DATA;
H
Hongze Cheng 已提交
487
    TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD);
H
Hongze Cheng 已提交
488
    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
H
Hongze Cheng 已提交
489
    return 0;
H
refact  
Hongze Cheng 已提交
490 491
  } else {
    tsdbDebug("vgId:%d file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile));
H
Hongze Cheng 已提交
492 493 494 495 496
  }

  return 0;
}

H
Hongze Cheng 已提交
497
static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) {
H
TD-353  
Hongze Cheng 已提交
498
  int tlen = 0;
H
Hongze Cheng 已提交
499

H
Hongze Cheng 已提交
500
  tlen += taosEncodeFixedU32(buf, pInfo->magic);
501
  tlen += taosEncodeFixedU32(buf, pInfo->fver);
H
TD-353  
Hongze Cheng 已提交
502 503 504
  tlen += taosEncodeFixedU32(buf, pInfo->len);
  tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks);
  tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks);
H
Hongze Cheng 已提交
505 506 507
  tlen += taosEncodeFixedU32(buf, pInfo->offset);
  tlen += taosEncodeFixedU64(buf, pInfo->size);
  tlen += taosEncodeFixedU64(buf, pInfo->tombSize);
H
TD-353  
Hongze Cheng 已提交
508

H
TD-353  
Hongze Cheng 已提交
509
  return tlen;
H
TD-353  
Hongze Cheng 已提交
510 511
}

H
Hongze Cheng 已提交
512
static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) {
H
Hongze Cheng 已提交
513
  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
514
  buf = taosDecodeFixedU32(buf, &(pInfo->fver));
H
TD-353  
Hongze Cheng 已提交
515 516 517
  buf = taosDecodeFixedU32(buf, &(pInfo->len));
  buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks));
  buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks));
H
Hongze Cheng 已提交
518 519 520
  buf = taosDecodeFixedU32(buf, &(pInfo->offset));
  buf = taosDecodeFixedU64(buf, &(pInfo->size));
  buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
H
TD-353  
Hongze Cheng 已提交
521 522 523 524

  return buf;
}

H
Hongze Cheng 已提交
525 526 527 528 529 530 531 532 533
static int tsdbApplyDFileChange(SDFile *from, SDFile *to) {
  ASSERT(from != NULL || to != NULL);

  if (from != NULL) {
    if (to == NULL) {
      tsdbRemoveDFile(from);
    } else {
      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
        if (from->info.size > to->info.size) {
H
Hongze Cheng 已提交
534
          tsdbRollBackDFile(to);
H
Hongze Cheng 已提交
535 536
        }
      } else {
537
        (void)tsdbRemoveDFile(from);
H
Hongze Cheng 已提交
538 539 540 541 542 543 544
      }
    }
  }

  return 0;
}

H
Hongze Cheng 已提交
545
static int tsdbRollBackDFile(SDFile *pDFile) {
H
Hongze Cheng 已提交
546 547
  SDFile df = *pDFile;

548 549
  // if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
  if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) {
H
Hongze Cheng 已提交
550 551 552
    return -1;
  }

553
  if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) {
H
Hongze Cheng 已提交
554 555 556 557 558 559 560 561 562 563
    terrno = TAOS_SYSTEM_ERROR(errno);
    tsdbCloseDFile(&df);
    return -1;
  }

  if (tsdbUpdateDFileHeader(&df) < 0) {
    tsdbCloseDFile(&df);
    return -1;
  }

H
Hongze Cheng 已提交
564 565
  TSDB_FILE_FSYNC(&df);

H
Hongze Cheng 已提交
566 567 568 569
  tsdbCloseDFile(&df);
  return 0;
}

H
Hongze Cheng 已提交
570
// ============== Operations on SDFileSet
S
Shengliang Guan 已提交
571
void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver) {
572 573 574 575
  TSDB_FSET_FID(pSet) = fid;
  TSDB_FSET_VER(pSet) = TSDB_LATEST_FSET_VER;
  TSDB_FSET_STATE(pSet) = 0;
  pSet->reserve = 0;
H
Hongze Cheng 已提交
576

H
Hongze Cheng 已提交
577 578
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype);
S
Shengliang Guan 已提交
579
    tsdbInitDFile(pRepo, pDFile, did, fid, ver, ftype);
H
Hongze Cheng 已提交
580 581 582
  }
}

H
Hongze Cheng 已提交
583
void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) {
584
  TSDB_FSET_FID(pSet) = TSDB_FSET_FID(pOSet);
H
Hongze Cheng 已提交
585
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
586
    tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype));
H
Hongze Cheng 已提交
587
  }
H
Hongze Cheng 已提交
588 589
}

H
Hongze Cheng 已提交
590 591
int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) {
  int tlen = 0;
H
Hongze Cheng 已提交
592

593 594 595 596
  tlen += taosEncodeFixedI32(buf, TSDB_FSET_FID(pSet));
  // state not included
  tlen += taosEncodeFixedU8(buf, TSDB_FSET_VER(pSet));
  tlen += taosEncodeFixedU16(buf, pSet->reserve);
H
Hongze Cheng 已提交
597 598
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype));
H
Hongze Cheng 已提交
599
  }
H
Hongze Cheng 已提交
600

H
Hongze Cheng 已提交
601
  return tlen;
H
Hongze Cheng 已提交
602 603
}

S
Shengliang Guan 已提交
604
void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet) {
605 606 607 608
  buf = taosDecodeFixedI32(buf, &(TSDB_FSET_FID(pSet)));
  TSDB_FSET_STATE(pSet) = 0;
  buf = taosDecodeFixedU8(buf, &(TSDB_FSET_VER(pSet)));
  buf = taosDecodeFixedU16(buf, &(pSet->reserve));
H
Hongze Cheng 已提交
609

H
Hongze Cheng 已提交
610
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
S
Shengliang Guan 已提交
611
    buf = tsdbDecodeSDFile(pRepo, buf, TSDB_DFILE_IN_SET(pSet, ftype));
H
Hongze Cheng 已提交
612
  }
H
Hongze Cheng 已提交
613
  return buf;
H
Hongze Cheng 已提交
614 615
}

H
Hongze Cheng 已提交
616 617 618
int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) {
  int tlen = 0;

619 620 621
  tlen += taosEncodeFixedI32(buf, TSDB_FSET_FID(pSet));
  tlen += taosEncodeFixedU8(buf, TSDB_FSET_VER(pSet));
  tlen += taosEncodeFixedU16(buf, pSet->reserve);
H
Hongze Cheng 已提交
622 623 624 625 626 627 628 629
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
  }

  return tlen;
}

void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) {
630 631 632
  buf = taosDecodeFixedI32(buf, &(TSDB_FSET_FID(pSet)));
  buf = taosDecodeFixedU8(buf, &(TSDB_FSET_VER(pSet)));
  buf = taosDecodeFixedU16(buf, &(pSet->reserve));
H
Hongze Cheng 已提交
633 634 635 636 637 638 639

  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
  }
  return buf;
}

H
Hongze Cheng 已提交
640
int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) {
H
Hongze Cheng 已提交
641
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
642 643 644
    SDFile *pDFileFrom = (from) ? TSDB_DFILE_IN_SET(from, ftype) : NULL;
    SDFile *pDFileTo = (to) ? TSDB_DFILE_IN_SET(to, ftype) : NULL;
    if (tsdbApplyDFileChange(pDFileFrom, pDFileTo) < 0) {
H
Hongze Cheng 已提交
645 646 647 648 649 650 651
      return -1;
    }
  }

  return 0;
}

S
Shengliang Guan 已提交
652
int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader) {
H
Hongze Cheng 已提交
653
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
654
    if (tsdbCreateDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype), updateHeader, ftype) < 0) {
H
Hongze Cheng 已提交
655 656 657 658 659 660
      tsdbCloseDFileSet(pSet);
      tsdbRemoveDFileSet(pSet);
      return -1;
    }
  }

H
Hongze Cheng 已提交
661 662
  return 0;
}
H
Hongze Cheng 已提交
663

H
Hongze Cheng 已提交
664
int tsdbUpdateDFileSetHeader(SDFileSet *pSet) {
H
Hongze Cheng 已提交
665
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
666
    if (tsdbUpdateDFileHeader(TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
H
Hongze Cheng 已提交
667 668 669
      return -1;
    }
  }
H
Hongze Cheng 已提交
670
  return 0;
H
refact  
Hongze Cheng 已提交
671 672
}

H
Hongze Cheng 已提交
673
int tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet) {
H
Hongze Cheng 已提交
674
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
675
    if (tsdbScanAndTryFixDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
H
Hongze Cheng 已提交
676 677 678 679 680 681
      return -1;
    }
  }
  return 0;
}

682
int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *_version) {
H
Hongze Cheng 已提交
683
  char *p = NULL;
684
  *_version = 0;
H
Hongze Cheng 已提交
685 686
  *ftype = TSDB_FILE_MAX;

687
  sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, _version);
H
Hongze Cheng 已提交
688 689 690 691 692 693 694
  for (TSDB_FILE_T i = 0; i < TSDB_FILE_MAX; i++) {
    if (strcmp(p, TSDB_FNAME_SUFFIX[i]) == 0) {
      *ftype = i;
      break;
    }
  }

wafwerar's avatar
wafwerar 已提交
695
  taosMemoryFreeClear(p);
H
Hongze Cheng 已提交
696 697 698
  return 0;
}

H
Hongze Cheng 已提交
699
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) {
H
refact  
Hongze Cheng 已提交
700 701 702 703
  ASSERT(ftype != TSDB_FILE_MAX);

  if (ftype < TSDB_FILE_MAX) {
    if (ver == 0) {
H
Hongze Cheng 已提交
704
      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]);
H
refact  
Hongze Cheng 已提交
705
    } else {
H
Hongze Cheng 已提交
706
      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid,
H
Hongze Cheng 已提交
707
               TSDB_FNAME_SUFFIX[ftype], ver);
H
refact  
Hongze Cheng 已提交
708 709 710
    }
  } else {
    if (ver == 0) {
H
Hongze Cheng 已提交
711
      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vid, TSDB_FNAME_SUFFIX[ftype]);
H
refact  
Hongze Cheng 已提交
712
    } else {
H
Hongze Cheng 已提交
713
      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver);
H
refact  
Hongze Cheng 已提交
714 715
    }
  }
H
Hongze Cheng 已提交
716
}