tsdbFile.c 17.9 KB
Newer Older
H
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
TD-353  
Hongze Cheng 已提交
15

H
refact  
Hongze Cheng 已提交
16
#include "tsdbint.h"
H
TD-353  
Hongze Cheng 已提交
17

H
refact  
Hongze Cheng 已提交
18
static const char *TSDB_FNAME_SUFFIX[] = {
H
Hongze Cheng 已提交
19 20 21 22 23
    "head",  // TSDB_FILE_HEAD
    "data",  // TSDB_FILE_DATA
    "last",  // TSDB_FILE_LAST
    "",      // TSDB_FILE_MAX
    "meta"   // TSDB_FILE_META
H
refact  
Hongze Cheng 已提交
24
};
H
TD-353  
Hongze Cheng 已提交
25

H
Hongze Cheng 已提交
26 27 28 29 30 31
static void  tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);
static int   tsdbRollBackMFile(SMFile *pMFile);
static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
static int   tsdbRollBackDFile(SDFile *pDFile);

H
refact  
Hongze Cheng 已提交
32
// ============== SMFile
H
Hongze Cheng 已提交
33
void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
H
refact  
Hongze Cheng 已提交
34 35
  char fname[TSDB_FILENAME_LEN];

H
Hongze Cheng 已提交
36
  TSDB_FILE_SET_CLOSED(pMFile);
H
refact  
Hongze Cheng 已提交
37

H
Hongze Cheng 已提交
38 39
  memset(&(pMFile->info), 0, sizeof(pMFile->info));
  pMFile->info.magic = TSDB_FILE_INIT_MAGIC;
H
Hongze Cheng 已提交
40

H
refact  
Hongze Cheng 已提交
41
  tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname);
H
Hongze Cheng 已提交
42 43 44
  tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
}

H
Hongze Cheng 已提交
45
void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) {
H
Hongze Cheng 已提交
46 47
  *pMFile = *pOMFile;
  TSDB_FILE_SET_CLOSED(pMFile);
H
Hongze Cheng 已提交
48
}
H
Hongze Cheng 已提交
49

H
refact  
Hongze Cheng 已提交
50
int tsdbEncodeSMFile(void **buf, SMFile *pMFile) {
H
Hongze Cheng 已提交
51
  int tlen = 0;
H
hzcheng 已提交
52

H
Hongze Cheng 已提交
53 54
  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
  tlen += tfsEncodeFile(buf, &(pMFile->f));
H
hzcheng 已提交
55

H
Hongze Cheng 已提交
56
  return tlen;
H
Hongze Cheng 已提交
57 58
}

H
refact  
Hongze Cheng 已提交
59
void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) {
H
Hongze Cheng 已提交
60 61
  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
  buf = tfsDecodeFile(buf, &(pMFile->f));
H
Hongze Cheng 已提交
62
  TSDB_FILE_SET_CLOSED(pMFile);
H
Hongze Cheng 已提交
63

H
Hongze Cheng 已提交
64 65
  return buf;
}
H
Hongze Cheng 已提交
66

H
Hongze Cheng 已提交
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) {
  int tlen = 0;

  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile));

  return tlen;
}

void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) {
  char *aname;
  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
  buf = taosDecodeString(buf, &aname);
  strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN);
  TSDB_FILE_SET_CLOSED(pMFile);

  tfree(aname);

  return buf;
}

H
Hongze Cheng 已提交
88
int tsdbApplyMFileChange(SMFile *from, SMFile *to) {
H
Hongze Cheng 已提交
89
  if (from == NULL && to == NULL) return 0;
H
Hongze Cheng 已提交
90 91 92

  if (from != NULL) {
    if (to == NULL) {
H
Hongze Cheng 已提交
93
      return tsdbRemoveMFile(from);
H
Hongze Cheng 已提交
94 95 96
    } else {
      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
        if (from->info.size > to->info.size) {
H
Hongze Cheng 已提交
97
          tsdbRollBackMFile(to);
H
Hongze Cheng 已提交
98 99
        }
      } else {
H
Hongze Cheng 已提交
100
        return tsdbRemoveMFile(from);
H
Hongze Cheng 已提交
101 102 103 104 105 106 107
      }
    }
  }

  return 0;
}

H
Hongze Cheng 已提交
108
int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) {
H
Hongze Cheng 已提交
109 110
  ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC);

S
Shengliang Guan 已提交
111
  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
H
Hongze Cheng 已提交
112
  if (pMFile->fd < 0) {
H
Hongze Cheng 已提交
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
    if (errno == ENOENT) {
      // Try to create directory recursively
      char *s = strdup(TFILE_REL_NAME(&(pMFile->f)));
      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) {
        tfree(s);
        return -1;
      }
      tfree(s);

      pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
      if (pMFile->fd < 0) {
        terrno = TAOS_SYSTEM_ERROR(errno);
        return -1;
      }
    } else {
      terrno = TAOS_SYSTEM_ERROR(errno);
      return -1;
    }
H
Hongze Cheng 已提交
131 132
  }

H
Hongze Cheng 已提交
133 134 135 136
  if (!updateHeader) {
    return 0;
  }

H
Hongze Cheng 已提交
137
  if (tsdbUpdateMFileHeader(pMFile) < 0) {
H
Hongze Cheng 已提交
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
    tsdbCloseMFile(pMFile);
    tsdbRemoveMFile(pMFile);
    return -1;
  }

  pMFile->info.size += TSDB_FILE_HEAD_SIZE;

  return 0;
}

int tsdbUpdateMFileHeader(SMFile *pMFile) {
  char buf[TSDB_FILE_HEAD_SIZE] = "\0";

  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
    return -1;
  }

  void *ptr = buf;
H
Hongze Cheng 已提交
156
  tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile));
H
Hongze Cheng 已提交
157

H
refact  
Hongze Cheng 已提交
158
  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
H
Hongze Cheng 已提交
159 160 161 162 163 164 165
  if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
    return -1;
  }

  return 0;
}

H
Hongze Cheng 已提交
166 167 168 169 170 171 172 173 174 175 176 177 178
int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) {
  char buf[TSDB_FILE_HEAD_SIZE] = "\0";

  ASSERT(TSDB_FILE_OPENED(pMFile));

  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
    return -1;
  }

  if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
    return -1;
  }

H
refact  
Hongze Cheng 已提交
179 180 181 182 183
  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
    return -1;
  }

H
Hongze Cheng 已提交
184 185 186 187
  tsdbDecodeMFInfo(buf, pInfo);
  return 0;
}

H
Hongze Cheng 已提交
188 189
int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) {
  SMFile *    pMFile = pRepo->fs->cstatus->pmf;
H
Hongze Cheng 已提交
190
  struct stat mfstat;
H
Hongze Cheng 已提交
191 192 193
  SMFile      mf;

  if (pMFile == NULL) {
H
refact  
Hongze Cheng 已提交
194
    // No meta file, no need to scan
H
Hongze Cheng 已提交
195 196 197
    return 0;
  }

H
refact  
Hongze Cheng 已提交
198
  tsdbInitMFileEx(&mf, pMFile);
H
Hongze Cheng 已提交
199 200 201 202 203 204 205

  if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) {
    tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
              TSDB_FILE_FULL_NAME(pMFile));
    pRepo->state |= TSDB_STATE_BAD_META;
    return 0;
  }
H
Hongze Cheng 已提交
206 207 208 209 210 211

  if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

H
Hongze Cheng 已提交
212
  if (pMFile->info.size < mfstat.st_size) {
H
Hongze Cheng 已提交
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
    if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
      return -1;
    }

    if (taosFtruncate(mf.fd, mf.info.size) < 0) {
      terrno = TAOS_SYSTEM_ERROR(errno);
      tsdbCloseMFile(&mf);
      return -1;
    }

    if (tsdbUpdateMFileHeader(&mf) < 0) {
      tsdbCloseMFile(&mf);
      return -1;
    }

    tsdbCloseMFile(&mf);
H
Hongze Cheng 已提交
229 230
    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
             mfstat.st_size, pMFile->info.size);
H
Hongze Cheng 已提交
231
  } else if (pMFile->info.size > mfstat.st_size) {
H
Hongze Cheng 已提交
232 233 234
    tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size);
    pRepo->state |= TSDB_STATE_BAD_META;
H
Hongze Cheng 已提交
235
    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
H
Hongze Cheng 已提交
236
    return 0;
H
refact  
Hongze Cheng 已提交
237 238
  } else {
    tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile));
H
Hongze Cheng 已提交
239 240 241 242 243
  }

  return 0;
}

H
Hongze Cheng 已提交
244
int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) {
H
Hongze Cheng 已提交
245
  int tlen = 0;
H
Hongze Cheng 已提交
246

H
Hongze Cheng 已提交
247 248 249 250 251
  tlen += taosEncodeVariantI64(buf, pInfo->size);
  tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
  tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
  tlen += taosEncodeVariantI64(buf, pInfo->nDels);
  tlen += taosEncodeFixedU32(buf, pInfo->magic);
H
Hongze Cheng 已提交
252

H
Hongze Cheng 已提交
253
  return tlen;
H
Hongze Cheng 已提交
254 255
}

H
Hongze Cheng 已提交
256
void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) {
H
Hongze Cheng 已提交
257 258 259 260 261 262 263
  buf = taosDecodeVariantI64(buf, &(pInfo->size));
  buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
  buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
  buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
  buf = taosDecodeFixedU32(buf, &(pInfo->magic));

  return buf;
H
Hongze Cheng 已提交
264 265
}

H
Hongze Cheng 已提交
266
static int tsdbRollBackMFile(SMFile *pMFile) {
H
refact  
Hongze Cheng 已提交
267 268 269
  SMFile mf;

  tsdbInitMFileEx(&mf, pMFile);
H
Hongze Cheng 已提交
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291

  if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
    return -1;
  }

  if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    tsdbCloseMFile(&mf);
    return -1;
  }

  if (tsdbUpdateMFileHeader(&mf) < 0) {
    tsdbCloseMFile(&mf);
    return -1;
  }

  TSDB_FILE_FSYNC(&mf);

  tsdbCloseMFile(&mf);
  return 0;
}

H
Hongze Cheng 已提交
292
// ============== Operations on SDFile
H
Hongze Cheng 已提交
293
void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) {
H
refact  
Hongze Cheng 已提交
294 295
  char fname[TSDB_FILENAME_LEN];

H
Hongze Cheng 已提交
296
  TSDB_FILE_SET_CLOSED(pDFile);
H
refact  
Hongze Cheng 已提交
297

H
Hongze Cheng 已提交
298 299
  memset(&(pDFile->info), 0, sizeof(pDFile->info));
  pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
H
refact  
Hongze Cheng 已提交
300

H
Hongze Cheng 已提交
301
  tsdbGetFilename(vid, fid, ver, ftype, fname);
H
Hongze Cheng 已提交
302
  tfsInitFile(&(pDFile->f), did.level, did.id, fname);
H
Hongze Cheng 已提交
303 304
}

H
Hongze Cheng 已提交
305 306
void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) {
  *pDFile = *pODFile;
H
Hongze Cheng 已提交
307 308 309
  TSDB_FILE_SET_CLOSED(pDFile);
}

H
refact  
Hongze Cheng 已提交
310
int tsdbEncodeSDFile(void **buf, SDFile *pDFile) {
H
Hongze Cheng 已提交
311
  int tlen = 0;
312

H
Hongze Cheng 已提交
313 314
  tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
  tlen += tfsEncodeFile(buf, &(pDFile->f));
H
TD-34  
hzcheng 已提交
315

H
Hongze Cheng 已提交
316
  return tlen;
H
TD-34  
hzcheng 已提交
317 318
}

H
refact  
Hongze Cheng 已提交
319
void *tsdbDecodeSDFile(void *buf, SDFile *pDFile) {
H
Hongze Cheng 已提交
320 321
  buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
  buf = tfsDecodeFile(buf, &(pDFile->f));
H
Hongze Cheng 已提交
322
  TSDB_FILE_SET_CLOSED(pDFile);
H
TD-353  
Hongze Cheng 已提交
323

H
Hongze Cheng 已提交
324
  return buf;
H
TD-353  
Hongze Cheng 已提交
325
}
H
TD-353  
Hongze Cheng 已提交
326

H
Hongze Cheng 已提交
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) {
  int tlen = 0;

  tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pDFile));

  return tlen;
}

static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
  char *aname;

  buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
  buf = taosDecodeString(buf, &aname);
  strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN);
  TSDB_FILE_SET_CLOSED(pDFile);
  tfree(aname);

  return buf;
}

H
Hongze Cheng 已提交
348
int tsdbCreateDFile(SDFile *pDFile, bool updateHeader) {
H
Hongze Cheng 已提交
349 350
  ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);

S
Shengliang Guan 已提交
351
  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
H
Hongze Cheng 已提交
352
  if (pDFile->fd < 0) {
H
Hongze Cheng 已提交
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    if (errno == ENOENT) {
      // Try to create directory recursively
      char *s = strdup(TFILE_REL_NAME(&(pDFile->f)));
      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pDFile), TSDB_FILE_ID(pDFile)) < 0) {
        tfree(s);
        return -1;
      }
      tfree(s);

      pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
      if (pDFile->fd < 0) {
        terrno = TAOS_SYSTEM_ERROR(errno);
        return -1;
      }
    } else {
      terrno = TAOS_SYSTEM_ERROR(errno);
      return -1;
    }
H
Hongze Cheng 已提交
371 372
  }

H
Hongze Cheng 已提交
373 374 375 376
  if (!updateHeader) {
    return 0;
  }

H
Hongze Cheng 已提交
377
  if (tsdbUpdateDFileHeader(pDFile) < 0) {
H
Hongze Cheng 已提交
378 379 380 381 382 383 384
    tsdbCloseDFile(pDFile);
    tsdbRemoveDFile(pDFile);
    return -1;
  }

  pDFile->info.size += TSDB_FILE_HEAD_SIZE;

H
Hongze Cheng 已提交
385 386 387
  return 0;
}

H
Hongze Cheng 已提交
388 389 390 391 392 393
int tsdbUpdateDFileHeader(SDFile *pDFile) {
  char buf[TSDB_FILE_HEAD_SIZE] = "\0";

  if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
    return -1;
  }
H
Hongze Cheng 已提交
394

H
Hongze Cheng 已提交
395
  void *ptr = buf;
H
Hongze Cheng 已提交
396
  taosEncodeFixedU32(&ptr, TSDB_FS_VERSION);
H
Hongze Cheng 已提交
397
  tsdbEncodeDFInfo(&ptr, &(pDFile->info));
H
Hongze Cheng 已提交
398

H
refact  
Hongze Cheng 已提交
399
  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
H
Hongze Cheng 已提交
400
  if (tsdbWriteDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
H
Hongze Cheng 已提交
401 402
    return -1;
  }
H
Hongze Cheng 已提交
403 404

  return 0;
H
Hongze Cheng 已提交
405 406
}

H
Hongze Cheng 已提交
407
int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
H
Hongze Cheng 已提交
408 409
  char     buf[TSDB_FILE_HEAD_SIZE] = "\0";
  uint32_t version;
H
Hongze Cheng 已提交
410 411 412 413 414 415 416 417 418 419 420

  ASSERT(TSDB_FILE_OPENED(pDFile));

  if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
    return -1;
  }

  if (tsdbReadDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
    return -1;
  }

H
refact  
Hongze Cheng 已提交
421 422 423 424 425
  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
    return -1;
  }

H
Hongze Cheng 已提交
426 427
  void *pBuf = buf;
  pBuf = taosDecodeFixedU32(pBuf, &version);
H
Hongze Cheng 已提交
428
  pBuf = tsdbDecodeDFInfo(pBuf, pInfo);
H
Hongze Cheng 已提交
429 430 431
  return 0;
}

H
Hongze Cheng 已提交
432
static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) {
H
Hongze Cheng 已提交
433
  struct stat dfstat;
H
refact  
Hongze Cheng 已提交
434 435 436
  SDFile      df;

  tsdbInitDFileEx(&df, pDFile);
H
Hongze Cheng 已提交
437

H
Hongze Cheng 已提交
438 439 440 441 442 443 444
  if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) {
    tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
              TSDB_FILE_FULL_NAME(pDFile));
    pRepo->state |= TSDB_STATE_BAD_DATA;
    return 0;
  }

H
Hongze Cheng 已提交
445 446 447 448 449
  if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    return -1;
  }

H
Hongze Cheng 已提交
450
  if (pDFile->info.size < dfstat.st_size) {
H
Hongze Cheng 已提交
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
    if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
      return -1;
    }

    if (taosFtruncate(df.fd, df.info.size) < 0) {
      terrno = TAOS_SYSTEM_ERROR(errno);
      tsdbCloseDFile(&df);
      return -1;
    }

    if (tsdbUpdateDFileHeader(&df) < 0) {
      tsdbCloseDFile(&df);
      return -1;
    }

    tsdbCloseDFile(&df);
H
Hongze Cheng 已提交
467 468
    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
             dfstat.st_size, pDFile->info.size);
H
Hongze Cheng 已提交
469
  } else if (pDFile->info.size > dfstat.st_size) {
H
Hongze Cheng 已提交
470 471 472
    tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size);
    pRepo->state |= TSDB_STATE_BAD_DATA;
H
Hongze Cheng 已提交
473
    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
H
Hongze Cheng 已提交
474
    return 0;
H
refact  
Hongze Cheng 已提交
475 476
  } else {
    tsdbDebug("vgId:%d file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile));
H
Hongze Cheng 已提交
477 478 479 480 481
  }

  return 0;
}

H
Hongze Cheng 已提交
482
static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) {
H
TD-353  
Hongze Cheng 已提交
483
  int tlen = 0;
H
Hongze Cheng 已提交
484

H
Hongze Cheng 已提交
485
  tlen += taosEncodeFixedU32(buf, pInfo->magic);
H
TD-353  
Hongze Cheng 已提交
486 487 488
  tlen += taosEncodeFixedU32(buf, pInfo->len);
  tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks);
  tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks);
H
Hongze Cheng 已提交
489 490 491
  tlen += taosEncodeFixedU32(buf, pInfo->offset);
  tlen += taosEncodeFixedU64(buf, pInfo->size);
  tlen += taosEncodeFixedU64(buf, pInfo->tombSize);
H
TD-353  
Hongze Cheng 已提交
492

H
TD-353  
Hongze Cheng 已提交
493
  return tlen;
H
TD-353  
Hongze Cheng 已提交
494 495
}

H
Hongze Cheng 已提交
496
static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) {
H
Hongze Cheng 已提交
497
  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
H
TD-353  
Hongze Cheng 已提交
498 499 500
  buf = taosDecodeFixedU32(buf, &(pInfo->len));
  buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks));
  buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks));
H
Hongze Cheng 已提交
501 502 503
  buf = taosDecodeFixedU32(buf, &(pInfo->offset));
  buf = taosDecodeFixedU64(buf, &(pInfo->size));
  buf = taosDecodeFixedU64(buf, &(pInfo->tombSize));
H
TD-353  
Hongze Cheng 已提交
504 505 506 507

  return buf;
}

H
Hongze Cheng 已提交
508 509 510 511 512 513 514 515 516
static int tsdbApplyDFileChange(SDFile *from, SDFile *to) {
  ASSERT(from != NULL || to != NULL);

  if (from != NULL) {
    if (to == NULL) {
      tsdbRemoveDFile(from);
    } else {
      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
        if (from->info.size > to->info.size) {
H
Hongze Cheng 已提交
517
          tsdbRollBackDFile(to);
H
Hongze Cheng 已提交
518 519 520 521 522 523 524 525 526 527
        }
      } else {
        tsdbRemoveDFile(from);
      }
    }
  }

  return 0;
}

H
Hongze Cheng 已提交
528
static int tsdbRollBackDFile(SDFile *pDFile) {
H
Hongze Cheng 已提交
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
  SDFile df = *pDFile;

  if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
    return -1;
  }

  if (taosFtruncate(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
    terrno = TAOS_SYSTEM_ERROR(errno);
    tsdbCloseDFile(&df);
    return -1;
  }

  if (tsdbUpdateDFileHeader(&df) < 0) {
    tsdbCloseDFile(&df);
    return -1;
  }

H
Hongze Cheng 已提交
546 547
  TSDB_FILE_FSYNC(&df);

H
Hongze Cheng 已提交
548 549 550 551
  tsdbCloseDFile(&df);
  return 0;
}

H
Hongze Cheng 已提交
552
// ============== Operations on SDFileSet
H
Hongze Cheng 已提交
553
void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t ver) {
H
Hongze Cheng 已提交
554 555
  pSet->fid = fid;
  pSet->state = 0;
H
Hongze Cheng 已提交
556

H
Hongze Cheng 已提交
557 558
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype);
H
Hongze Cheng 已提交
559
    tsdbInitDFile(pDFile, did, vid, fid, ver, ftype);
H
Hongze Cheng 已提交
560 561 562
  }
}

H
Hongze Cheng 已提交
563
void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) {
H
Hongze Cheng 已提交
564
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
565
    tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype));
H
Hongze Cheng 已提交
566
  }
H
Hongze Cheng 已提交
567 568
}

H
Hongze Cheng 已提交
569 570
int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) {
  int tlen = 0;
H
Hongze Cheng 已提交
571

H
Hongze Cheng 已提交
572
  tlen += taosEncodeFixedI32(buf, pSet->fid);
H
Hongze Cheng 已提交
573 574
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype));
H
Hongze Cheng 已提交
575
  }
H
Hongze Cheng 已提交
576

H
Hongze Cheng 已提交
577
  return tlen;
H
Hongze Cheng 已提交
578 579
}

H
Hongze Cheng 已提交
580
void *tsdbDecodeDFileSet(void *buf, SDFileSet *pSet) {
H
Hongze Cheng 已提交
581 582 583
  int32_t fid;

  buf = taosDecodeFixedI32(buf, &(fid));
H
refact  
Hongze Cheng 已提交
584
  pSet->state = 0;
H
Hongze Cheng 已提交
585
  pSet->fid = fid;
H
Hongze Cheng 已提交
586
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
587
    buf = tsdbDecodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype));
H
Hongze Cheng 已提交
588
  }
H
Hongze Cheng 已提交
589
  return buf;
H
Hongze Cheng 已提交
590 591
}

H
Hongze Cheng 已提交
592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) {
  int tlen = 0;

  tlen += taosEncodeFixedI32(buf, pSet->fid);
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
  }

  return tlen;
}

void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) {
  int32_t fid;

  buf = taosDecodeFixedI32(buf, &(fid));
  pSet->fid = fid;
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
    buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
  }
  return buf;
}

H
Hongze Cheng 已提交
614
int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) {
H
Hongze Cheng 已提交
615
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
616 617 618
    SDFile *pDFileFrom = (from) ? TSDB_DFILE_IN_SET(from, ftype) : NULL;
    SDFile *pDFileTo = (to) ? TSDB_DFILE_IN_SET(to, ftype) : NULL;
    if (tsdbApplyDFileChange(pDFileFrom, pDFileTo) < 0) {
H
Hongze Cheng 已提交
619 620 621 622 623 624 625
      return -1;
    }
  }

  return 0;
}

H
Hongze Cheng 已提交
626
int tsdbCreateDFileSet(SDFileSet *pSet, bool updateHeader) {
H
Hongze Cheng 已提交
627
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
628
    if (tsdbCreateDFile(TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) {
H
Hongze Cheng 已提交
629 630 631 632 633 634
      tsdbCloseDFileSet(pSet);
      tsdbRemoveDFileSet(pSet);
      return -1;
    }
  }

H
Hongze Cheng 已提交
635 636
  return 0;
}
H
Hongze Cheng 已提交
637

H
Hongze Cheng 已提交
638
int tsdbUpdateDFileSetHeader(SDFileSet *pSet) {
H
Hongze Cheng 已提交
639
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
640
    if (tsdbUpdateDFileHeader(TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
H
Hongze Cheng 已提交
641 642 643
      return -1;
    }
  }
H
Hongze Cheng 已提交
644
  return 0;
H
refact  
Hongze Cheng 已提交
645 646
}

H
Hongze Cheng 已提交
647
int tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet *pSet) {
H
Hongze Cheng 已提交
648
  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
H
Hongze Cheng 已提交
649
    if (tsdbScanAndTryFixDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
H
Hongze Cheng 已提交
650 651 652 653 654 655
      return -1;
    }
  }
  return 0;
}

H
Hongze Cheng 已提交
656 657 658 659 660 661 662 663 664 665 666 667 668
int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version) {
  char *p = NULL;
  *version = 0;
  *ftype = TSDB_FILE_MAX;

  sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, version);
  for (TSDB_FILE_T i = 0; i < TSDB_FILE_MAX; i++) {
    if (strcmp(p, TSDB_FNAME_SUFFIX[i]) == 0) {
      *ftype = i;
      break;
    }
  }

H
Hongze Cheng 已提交
669
  tfree(p);
H
Hongze Cheng 已提交
670 671 672
  return 0;
}

H
Hongze Cheng 已提交
673
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) {
H
refact  
Hongze Cheng 已提交
674 675 676 677
  ASSERT(ftype != TSDB_FILE_MAX);

  if (ftype < TSDB_FILE_MAX) {
    if (ver == 0) {
H
Hongze Cheng 已提交
678
      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]);
H
refact  
Hongze Cheng 已提交
679
    } else {
H
Hongze Cheng 已提交
680
      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid,
H
Hongze Cheng 已提交
681
               TSDB_FNAME_SUFFIX[ftype], ver);
H
refact  
Hongze Cheng 已提交
682 683 684
    }
  } else {
    if (ver == 0) {
H
Hongze Cheng 已提交
685
      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vid, TSDB_FNAME_SUFFIX[ftype]);
H
refact  
Hongze Cheng 已提交
686
    } else {
H
Hongze Cheng 已提交
687
      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver);
H
refact  
Hongze Cheng 已提交
688 689
    }
  }
H
Hongze Cheng 已提交
690
}