tsdbReaderWriter.c 30.0 KB
Newer Older
H
refact  
Hongze Cheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "tsdb.h"

H
Hongze Cheng 已提交
18 19
#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F)

H
Hongze Cheng 已提交
20 21
// SDelFWriter ====================================================
struct SDelFWriter {
H
Hongze Cheng 已提交
22
  STsdb    *pTsdb;
H
Hongze Cheng 已提交
23 24
  SDelFile *pFile;
  TdFilePtr pWriteH;
H
refact  
Hongze Cheng 已提交
25 26
};

H
Hongze Cheng 已提交
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) {
  int32_t      code = 0;
  char        *fname = NULL;  // TODO
  SDelFWriter *pDelFWriter;

  pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter));
  if (pDelFWriter == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
  }

  pDelFWriter->pTsdb = pTsdb;
  pDelFWriter->pFile = pFile;
  pDelFWriter->pWriteH = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE);
  if (pDelFWriter->pWriteH == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  if (taosLSeekFile(pDelFWriter->pWriteH, TSDB_FHDR_SIZE, SEEK_SET) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
more  
Hongze Cheng 已提交
51 52 53
  pDelFWriter->pFile->size = TSDB_FHDR_SIZE;
  pDelFWriter->pFile->offset = 0;

H
Hongze Cheng 已提交
54 55 56 57
  return code;

_err:
  tsdbError("vgId:%d failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
58 59 60
  return code;
}

H
Hongze Cheng 已提交
61
int32_t tsdbDelFWriterClose(SDelFWriter *pWriter, int8_t sync) {
H
Hongze Cheng 已提交
62
  int32_t code = 0;
H
Hongze Cheng 已提交
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79

  // sync
  if (sync && taosFsyncFile(pWriter->pWriteH) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // close
  if (taosCloseFile(&pWriter->pWriteH) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  return code;

_err:
  tsdbError("vgId:%d failed to close del file writer since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
80 81 82
  return code;
}

H
Hongze Cheng 已提交
83
int32_t tsdbWriteDelData(SDelFWriter *pWriter, SMapData *pDelDataMap, uint8_t **ppBuf, SDelIdx *pDelIdx) {
H
Hongze Cheng 已提交
84 85
  int32_t  code = 0;
  uint8_t *pBuf = NULL;
H
Hongze Cheng 已提交
86 87
  int64_t  size = 0;
  int64_t  n = 0;
H
Hongze Cheng 已提交
88

H
Hongze Cheng 已提交
89
  // prepare
H
Hongze Cheng 已提交
90
  size += tPutU32(NULL, TSDB_FILE_DLMT);
H
Hongze Cheng 已提交
91 92
  size += tPutI64(NULL, pDelIdx->suid);
  size += tPutI64(NULL, pDelIdx->uid);
H
Hongze Cheng 已提交
93
  size = size + tPutMapData(NULL, pDelDataMap) + sizeof(TSCKSUM);
H
Hongze Cheng 已提交
94 95 96 97 98 99 100

  // alloc
  if (!ppBuf) ppBuf = &pBuf;
  code = tsdbRealloc(ppBuf, size);
  if (code) goto _err;

  // build
H
Hongze Cheng 已提交
101
  n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT);
H
Hongze Cheng 已提交
102 103
  n += tPutI64(*ppBuf + n, pDelIdx->suid);
  n += tPutI64(*ppBuf + n, pDelIdx->uid);
H
Hongze Cheng 已提交
104
  n += tPutMapData(*ppBuf + n, pDelDataMap);
H
Hongze Cheng 已提交
105 106 107 108 109 110 111 112 113 114 115 116 117 118
  taosCalcChecksumAppend(0, *ppBuf, size);

  ASSERT(n + sizeof(TSCKSUM) == size);

  // write
  n = taosWriteFile(pWriter->pWriteH, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  ASSERT(n == size);

  // update
H
Hongze Cheng 已提交
119 120
  pDelIdx->offset = pWriter->pFile->size;
  pDelIdx->size = size;
H
Hongze Cheng 已提交
121 122 123 124 125 126 127 128 129
  pWriter->pFile->offset = pWriter->pFile->size;
  pWriter->pFile->size += size;

  tsdbFree(pBuf);
  return code;

_err:
  tsdbError("vgId:%d failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
  tsdbFree(pBuf);
H
Hongze Cheng 已提交
130 131 132
  return code;
}

H
Hongze Cheng 已提交
133
int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SMapData *pDelIdxMap, uint8_t **ppBuf) {
H
Hongze Cheng 已提交
134
  int32_t  code = 0;
H
Hongze Cheng 已提交
135 136
  int64_t  size = 0;
  int64_t  n = 0;
H
Hongze Cheng 已提交
137
  uint8_t *pBuf = NULL;
H
Hongze Cheng 已提交
138

H
Hongze Cheng 已提交
139
  // prepare
H
Hongze Cheng 已提交
140 141
  size += tPutU32(NULL, TSDB_FILE_DLMT);
  size = size + tPutMapData(NULL, pDelIdxMap) + sizeof(TSCKSUM);
H
Hongze Cheng 已提交
142 143

  // alloc
H
Hongze Cheng 已提交
144
  if (!ppBuf) ppBuf = &pBuf;
H
Hongze Cheng 已提交
145
  code = tsdbRealloc(ppBuf, size);
H
Hongze Cheng 已提交
146
  if (code) goto _err;
H
Hongze Cheng 已提交
147

H
Hongze Cheng 已提交
148
  // build
H
Hongze Cheng 已提交
149 150
  n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT);
  n += tPutMapData(*ppBuf + n, pDelIdxMap);
H
Hongze Cheng 已提交
151 152
  taosCalcChecksumAppend(0, *ppBuf, size);

H
Hongze Cheng 已提交
153 154
  ASSERT(n + sizeof(TSCKSUM) == size);

H
Hongze Cheng 已提交
155
  // write
H
Hongze Cheng 已提交
156 157
  n = taosWriteFile(pWriter->pWriteH, *ppBuf, size);
  if (n < 0) {
H
Hongze Cheng 已提交
158 159 160 161
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
162 163 164
  ASSERT(n == size);

  // update
H
Hongze Cheng 已提交
165 166 167
  pWriter->pFile->offset = pWriter->pFile->size;
  pWriter->pFile->size += size;

H
Hongze Cheng 已提交
168 169 170 171
  tsdbFree(pBuf);
  return code;

_err:
H
more  
Hongze Cheng 已提交
172
  tsdbError("vgId:%d write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
  tsdbFree(pBuf);
  return code;
}

int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter, uint8_t **ppBuf) {
  int32_t  code = 0;
  uint8_t *pBuf = NULL;
  int64_t  size = TSDB_FHDR_SIZE;
  int64_t  n;

  // alloc
  if (!ppBuf) ppBuf = &pBuf;
  code = tsdbRealloc(ppBuf, size);
  if (code) goto _err;

  // build
  memset(*ppBuf, 0, size);
  n = tPutDelFileHdr(*ppBuf, pWriter->pFile);
  taosCalcChecksumAppend(0, *ppBuf, size);

  ASSERT(n <= size - sizeof(TSCKSUM));

  // seek
  if (taosLSeekFile(pWriter->pWriteH, 0, SEEK_SET) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // write
  if (taosWriteFile(pWriter->pWriteH, *ppBuf, size) < size) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  tsdbFree(pBuf);
H
Hongze Cheng 已提交
208 209 210
  return code;

_err:
H
Hongze Cheng 已提交
211
  tsdbError("vgId:%d update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
212
  tsdbFree(pBuf);
H
Hongze Cheng 已提交
213 214 215 216 217
  return code;
}

// SDelFReader ====================================================
struct SDelFReader {
H
Hongze Cheng 已提交
218
  STsdb    *pTsdb;
H
Hongze Cheng 已提交
219 220
  SDelFile *pFile;
  TdFilePtr pReadH;
H
refact  
Hongze Cheng 已提交
221
};
H
Hongze Cheng 已提交
222

H
Hongze Cheng 已提交
223 224 225 226
int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb, uint8_t **ppBuf) {
  int32_t      code = 0;
  char        *fname = NULL;  // todo
  SDelFReader *pDelFReader;
H
Hongze Cheng 已提交
227
  int64_t      n;
H
Hongze Cheng 已提交
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252

  // alloc
  pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
  if (pDelFReader == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
  }

  // open impl
  pDelFReader->pTsdb = pTsdb;
  pDelFReader->pFile = pFile;
  pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ);
  if (pDelFReader == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    taosMemoryFree(pDelFReader);
    goto _err;
  }

  // load and check hdr if buffer is given
  if (ppBuf) {
    code = tsdbRealloc(ppBuf, TSDB_FHDR_SIZE);
    if (code) {
      goto _err;
    }

H
Hongze Cheng 已提交
253 254 255 256 257
    n = taosReadFile(pDelFReader->pReadH, *ppBuf, TSDB_FHDR_SIZE);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    } else if (n < TSDB_FHDR_SIZE) {
H
Hongze Cheng 已提交
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
      code = TSDB_CODE_FILE_CORRUPTED;
      goto _err;
    }

    if (!taosCheckChecksumWhole(*ppBuf, TSDB_FHDR_SIZE)) {
      code = TSDB_CODE_FILE_CORRUPTED;
      goto _err;
    }

    // TODO: check the content
  }

_exit:
  *ppReader = pDelFReader;
  return code;

_err:
H
Hongze Cheng 已提交
275
  tsdbError("vgId:%d del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
276
  *ppReader = NULL;
H
Hongze Cheng 已提交
277 278 279 280 281
  return code;
}

int32_t tsdbDelFReaderClose(SDelFReader *pReader) {
  int32_t code = 0;
H
Hongze Cheng 已提交
282 283

  if (pReader) {
H
Hongze Cheng 已提交
284 285 286 287
    if (taosCloseFile(&pReader->pReadH) < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _exit;
    }
H
Hongze Cheng 已提交
288 289 290
    taosMemoryFree(pReader);
  }

H
Hongze Cheng 已提交
291
_exit:
H
Hongze Cheng 已提交
292 293 294
  return code;
}

H
Hongze Cheng 已提交
295
int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SMapData *pDelDataMap, uint8_t **ppBuf) {
H
Hongze Cheng 已提交
296 297 298 299 300
  int32_t  code = 0;
  int64_t  n;
  uint32_t delimiter;
  tb_uid_t suid;
  tb_uid_t uid;
H
Hongze Cheng 已提交
301 302

  // seek
H
Hongze Cheng 已提交
303
  if (taosLSeekFile(pReader->pReadH, pDelIdx->offset, SEEK_SET) < 0) {
H
Hongze Cheng 已提交
304 305 306 307 308
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // alloc
H
Hongze Cheng 已提交
309 310
  if (!ppBuf) ppBuf = &pDelDataMap->pBuf;
  code = tsdbRealloc(ppBuf, pDelIdx->size);
H
Hongze Cheng 已提交
311 312 313
  if (code) goto _err;

  // read
H
Hongze Cheng 已提交
314
  n = taosReadFile(pReader->pReadH, *ppBuf, pDelIdx->size);
H
Hongze Cheng 已提交
315 316 317
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
H
Hongze Cheng 已提交
318 319 320
  } else if (n < pDelIdx->size) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _err;
H
Hongze Cheng 已提交
321 322 323
  }

  // check
H
Hongze Cheng 已提交
324
  if (!taosCheckChecksumWhole(*ppBuf, pDelIdx->size)) {
H
Hongze Cheng 已提交
325 326 327 328
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _err;
  }

H
Hongze Cheng 已提交
329 330 331 332
  // // decode
  n = 0;
  n += tGetU32(*ppBuf + n, &delimiter);
  ASSERT(delimiter == TSDB_FILE_DLMT);
H
Hongze Cheng 已提交
333 334 335 336
  n += tGetI64(*ppBuf + n, &suid);
  ASSERT(suid == pDelIdx->suid);
  n += tGetI64(*ppBuf + n, &uid);
  ASSERT(uid == pDelIdx->uid);
H
Hongze Cheng 已提交
337 338
  n += tGetMapData(*ppBuf + n, pDelDataMap);
  ASSERT(n + sizeof(TSCKSUM) == pDelIdx->size);
H
Hongze Cheng 已提交
339 340 341 342 343

  return code;

_err:
  tsdbError("vgId:%d read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
344 345 346
  return code;
}

H
Hongze Cheng 已提交
347 348 349 350 351 352
int32_t tsdbReadDelIdx(SDelFReader *pReader, SMapData *pDelIdxMap, uint8_t **ppBuf) {
  int32_t  code = 0;
  int32_t  n;
  int64_t  offset = pReader->pFile->offset;
  int64_t  size = pReader->pFile->size - offset;
  uint32_t delimiter;
H
Hongze Cheng 已提交
353

H
more  
Hongze Cheng 已提交
354 355
  ASSERT(ppBuf && *ppBuf);

H
Hongze Cheng 已提交
356
  // seek
H
Hongze Cheng 已提交
357
  if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) {
H
Hongze Cheng 已提交
358 359 360 361
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
362
  // alloc
H
Hongze Cheng 已提交
363
  if (!ppBuf) ppBuf = &pDelIdxMap->pBuf;
H
Hongze Cheng 已提交
364
  code = tsdbRealloc(ppBuf, size);
H
Hongze Cheng 已提交
365
  if (code) goto _err;
H
Hongze Cheng 已提交
366

H
Hongze Cheng 已提交
367
  // read
H
Hongze Cheng 已提交
368 369
  n = taosReadFile(pReader->pReadH, *ppBuf, size);
  if (n < 0) {
H
Hongze Cheng 已提交
370
    code = TAOS_SYSTEM_ERROR(errno);
H
Hongze Cheng 已提交
371
    goto _err;
H
Hongze Cheng 已提交
372 373 374
  } else if (n < size) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _err;
H
Hongze Cheng 已提交
375 376 377 378 379 380 381 382 383
  }

  // check
  if (!taosCheckChecksumWhole(*ppBuf, size)) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _err;
  }

  // decode
H
Hongze Cheng 已提交
384 385 386 387 388
  n = 0;
  n += tGetU32(*ppBuf + n, &delimiter);
  ASSERT(delimiter == TSDB_FILE_DLMT);
  n += tGetMapData(*ppBuf + n, pDelIdxMap);
  ASSERT(n + sizeof(TSCKSUM) == size);
H
Hongze Cheng 已提交
389 390 391 392

  return code;

_err:
H
Hongze Cheng 已提交
393
  tsdbError("vgId:%d read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
394
  return code;
H
Hongze Cheng 已提交
395 396 397 398 399 400
}

// SDataFReader ====================================================
struct SDataFReader {
  STsdb     *pTsdb;
  SDFileSet *pSet;
H
Hongze Cheng 已提交
401 402 403 404
  TdFilePtr  pHeadFD;
  TdFilePtr  pDataFD;
  TdFilePtr  pLastFD;
  TdFilePtr  pSmaFD;
H
Hongze Cheng 已提交
405 406 407
};

int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
H
Hongze Cheng 已提交
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
  int32_t       code = 0;
  SDataFReader *pReader;
  char          fname[TSDB_FILENAME_LEN];

  // alloc
  pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
  if (pReader == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
  }
  pReader->pTsdb = pTsdb;
  pReader->pSet = pSet;

  // open impl
  // head
  tsdbDataFileName(pTsdb, pSet, TSDB_HEAD_FILE, fname);
  pReader->pHeadFD = taosOpenFile(fname, TD_FILE_READ);
  if (pReader->pHeadFD == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // data
  tsdbDataFileName(pTsdb, pSet, TSDB_DATA_FILE, fname);
  pReader->pDataFD = taosOpenFile(fname, TD_FILE_READ);
  if (pReader->pDataFD == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // last
  tsdbDataFileName(pTsdb, pSet, TSDB_LAST_FILE, fname);
  pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ);
  if (pReader->pLastFD == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // sma
  tsdbDataFileName(pTsdb, pSet, TSDB_SMA_FILE, fname);
  pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ);
  if (pReader->pSmaFD == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  *ppReader = pReader;
  return code;

_err:
  tsdbError("vgId:%d tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
  *ppReader = NULL;
H
Hongze Cheng 已提交
460 461 462
  return code;
}

H
Hongze Cheng 已提交
463
int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
H
Hongze Cheng 已提交
464
  int32_t code = 0;
H
Hongze Cheng 已提交
465

H
Hongze Cheng 已提交
466
  if (taosCloseFile(&(*ppReader)->pHeadFD) < 0) {
H
Hongze Cheng 已提交
467 468 469 470
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
471
  if (taosCloseFile(&(*ppReader)->pDataFD) < 0) {
H
Hongze Cheng 已提交
472 473 474 475
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
476
  if (taosCloseFile(&(*ppReader)->pLastFD) < 0) {
H
Hongze Cheng 已提交
477 478 479 480
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
481
  if (taosCloseFile(&(*ppReader)->pSmaFD) < 0) {
H
Hongze Cheng 已提交
482 483 484 485
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
486 487
  taosMemoryFree(*ppReader);
  *ppReader = NULL;
H
Hongze Cheng 已提交
488 489 490
  return code;

_err:
H
Hongze Cheng 已提交
491
  tsdbError("vgId:%d data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
492 493 494
  return code;
}

H
Hongze Cheng 已提交
495
int32_t tsdbReadBlockIdx(SDataFReader *pReader, SMapData *mBlockIdx, uint8_t **ppBuf) {
H
Hongze Cheng 已提交
496
  int32_t  code = 0;
H
Hongze Cheng 已提交
497 498
  int64_t  offset = pReader->pSet->fHead.offset;
  int64_t  size = pReader->pSet->fHead.size - offset;
H
Hongze Cheng 已提交
499 500 501 502
  int64_t  n;
  uint32_t delimiter;

  // alloc
H
Hongze Cheng 已提交
503
  if (!ppBuf) ppBuf = &mBlockIdx->pBuf;
H
Hongze Cheng 已提交
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
  code = tsdbRealloc(ppBuf, size);
  if (code) goto _err;

  // seek
  if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // read
  n = taosReadFile(pReader->pHeadFD, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  } else if (n < size) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _err;
  }

  // check
  if (!taosCheckChecksumWhole(*ppBuf, size)) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _err;
  }

  // decode
  n = 0;
  n += tGetU32(*ppBuf + n, &delimiter);
  ASSERT(delimiter == TSDB_FILE_DLMT);
H
Hongze Cheng 已提交
533
  n += tGetMapData(*ppBuf + n, mBlockIdx);
H
Hongze Cheng 已提交
534 535 536 537 538 539
  ASSERT(n + sizeof(TSCKSUM) == size);

  return code;

_err:
  tsdbError("vgId:%d read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
540 541 542
  return code;
}

H
Hongze Cheng 已提交
543
int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock, uint8_t **ppBuf) {
H
Hongze Cheng 已提交
544 545 546 547 548
  int32_t  code = 0;
  int64_t  offset = pBlockIdx->offset;
  int64_t  size = pBlockIdx->size;
  int64_t  n;
  uint32_t delimiter;
H
Hongze Cheng 已提交
549 550
  int64_t  suid;
  int64_t  uid;
H
Hongze Cheng 已提交
551 552

  // alloc
H
Hongze Cheng 已提交
553
  if (!ppBuf) ppBuf = &mBlock->pBuf;
H
Hongze Cheng 已提交
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
  code = tsdbRealloc(ppBuf, size);
  if (code) goto _err;

  // seek
  if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // read
  n = taosReadFile(pReader->pHeadFD, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  } else if (n < size) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _err;
  }

  // check
  if (!taosCheckChecksumWhole(*ppBuf, size)) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _err;
  }

  // decode
  n = 0;
  n += tGetU32(*ppBuf + n, &delimiter);
  ASSERT(delimiter == TSDB_FILE_DLMT);
  n += tGetI64(*ppBuf + n, &suid);
  ASSERT(suid == pBlockIdx->suid);
  n += tGetI64(*ppBuf + n, &uid);
  ASSERT(uid == pBlockIdx->uid);
H
Hongze Cheng 已提交
587
  n += tGetMapData(*ppBuf + n, mBlock);
H
Hongze Cheng 已提交
588 589 590 591 592 593
  ASSERT(n + sizeof(TSCKSUM) == size);

  return code;

_err:
  tsdbError("vgId:%d read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
594 595 596
  return code;
}

H
Hongze Cheng 已提交
597 598
int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBlock, SBlockData *pBlockData,
                          int16_t *aColId, int32_t nCol, uint8_t **ppBuf1, uint8_t **ppBuf2) {
H
Hongze Cheng 已提交
599 600 601 602 603 604 605 606 607 608 609 610 611
  int32_t code = 0;
  // TODO
  return code;
}

int32_t tsdbReadBlockSMA(SDataFReader *pReader, SBlockSMA *pBlkSMA) {
  int32_t code = 0;
  // TODO
  return code;
}

// SDataFWriter ====================================================
struct SDataFWriter {
H
Hongze Cheng 已提交
612 613 614 615 616 617
  STsdb    *pTsdb;
  SDFileSet wSet;
  TdFilePtr pHeadFD;
  TdFilePtr pDataFD;
  TdFilePtr pLastFD;
  TdFilePtr pSmaFD;
H
Hongze Cheng 已提交
618 619
};

H
Hongze Cheng 已提交
620 621
SDFileSet *tsdbDataFWriterGetWSet(SDataFWriter *pWriter) { return &pWriter->wSet; }

H
Hongze Cheng 已提交
622
int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) {
H
Hongze Cheng 已提交
623 624 625 626 627 628 629 630 631 632 633 634 635 636
  int32_t       code = 0;
  int32_t       flag;
  int64_t       n;
  SDataFWriter *pWriter = NULL;
  char          fname[TSDB_FILENAME_LEN];
  char          hdr[TSDB_FHDR_SIZE] = {0};

  // alloc
  pWriter = taosMemoryCalloc(1, sizeof(*pWriter));
  if (pWriter == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
  }
  pWriter->pTsdb = pTsdb;
H
Hongze Cheng 已提交
637 638
  pWriter->wSet = *pSet;
  pSet = &pWriter->wSet;
H
Hongze Cheng 已提交
639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754

  // head
  flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
  tsdbDataFileName(pTsdb, pSet, TSDB_HEAD_FILE, fname);
  pWriter->pHeadFD = taosOpenFile(fname, flag);
  if (pWriter->pHeadFD == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  ASSERT(n == TSDB_FHDR_SIZE);

  pSet->fHead.size += TSDB_FHDR_SIZE;

  // data
  if (pSet->fData.size == 0) {
    flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
  } else {
    flag = TD_FILE_WRITE;
  }
  tsdbDataFileName(pTsdb, pSet, TSDB_DATA_FILE, fname);
  pWriter->pDataFD = taosOpenFile(fname, flag);
  if (pWriter->pDataFD == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }
  if (pSet->fData.size == 0) {
    n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

    pSet->fData.size += TSDB_FHDR_SIZE;
  } else {
    n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

    ASSERT(n == pSet->fData.size);
  }

  // last
  if (pSet->fLast.size == 0) {
    flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
  } else {
    flag = TD_FILE_WRITE;
  }
  tsdbDataFileName(pTsdb, pSet, TSDB_LAST_FILE, fname);
  pWriter->pLastFD = taosOpenFile(fname, flag);
  if (pWriter->pLastFD == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }
  if (pSet->fLast.size == 0) {
    n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

    pSet->fLast.size += TSDB_FHDR_SIZE;
  } else {
    n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_END);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

    ASSERT(n == pSet->fLast.size);
  }

  // sma
  if (pSet->fSma.size == 0) {
    flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
  } else {
    flag = TD_FILE_WRITE;
  }
  tsdbDataFileName(pTsdb, pSet, TSDB_SMA_FILE, fname);
  pWriter->pSmaFD = taosOpenFile(fname, flag);
  if (pWriter->pSmaFD == NULL) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }
  if (pSet->fSma.size == 0) {
    n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

    pSet->fSma.size += TSDB_FHDR_SIZE;
  } else {
    n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

    ASSERT(n == pSet->fSma.size);
  }

  *ppWriter = pWriter;
  return code;

_err:
  tsdbError("vgId:%d tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
  *ppWriter = NULL;
H
Hongze Cheng 已提交
755 756 757
  return code;
}

H
Hongze Cheng 已提交
758
int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) {
H
Hongze Cheng 已提交
759
  int32_t code = 0;
H
Hongze Cheng 已提交
760
  STsdb  *pTsdb = (*ppWriter)->pTsdb;
H
Hongze Cheng 已提交
761 762

  if (sync) {
H
Hongze Cheng 已提交
763
    if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) {
H
Hongze Cheng 已提交
764 765 766 767
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

H
Hongze Cheng 已提交
768
    if (taosFsyncFile((*ppWriter)->pDataFD) < 0) {
H
Hongze Cheng 已提交
769 770 771 772
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

H
Hongze Cheng 已提交
773
    if (taosFsyncFile((*ppWriter)->pLastFD) < 0) {
H
Hongze Cheng 已提交
774 775 776 777
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }

H
Hongze Cheng 已提交
778
    if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) {
H
Hongze Cheng 已提交
779 780 781 782 783
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }
  }

H
Hongze Cheng 已提交
784
  if (taosCloseFile(&(*ppWriter)->pHeadFD) < 0) {
H
Hongze Cheng 已提交
785 786 787 788
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
789
  if (taosCloseFile(&(*ppWriter)->pDataFD) < 0) {
H
Hongze Cheng 已提交
790 791 792 793
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
794
  if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) {
H
Hongze Cheng 已提交
795 796 797 798
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
799
  if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) {
H
Hongze Cheng 已提交
800 801 802 803
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
804 805
  taosMemoryFree(*ppWriter);
  *ppWriter = NULL;
H
Hongze Cheng 已提交
806 807 808
  return code;

_err:
H
Hongze Cheng 已提交
809
  tsdbError("vgId:%d data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
810 811 812 813
  return code;
}

int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter, uint8_t **ppBuf) {
H
Hongze Cheng 已提交
814 815 816 817
  int32_t    code = 0;
  int64_t    size = TSDB_FHDR_SIZE;
  int64_t    n;
  uint8_t   *pBuf = NULL;
H
Hongze Cheng 已提交
818 819 820 821
  SHeadFile *pHeadFile = &pWriter->wSet.fHead;
  SDataFile *pDataFile = &pWriter->wSet.fData;
  SLastFile *pLastFile = &pWriter->wSet.fLast;
  SSmaFile  *pSmaFile = &pWriter->wSet.fSma;
H
Hongze Cheng 已提交
822 823 824 825 826 827 828 829 830

  // alloc
  if (!ppBuf) ppBuf = &pBuf;
  code = tsdbRealloc(ppBuf, size);
  if (code) goto _err;

  // head ==============
  // build
  memset(*ppBuf, 0, size);
H
Hongze Cheng 已提交
831
  tPutDataFileHdr(*ppBuf, &pWriter->wSet, TSDB_HEAD_FILE);
H
Hongze Cheng 已提交
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
  taosCalcChecksumAppend(0, *ppBuf, size);

  // seek
  if (taosLSeekFile(pWriter->pHeadFD, 0, SEEK_SET) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // write
  n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // data ==============
  memset(*ppBuf, 0, size);
H
Hongze Cheng 已提交
849
  tPutDataFileHdr(*ppBuf, &pWriter->wSet, TSDB_DATA_FILE);
H
Hongze Cheng 已提交
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866
  taosCalcChecksumAppend(0, *ppBuf, size);

  // seek
  if (taosLSeekFile(pWriter->pDataFD, 0, SEEK_SET) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // write
  n = taosWriteFile(pWriter->pDataFD, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // last ==============
  memset(*ppBuf, 0, size);
H
Hongze Cheng 已提交
867
  tPutDataFileHdr(*ppBuf, &pWriter->wSet, TSDB_LAST_FILE);
H
Hongze Cheng 已提交
868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
  taosCalcChecksumAppend(0, *ppBuf, size);

  // seek
  if (taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // write
  n = taosWriteFile(pWriter->pLastFD, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // sma ==============
  memset(*ppBuf, 0, size);
H
Hongze Cheng 已提交
885
  tPutDataFileHdr(*ppBuf, &pWriter->wSet, TSDB_SMA_FILE);
H
Hongze Cheng 已提交
886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
  taosCalcChecksumAppend(0, *ppBuf, size);

  // seek
  if (taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET) < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  // write
  n = taosWriteFile(pWriter->pSmaFD, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

  tsdbFree(pBuf);
  return code;

_err:
  tsdbFree(pBuf);
  tsdbError("vgId:%d update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
907 908 909
  return code;
}

H
Hongze Cheng 已提交
910 911
int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SMapData *mBlockIdx, uint8_t **ppBuf) {
  int32_t    code = 0;
H
Hongze Cheng 已提交
912
  int64_t    size;
H
Hongze Cheng 已提交
913
  SHeadFile *pHeadFile = &pWriter->wSet.fHead;
H
Hongze Cheng 已提交
914
  int64_t    n;
H
Hongze Cheng 已提交
915
  uint8_t   *pBuf = NULL;
H
Hongze Cheng 已提交
916 917

  // prepare
H
Hongze Cheng 已提交
918
  size = 0;
H
Hongze Cheng 已提交
919
  size += tPutU32(NULL, TSDB_FILE_DLMT);
H
Hongze Cheng 已提交
920
  size = size + tPutMapData(NULL, mBlockIdx) + sizeof(TSCKSUM);
H
Hongze Cheng 已提交
921 922 923 924 925 926 927

  // alloc
  if (!ppBuf) ppBuf = &pBuf;
  code = tsdbRealloc(ppBuf, size);
  if (code) goto _err;

  // build
H
Hongze Cheng 已提交
928
  n = 0;
H
Hongze Cheng 已提交
929
  n += tPutU32(*ppBuf + n, TSDB_FILE_DLMT);
H
Hongze Cheng 已提交
930
  n += tPutMapData(*ppBuf, mBlockIdx);
H
Hongze Cheng 已提交
931 932 933 934 935 936 937 938 939 940 941
  taosCalcChecksumAppend(0, *ppBuf, size);

  ASSERT(n + sizeof(TSCKSUM) == size);

  // write
  n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
942 943 944
  // update
  pHeadFile->offset = pHeadFile->size;
  pHeadFile->size += size;
H
Hongze Cheng 已提交
945 946 947 948 949 950 951

  tsdbFree(pBuf);
  return code;

_err:
  tsdbError("vgId:%d write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
  tsdbFree(pBuf);
H
Hongze Cheng 已提交
952 953 954
  return code;
}

H
Hongze Cheng 已提交
955
int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, uint8_t **ppBuf, SBlockIdx *pBlockIdx) {
H
Hongze Cheng 已提交
956 957 958 959 960 961
  int32_t       code = 0;
  SHeadFile    *pHeadFile = &pWriter->wSet.fHead;
  SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
  uint8_t      *pBuf = NULL;
  int64_t       size;
  int64_t       n;
H
Hongze Cheng 已提交
962

H
Hongze Cheng 已提交
963
  ASSERT(mBlock->nItem > 0);
H
Hongze Cheng 已提交
964 965

  // prepare
H
Hongze Cheng 已提交
966
  size = sizeof(SBlockDataHdr) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM);
H
Hongze Cheng 已提交
967 968 969 970 971 972 973 974

  // alloc
  if (!ppBuf) ppBuf = &pBuf;
  code = tsdbRealloc(ppBuf, size);
  if (code) goto _err;

  // build
  n = 0;
H
Hongze Cheng 已提交
975 976
  *(SBlockDataHdr *)(*ppBuf) = hdr;
  n += sizeof(hdr);
H
Hongze Cheng 已提交
977
  n += tPutMapData(*ppBuf + n, mBlock);
H
Hongze Cheng 已提交
978 979 980 981 982 983 984 985 986 987 988
  taosCalcChecksumAppend(0, *ppBuf, size);

  ASSERT(n + sizeof(TSCKSUM) == size);

  // write
  n = taosWriteFile(pWriter->pHeadFD, *ppBuf, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }

H
Hongze Cheng 已提交
989 990
  // update
  pBlockIdx->offset = pHeadFile->size;
H
Hongze Cheng 已提交
991
  pBlockIdx->size = size;
H
Hongze Cheng 已提交
992
  pHeadFile->size += size;
H
Hongze Cheng 已提交
993

H
Hongze Cheng 已提交
994
  tsdbFree(pBuf);
H
Hongze Cheng 已提交
995 996 997 998 999
  tsdbTrace("vgId:%d write block, offset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), pBlockIdx->offset,
            pBlockIdx->size);
  return code;

_err:
H
Hongze Cheng 已提交
1000
  tsdbFree(pBuf);
H
Hongze Cheng 已提交
1001
  tsdbError("vgId:%d write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
1002 1003 1004
  return code;
}

H
Hongze Cheng 已提交
1005 1006 1007 1008
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, uint8_t **ppBuf1, uint8_t **ppBuf2,
                           SBlockIdx *pBlockIdx, SBlock *pBlock) {
  int32_t       code = 0;
  SSubBlock    *pSubBlock = &pBlock->aSubBlock[pBlock->nSubBlock++];
H
Hongze Cheng 已提交
1009
  SBlockCol    *pBlockCol = &(SBlockCol){0};
H
Hongze Cheng 已提交
1010 1011
  int64_t       size;
  int64_t       n;
H
Hongze Cheng 已提交
1012
  TdFilePtr     pFileFD = pBlock->last ? pWriter->pLastFD : pWriter->pDataFD;
H
Hongze Cheng 已提交
1013 1014 1015 1016
  SBlockDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
  TSCKSUM       cksm;
  uint8_t      *p;
  int64_t       offset;
H
Hongze Cheng 已提交
1017 1018
  uint8_t      *pBuf1 = NULL;
  uint8_t      *pBuf2 = NULL;
H
Hongze Cheng 已提交
1019

H
Hongze Cheng 已提交
1020 1021
  if (!ppBuf1) ppBuf1 = &pBuf1;
  if (!ppBuf2) ppBuf2 = &pBuf2;
H
Hongze Cheng 已提交
1022

H
Hongze Cheng 已提交
1023 1024 1025 1026 1027
  if (pBlock->last) {
    pSubBlock->offset = pWriter->wSet.fLast.size;
  } else {
    pSubBlock->offset = pWriter->wSet.fData.size;
  }
H
Hongze Cheng 已提交
1028
  pSubBlock->bsize = 0;
H
Hongze Cheng 已提交
1029 1030 1031 1032 1033 1034 1035 1036

  // HDR
  n = taosWriteFile(pFileFD, &hdr, sizeof(hdr));
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _err;
  }
  pSubBlock->bsize += n;
H
Hongze Cheng 已提交
1037 1038

  // TSDBKEY
H
Hongze Cheng 已提交
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
  pSubBlock->ksize = 0;
  if (pBlock->cmprAlg == NO_COMPRESSION) {
    // TSKEY
    size = sizeof(TSKEY) * pBlockData->nRow;
    n = taosWriteFile(pFileFD, pBlockData->aTSKEY, size);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }
    pSubBlock->ksize += size;
    cksm = taosCalcChecksum(0, (uint8_t *)pBlockData->aTSKEY, size);

    // version
    size = sizeof(int64_t) * pBlockData->nRow;
    n = taosWriteFile(pFileFD, pBlockData->aVersion, size);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }
    pSubBlock->ksize += size;
    cksm = taosCalcChecksum(cksm, (uint8_t *)pBlockData->aVersion, size);

    // cksm
    size = sizeof(cksm);
    n = taosWriteFile(pFileFD, (uint8_t *)&cksm, size);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }
    pSubBlock->ksize += size;
  } else {
    ASSERT(pBlock->cmprAlg == ONE_STAGE_COMP || pBlock->cmprAlg == TWO_STAGE_COMP);

    size = (sizeof(TSKEY) + sizeof(int64_t)) * pBlockData->nRow + COMP_OVERFLOW_BYTES * 2 + sizeof(TSCKSUM);

    code = tsdbRealloc(ppBuf1, size);
    if (code) goto _err;

    if (pBlock->cmprAlg == TWO_STAGE_COMP) {
      code = tsdbRealloc(ppBuf2, size);
      if (code) goto _err;
    }

    // TSKEY
H
more  
Hongze Cheng 已提交
1083 1084
    n = tsCompressTimestamp((char *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, pBlockData->nRow, *ppBuf1,
                            size, pBlock->cmprAlg, *ppBuf2, size);
H
Hongze Cheng 已提交
1085 1086 1087 1088 1089 1090 1091
    if (n <= 0) {
      code = TSDB_CODE_COMPRESS_ERROR;
      goto _err;
    }
    pSubBlock->ksize += n;

    // version
H
more  
Hongze Cheng 已提交
1092
    n = tsCompressBigint((char *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, pBlockData->nRow,
H
Hongze Cheng 已提交
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
                         *ppBuf1 + pSubBlock->ksize, size - pSubBlock->ksize, pBlock->cmprAlg, *ppBuf2, size);
    if (n <= 0) {
      code = TSDB_CODE_COMPRESS_ERROR;
      goto _err;
    }
    pSubBlock->ksize += n;

    // cksm
    pSubBlock->ksize += sizeof(TSCKSUM);
    ASSERT(pSubBlock->ksize <= size);
    taosCalcChecksumAppend(0, *ppBuf1, pSubBlock->ksize);

    // write
    n = taosWriteFile(pFileFD, *ppBuf1, pSubBlock->ksize);
    if (n < 0) {
      code = TAOS_SYSTEM_ERROR(errno);
      goto _err;
    }
  }
  pSubBlock->bsize += pSubBlock->ksize;
H
Hongze Cheng 已提交
1113 1114

  // other columns
H
Hongze Cheng 已提交
1115
  offset = 0;
H
Hongze Cheng 已提交
1116
  tMapDataReset(&pSubBlock->mBlockCol);
H
Hongze Cheng 已提交
1117 1118
  for (int32_t iCol = 0; iCol < taosArrayGetSize(pBlockData->aColDataP); iCol++) {
    SColData *pColData = (SColData *)taosArrayGetP(pBlockData->aColDataP, iCol);
H
Hongze Cheng 已提交
1119

H
Hongze Cheng 已提交
1120
    ASSERT(pColData->flag);
H
Hongze Cheng 已提交
1121

H
Hongze Cheng 已提交
1122
    if (pColData->flag == HAS_NONE) continue;
H
Hongze Cheng 已提交
1123

H
Hongze Cheng 已提交
1124 1125 1126
    pBlockCol->cid = pColData->cid;
    pBlockCol->type = pColData->type;
    pBlockCol->flag = pColData->flag;
H
Hongze Cheng 已提交
1127

H
Hongze Cheng 已提交
1128
    if (pColData->flag != HAS_NULL) {
H
Hongze Cheng 已提交
1129
      cksm = 0;
H
Hongze Cheng 已提交
1130 1131
      pBlockCol->offset = offset;
      pBlockCol->size = 0;
H
Hongze Cheng 已提交
1132 1133

      // bitmap
H
Hongze Cheng 已提交
1134
      if (pColData->flag != HAS_VALUE) {
H
Hongze Cheng 已提交
1135
        // optimize bitmap storage (todo)
H
Hongze Cheng 已提交
1136 1137 1138 1139 1140 1141 1142
        n = taosWriteFile(pFileFD, pColData->pBitMap, BIT2_SIZE(pBlockData->nRow));
        if (n < 0) {
          code = TAOS_SYSTEM_ERROR(errno);
          goto _err;
        }

        cksm = taosCalcChecksum(cksm, pColData->pBitMap, n);
H
Hongze Cheng 已提交
1143
        pBlockCol->size += n;
H
Hongze Cheng 已提交
1144 1145
      }

H
Hongze Cheng 已提交
1146 1147 1148 1149 1150 1151 1152 1153
      // data
      if (pBlock->cmprAlg == NO_COMPRESSION) {
        // data
        n = taosWriteFile(pFileFD, pColData->pData, pColData->nData);
        if (n < 0) {
          code = TAOS_SYSTEM_ERROR(errno);
          goto _err;
        }
H
Hongze Cheng 已提交
1154
        pBlockCol->size += n;
H
Hongze Cheng 已提交
1155 1156 1157 1158 1159 1160 1161 1162

        // checksum
        cksm = taosCalcChecksum(cksm, pColData->pData, pColData->nData);
        n = taosWriteFile(pFileFD, &cksm, sizeof(cksm));
        if (n < 0) {
          code = TAOS_SYSTEM_ERROR(errno);
          goto _err;
        }
H
Hongze Cheng 已提交
1163
        pBlockCol->size += n;
H
Hongze Cheng 已提交
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
      } else {
        size = pColData->nData + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM);

        code = tsdbRealloc(ppBuf1, size);
        if (code) goto _err;

        if (pBlock->cmprAlg == TWO_STAGE_COMP) {
          code = tsdbRealloc(ppBuf2, size);
          if (code) goto _err;
        }

        // data
H
Hongze Cheng 已提交
1176 1177
        n = tDataTypes[pColData->type].compFunc(pColData->pData, pColData->nData, pBlockData->nRow, *ppBuf1, size,
                                                pBlock->cmprAlg, *ppBuf2, size);
H
Hongze Cheng 已提交
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188
        if (n <= 0) {
          code = TSDB_CODE_COMPRESS_ERROR;
          goto _err;
        }

        // cksm
        n += sizeof(TSCKSUM);
        ASSERT(n <= size);
        taosCalcChecksumAppend(cksm, *ppBuf1, n);

        // write
H
Hongze Cheng 已提交
1189
        n = taosWriteFile(pFileFD, *ppBuf1, n);
H
Hongze Cheng 已提交
1190 1191 1192 1193
        if (n < 0) {
          code = TAOS_SYSTEM_ERROR(errno);
          goto _err;
        }
H
Hongze Cheng 已提交
1194 1195

        pBlockCol->size += n;
H
Hongze Cheng 已提交
1196
      }
H
Hongze Cheng 已提交
1197

H
Hongze Cheng 已提交
1198
      // state
H
Hongze Cheng 已提交
1199 1200
      offset += pBlockCol->size;
      pSubBlock->bsize += pBlockCol->size;
H
Hongze Cheng 已提交
1201 1202
    }

H
Hongze Cheng 已提交
1203
    code = tMapDataPutItem(&pSubBlock->mBlockCol, pBlockCol, tPutBlockCol);
H
Hongze Cheng 已提交
1204 1205 1206
    if (code) goto _err;
  }

H
Hongze Cheng 已提交
1207 1208 1209 1210 1211 1212 1213 1214
  if (pBlock->last) {
    pWriter->wSet.fLast.size += pSubBlock->bsize;
  } else {
    pWriter->wSet.fData.size += pSubBlock->bsize;
  }

  tsdbFree(pBuf1);
  tsdbFree(pBuf2);
H
Hongze Cheng 已提交
1215 1216 1217
  return code;

_err:
H
Hongze Cheng 已提交
1218
  tsdbError("vgId:%d write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
H
Hongze Cheng 已提交
1219 1220
  tsdbFree(pBuf1);
  tsdbFree(pBuf2);
H
Hongze Cheng 已提交
1221 1222 1223 1224 1225 1226 1227 1228
  return code;
}

int32_t tsdbWriteBlockSMA(SDataFWriter *pWriter, SBlockSMA *pBlockSMA, int64_t *rOffset, int64_t *rSize) {
  int32_t code = 0;
  // TODO
  return code;
}