tsdbUtil.c 58.5 KB
Newer Older
H
Hongze Cheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16
#include "tdataformat.h"
H
Hongze Cheng 已提交
17 18
#include "tsdb.h"

H
Hongze Cheng 已提交
19
// SMapData =======================================================================
H
Hongze Cheng 已提交
20 21 22 23
void tMapDataReset(SMapData *pMapData) {
  pMapData->nItem = 0;
  pMapData->nData = 0;
}
H
Hongze Cheng 已提交
24

H
Hongze Cheng 已提交
25
void tMapDataClear(SMapData *pMapData) {
H
Hongze Cheng 已提交
26 27
  tFree((uint8_t *)pMapData->aOffset);
  tFree(pMapData->pData);
28 29
  pMapData->pData = NULL;
  pMapData->aOffset = NULL;
H
Hongze Cheng 已提交
30 31
}

H
Hongze Cheng 已提交
32 33 34 35 36 37 38 39 40
int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *)) {
  int32_t code = 0;
  int32_t offset = pMapData->nData;
  int32_t nItem = pMapData->nItem;

  pMapData->nItem++;
  pMapData->nData += tPutItemFn(NULL, pItem);

  // alloc
H
Hongze Cheng 已提交
41
  code = tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem);
H
Hongze Cheng 已提交
42
  if (code) goto _exit;
H
Hongze Cheng 已提交
43
  code = tRealloc(&pMapData->pData, pMapData->nData);
H
Hongze Cheng 已提交
44
  if (code) goto _exit;
H
Hongze Cheng 已提交
45 46

  // put
H
Hongze Cheng 已提交
47
  pMapData->aOffset[nItem] = offset;
H
Hongze Cheng 已提交
48 49
  tPutItemFn(pMapData->pData + offset, pItem);

H
Hongze Cheng 已提交
50
_exit:
H
Hongze Cheng 已提交
51 52 53
  return code;
}

H
Hongze Cheng 已提交
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
                       int32_t (*tItemCmprFn)(const void *, const void *), void *pItem) {
  int32_t code = 0;
  int32_t lidx = 0;
  int32_t ridx = pMapData->nItem - 1;
  int32_t midx;
  int32_t c;

  while (lidx <= ridx) {
    midx = (lidx + ridx) / 2;

    tMapDataGetItemByIdx(pMapData, midx, pItem, tGetItemFn);

    c = tItemCmprFn(pSearchItem, pItem);
    if (c == 0) {
      goto _exit;
    } else if (c < 0) {
      ridx = midx - 1;
    } else {
      lidx = midx + 1;
    }
  }

  code = TSDB_CODE_NOT_FOUND;

_exit:
  return code;
}
H
Hongze Cheng 已提交
82

H
Hongze Cheng 已提交
83 84 85
void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *)) {
  ASSERT(idx >= 0 && idx < pMapData->nItem);
  tGetItemFn(pMapData->pData + pMapData->aOffset[idx], pItem);
H
Hongze Cheng 已提交
86 87 88 89 90 91
}

int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {
  int32_t n = 0;

  n += tPutI32v(p ? p + n : p, pMapData->nItem);
H
Hongze Cheng 已提交
92
  if (pMapData->nItem) {
H
Hongze Cheng 已提交
93
    int32_t lOffset = 0;
H
Hongze Cheng 已提交
94
    for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
H
Hongze Cheng 已提交
95 96
      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset);
      lOffset = pMapData->aOffset[iItem];
H
Hongze Cheng 已提交
97
    }
H
Hongze Cheng 已提交
98 99 100 101 102 103

    n += tPutI32v(p ? p + n : p, pMapData->nData);
    if (p) {
      memcpy(p + n, pMapData->pData, pMapData->nData);
    }
    n += pMapData->nData;
H
Hongze Cheng 已提交
104 105 106 107 108 109 110
  }

  return n;
}

int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
  int32_t n = 0;
H
Hongze Cheng 已提交
111 112 113
  int32_t offset;

  tMapDataReset(pMapData);
H
Hongze Cheng 已提交
114 115

  n += tGetI32v(p + n, &pMapData->nItem);
H
Hongze Cheng 已提交
116
  if (pMapData->nItem) {
H
Hongze Cheng 已提交
117
    if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1;
H
Hongze Cheng 已提交
118

H
Hongze Cheng 已提交
119
    int32_t lOffset = 0;
H
Hongze Cheng 已提交
120 121
    for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
      n += tGetI32v(p + n, &pMapData->aOffset[iItem]);
H
Hongze Cheng 已提交
122 123
      pMapData->aOffset[iItem] += lOffset;
      lOffset = pMapData->aOffset[iItem];
H
Hongze Cheng 已提交
124
    }
H
Hongze Cheng 已提交
125 126

    n += tGetI32v(p + n, &pMapData->nData);
H
Hongze Cheng 已提交
127
    if (tRealloc(&pMapData->pData, pMapData->nData)) return -1;
H
Hongze Cheng 已提交
128 129
    memcpy(pMapData->pData, p + n, pMapData->nData);
    n += pMapData->nData;
H
Hongze Cheng 已提交
130 131 132 133 134
  }

  return n;
}

H
Hongze Cheng 已提交
135
// TABLEID =======================================================================
H
Hongze Cheng 已提交
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
int32_t tTABLEIDCmprFn(const void *p1, const void *p2) {
  TABLEID *pId1 = (TABLEID *)p1;
  TABLEID *pId2 = (TABLEID *)p2;

  if (pId1->suid < pId2->suid) {
    return -1;
  } else if (pId1->suid > pId2->suid) {
    return 1;
  }

  if (pId1->uid < pId2->uid) {
    return -1;
  } else if (pId1->uid > pId2->uid) {
    return 1;
  }

  return 0;
}

H
Hongze Cheng 已提交
155
// SBlockIdx ======================================================
H
Hongze Cheng 已提交
156 157 158 159 160 161 162 163
int32_t tPutBlockIdx(uint8_t *p, void *ph) {
  int32_t    n = 0;
  SBlockIdx *pBlockIdx = (SBlockIdx *)ph;

  n += tPutI64(p ? p + n : p, pBlockIdx->suid);
  n += tPutI64(p ? p + n : p, pBlockIdx->uid);
  n += tPutI64v(p ? p + n : p, pBlockIdx->offset);
  n += tPutI64v(p ? p + n : p, pBlockIdx->size);
H
Hongze Cheng 已提交
164 165 166 167

  return n;
}

H
Hongze Cheng 已提交
168 169 170
int32_t tGetBlockIdx(uint8_t *p, void *ph) {
  int32_t    n = 0;
  SBlockIdx *pBlockIdx = (SBlockIdx *)ph;
H
Hongze Cheng 已提交
171

H
Hongze Cheng 已提交
172 173 174 175
  n += tGetI64(p + n, &pBlockIdx->suid);
  n += tGetI64(p + n, &pBlockIdx->uid);
  n += tGetI64v(p + n, &pBlockIdx->offset);
  n += tGetI64v(p + n, &pBlockIdx->size);
H
Hongze Cheng 已提交
176 177 178 179

  return n;
}

180
int32_t tCmprBlockIdx(void const *lhs, void const *rhs) {
M
Minglei Jin 已提交
181 182
  SBlockIdx *lBlockIdx = (SBlockIdx *)lhs;
  SBlockIdx *rBlockIdx = (SBlockIdx *)rhs;
183

M
Minglei Jin 已提交
184
  if (lBlockIdx->suid < rBlockIdx->suid) {
185
    return -1;
M
Minglei Jin 已提交
186
  } else if (lBlockIdx->suid > rBlockIdx->suid) {
187 188 189
    return 1;
  }

M
Minglei Jin 已提交
190
  if (lBlockIdx->uid < rBlockIdx->uid) {
191
    return -1;
M
Minglei Jin 已提交
192
  } else if (lBlockIdx->uid > rBlockIdx->uid) {
193 194 195 196 197 198
    return 1;
  }

  return 0;
}

199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
int32_t tCmprBlockL(void const *lhs, void const *rhs) {
  SBlockIdx *lBlockIdx = (SBlockIdx *)lhs;
  SBlockL   *rBlockL = (SBlockL *)rhs;

  if (lBlockIdx->suid < rBlockL->suid) {
    return -1;
  } else if (lBlockIdx->suid > rBlockL->suid) {
    return 1;
  }

  if (lBlockIdx->uid < rBlockL->minUid) {
    return -1;
  } else if (lBlockIdx->uid > rBlockL->maxUid) {
    return 1;
  }

  return 0;
}

H
Hongze Cheng 已提交
218
// SBlock ======================================================
H
Hongze Cheng 已提交
219
void tBlockReset(SBlock *pBlock) {
H
Hongze Cheng 已提交
220
  *pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN};
H
Hongze Cheng 已提交
221 222
}

H
Hongze Cheng 已提交
223 224
int32_t tPutBlock(uint8_t *p, void *ph) {
  int32_t n = 0;
H
Hongze Cheng 已提交
225
  SBlock *pBlock = (SBlock *)ph;
H
Hongze Cheng 已提交
226

H
Hongze Cheng 已提交
227
  n += tPutI64v(p ? p + n : p, pBlock->minKey.version);
H
Hongze Cheng 已提交
228
  n += tPutI64v(p ? p + n : p, pBlock->minKey.ts);
H
Hongze Cheng 已提交
229
  n += tPutI64v(p ? p + n : p, pBlock->maxKey.version);
H
Hongze Cheng 已提交
230
  n += tPutI64v(p ? p + n : p, pBlock->maxKey.ts);
H
Hongze Cheng 已提交
231 232
  n += tPutI64v(p ? p + n : p, pBlock->minVer);
  n += tPutI64v(p ? p + n : p, pBlock->maxVer);
H
Hongze Cheng 已提交
233 234 235 236 237
  n += tPutI32v(p ? p + n : p, pBlock->nRow);
  n += tPutI8(p ? p + n : p, pBlock->hasDup);
  n += tPutI8(p ? p + n : p, pBlock->nSubBlock);
  for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
    n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].offset);
H
Hongze Cheng 已提交
238
    n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlock);
H
Hongze Cheng 已提交
239 240 241 242 243
    n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szKey);
  }
  if (pBlock->nSubBlock == 1 && !pBlock->hasDup) {
    n += tPutI64v(p ? p + n : p, pBlock->smaInfo.offset);
    n += tPutI32v(p ? p + n : p, pBlock->smaInfo.size);
H
Hongze Cheng 已提交
244 245
  }

H
Hongze Cheng 已提交
246 247 248 249 250
  return n;
}

int32_t tGetBlock(uint8_t *p, void *ph) {
  int32_t n = 0;
H
Hongze Cheng 已提交
251
  SBlock *pBlock = (SBlock *)ph;
H
Hongze Cheng 已提交
252

H
Hongze Cheng 已提交
253
  n += tGetI64v(p + n, &pBlock->minKey.version);
H
Hongze Cheng 已提交
254
  n += tGetI64v(p + n, &pBlock->minKey.ts);
H
Hongze Cheng 已提交
255
  n += tGetI64v(p + n, &pBlock->maxKey.version);
H
Hongze Cheng 已提交
256
  n += tGetI64v(p + n, &pBlock->maxKey.ts);
H
Hongze Cheng 已提交
257 258
  n += tGetI64v(p + n, &pBlock->minVer);
  n += tGetI64v(p + n, &pBlock->maxVer);
H
Hongze Cheng 已提交
259 260 261 262 263
  n += tGetI32v(p + n, &pBlock->nRow);
  n += tGetI8(p + n, &pBlock->hasDup);
  n += tGetI8(p + n, &pBlock->nSubBlock);
  for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
    n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].offset);
H
Hongze Cheng 已提交
264
    n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlock);
H
Hongze Cheng 已提交
265 266 267 268 269 270 271 272
    n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szKey);
  }
  if (pBlock->nSubBlock == 1 && !pBlock->hasDup) {
    n += tGetI64v(p + n, &pBlock->smaInfo.offset);
    n += tGetI32v(p + n, &pBlock->smaInfo.size);
  } else {
    pBlock->smaInfo.offset = 0;
    pBlock->smaInfo.size = 0;
H
Hongze Cheng 已提交
273 274
  }

H
Hongze Cheng 已提交
275 276 277
  return n;
}

H
Hongze Cheng 已提交
278 279 280 281
int32_t tBlockCmprFn(const void *p1, const void *p2) {
  SBlock *pBlock1 = (SBlock *)p1;
  SBlock *pBlock2 = (SBlock *)p2;

H
Hongze Cheng 已提交
282
  if (tsdbKeyCmprFn(&pBlock1->maxKey, &pBlock2->minKey) < 0) {
H
Hongze Cheng 已提交
283
    return -1;
H
Hongze Cheng 已提交
284
  } else if (tsdbKeyCmprFn(&pBlock1->minKey, &pBlock2->maxKey) > 0) {
H
Hongze Cheng 已提交
285
    return 1;
H
Hongze Cheng 已提交
286 287 288 289 290
  }

  return 0;
}

H
Hongze Cheng 已提交
291 292 293 294
bool tBlockHasSma(SBlock *pBlock) {
  if (pBlock->nSubBlock > 1) return false;
  if (pBlock->hasDup) return false;

H
Hongze Cheng 已提交
295
  return pBlock->smaInfo.size > 0;
H
Hongze Cheng 已提交
296
}
H
Hongze Cheng 已提交
297

H
Hongze Cheng 已提交
298 299 300 301 302 303
// SBlockL ======================================================
int32_t tPutBlockL(uint8_t *p, void *ph) {
  int32_t  n = 0;
  SBlockL *pBlockL = (SBlockL *)ph;

  n += tPutI64(p ? p + n : p, pBlockL->suid);
H
Hongze Cheng 已提交
304 305
  n += tPutI64(p ? p + n : p, pBlockL->minUid);
  n += tPutI64(p ? p + n : p, pBlockL->maxUid);
H
Hongze Cheng 已提交
306 307
  n += tPutI64v(p ? p + n : p, pBlockL->minKey);
  n += tPutI64v(p ? p + n : p, pBlockL->maxKey);
H
Hongze Cheng 已提交
308 309 310
  n += tPutI64v(p ? p + n : p, pBlockL->minVer);
  n += tPutI64v(p ? p + n : p, pBlockL->maxVer);
  n += tPutI32v(p ? p + n : p, pBlockL->nRow);
H
Hongze Cheng 已提交
311 312 313
  n += tPutI64v(p ? p + n : p, pBlockL->bInfo.offset);
  n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szBlock);
  n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szKey);
H
Hongze Cheng 已提交
314 315 316 317 318 319 320 321 322

  return n;
}

int32_t tGetBlockL(uint8_t *p, void *ph) {
  int32_t  n = 0;
  SBlockL *pBlockL = (SBlockL *)ph;

  n += tGetI64(p + n, &pBlockL->suid);
H
Hongze Cheng 已提交
323 324
  n += tGetI64(p + n, &pBlockL->minUid);
  n += tGetI64(p + n, &pBlockL->maxUid);
H
Hongze Cheng 已提交
325 326
  n += tGetI64v(p + n, &pBlockL->minKey);
  n += tGetI64v(p + n, &pBlockL->maxKey);
H
Hongze Cheng 已提交
327 328 329
  n += tGetI64v(p + n, &pBlockL->minVer);
  n += tGetI64v(p + n, &pBlockL->maxVer);
  n += tGetI32v(p + n, &pBlockL->nRow);
H
Hongze Cheng 已提交
330 331 332
  n += tGetI64v(p + n, &pBlockL->bInfo.offset);
  n += tGetI32v(p + n, &pBlockL->bInfo.szBlock);
  n += tGetI32v(p + n, &pBlockL->bInfo.szKey);
H
Hongze Cheng 已提交
333 334 335

  return n;
}
H
Hongze Cheng 已提交
336

H
Hongze Cheng 已提交
337 338 339 340 341 342 343 344 345
// SBlockCol ======================================================
int32_t tPutBlockCol(uint8_t *p, void *ph) {
  int32_t    n = 0;
  SBlockCol *pBlockCol = (SBlockCol *)ph;

  ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE));

  n += tPutI16v(p ? p + n : p, pBlockCol->cid);
  n += tPutI8(p ? p + n : p, pBlockCol->type);
H
Hongze Cheng 已提交
346
  n += tPutI8(p ? p + n : p, pBlockCol->smaOn);
H
Hongze Cheng 已提交
347
  n += tPutI8(p ? p + n : p, pBlockCol->flag);
H
Hongze Cheng 已提交
348
  n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin);
H
Hongze Cheng 已提交
349 350

  if (pBlockCol->flag != HAS_NULL) {
H
Hongze Cheng 已提交
351 352 353 354 355 356 357 358 359 360 361 362
    if (pBlockCol->flag != HAS_VALUE) {
      n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap);
    }

    if (IS_VAR_DATA_TYPE(pBlockCol->type)) {
      n += tPutI32v(p ? p + n : p, pBlockCol->szOffset);
    }

    if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) {
      n += tPutI32v(p ? p + n : p, pBlockCol->szValue);
    }

H
Hongze Cheng 已提交
363
    n += tPutI32v(p ? p + n : p, pBlockCol->offset);
H
Hongze Cheng 已提交
364 365
  }

H
Hongze Cheng 已提交
366
_exit:
H
Hongze Cheng 已提交
367 368 369 370 371 372 373 374 375
  return n;
}

int32_t tGetBlockCol(uint8_t *p, void *ph) {
  int32_t    n = 0;
  SBlockCol *pBlockCol = (SBlockCol *)ph;

  n += tGetI16v(p + n, &pBlockCol->cid);
  n += tGetI8(p + n, &pBlockCol->type);
H
Hongze Cheng 已提交
376
  n += tGetI8(p + n, &pBlockCol->smaOn);
H
Hongze Cheng 已提交
377
  n += tGetI8(p + n, &pBlockCol->flag);
H
Hongze Cheng 已提交
378
  n += tGetI32v(p + n, &pBlockCol->szOrigin);
H
Hongze Cheng 已提交
379 380 381

  ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE));

H
Hongze Cheng 已提交
382 383 384 385 386
  pBlockCol->szBitmap = 0;
  pBlockCol->szOffset = 0;
  pBlockCol->szValue = 0;
  pBlockCol->offset = 0;

H
Hongze Cheng 已提交
387
  if (pBlockCol->flag != HAS_NULL) {
H
Hongze Cheng 已提交
388 389 390 391 392 393 394 395 396 397 398 399
    if (pBlockCol->flag != HAS_VALUE) {
      n += tGetI32v(p + n, &pBlockCol->szBitmap);
    }

    if (IS_VAR_DATA_TYPE(pBlockCol->type)) {
      n += tGetI32v(p + n, &pBlockCol->szOffset);
    }

    if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) {
      n += tGetI32v(p + n, &pBlockCol->szValue);
    }

H
Hongze Cheng 已提交
400
    n += tGetI32v(p + n, &pBlockCol->offset);
H
Hongze Cheng 已提交
401 402 403 404 405
  }

  return n;
}

H
Hongze Cheng 已提交
406 407 408 409 410 411 412 413 414 415
int32_t tBlockColCmprFn(const void *p1, const void *p2) {
  if (((SBlockCol *)p1)->cid < ((SBlockCol *)p2)->cid) {
    return -1;
  } else if (((SBlockCol *)p1)->cid > ((SBlockCol *)p2)->cid) {
    return 1;
  }

  return 0;
}

H
Hongze Cheng 已提交
416
// SDelIdx ======================================================
417
int32_t tCmprDelIdx(void const *lhs, void const *rhs) {
M
Minglei Jin 已提交
418 419
  SDelIdx *lDelIdx = (SDelIdx *)lhs;
  SDelIdx *rDelIdx = (SDelIdx *)rhs;
420

M
Minglei Jin 已提交
421
  if (lDelIdx->suid < rDelIdx->suid) {
422
    return -1;
M
Minglei Jin 已提交
423
  } else if (lDelIdx->suid > rDelIdx->suid) {
424 425 426
    return 1;
  }

M
Minglei Jin 已提交
427
  if (lDelIdx->uid < rDelIdx->uid) {
428
    return -1;
M
Minglei Jin 已提交
429
  } else if (lDelIdx->uid > rDelIdx->uid) {
430 431 432 433 434 435
    return 1;
  }

  return 0;
}

H
Hongze Cheng 已提交
436 437 438 439 440 441 442 443
int32_t tPutDelIdx(uint8_t *p, void *ph) {
  SDelIdx *pDelIdx = (SDelIdx *)ph;
  int32_t  n = 0;

  n += tPutI64(p ? p + n : p, pDelIdx->suid);
  n += tPutI64(p ? p + n : p, pDelIdx->uid);
  n += tPutI64v(p ? p + n : p, pDelIdx->offset);
  n += tPutI64v(p ? p + n : p, pDelIdx->size);
H
Hongze Cheng 已提交
444 445 446 447

  return n;
}

H
Hongze Cheng 已提交
448 449 450
int32_t tGetDelIdx(uint8_t *p, void *ph) {
  SDelIdx *pDelIdx = (SDelIdx *)ph;
  int32_t  n = 0;
H
Hongze Cheng 已提交
451

H
Hongze Cheng 已提交
452 453 454 455
  n += tGetI64(p + n, &pDelIdx->suid);
  n += tGetI64(p + n, &pDelIdx->uid);
  n += tGetI64v(p + n, &pDelIdx->offset);
  n += tGetI64v(p + n, &pDelIdx->size);
H
Hongze Cheng 已提交
456 457 458 459

  return n;
}

H
Hongze Cheng 已提交
460
// SDelData ======================================================
H
Hongze Cheng 已提交
461 462 463 464 465 466 467
int32_t tPutDelData(uint8_t *p, void *ph) {
  SDelData *pDelData = (SDelData *)ph;
  int32_t   n = 0;

  n += tPutI64v(p ? p + n : p, pDelData->version);
  n += tPutI64(p ? p + n : p, pDelData->sKey);
  n += tPutI64(p ? p + n : p, pDelData->eKey);
H
Hongze Cheng 已提交
468 469 470 471

  return n;
}

H
Hongze Cheng 已提交
472 473 474
int32_t tGetDelData(uint8_t *p, void *ph) {
  SDelData *pDelData = (SDelData *)ph;
  int32_t   n = 0;
H
Hongze Cheng 已提交
475

H
Hongze Cheng 已提交
476 477 478
  n += tGetI64v(p + n, &pDelData->version);
  n += tGetI64(p + n, &pDelData->sKey);
  n += tGetI64(p + n, &pDelData->eKey);
H
Hongze Cheng 已提交
479 480

  return n;
H
Hongze Cheng 已提交
481 482
}

H
Hongze Cheng 已提交
483 484 485 486 487 488
int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision) {
  if (key < 0) {
    return (int)((key + 1) / tsTickPerMin[precision] / minutes - 1);
  } else {
    return (int)((key / tsTickPerMin[precision] / minutes));
  }
H
Hongze Cheng 已提交
489 490
}

H
Hongze Cheng 已提交
491 492 493 494 495
void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey) {
  *minKey = fid * minutes * tsTickPerMin[precision];
  *maxKey = *minKey + minutes * tsTickPerMin[precision] - 1;
}

H
Hongze Cheng 已提交
496 497 498 499
int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now) {
  int32_t aFid[3];
  TSKEY   key;

H
Hongze Cheng 已提交
500 501 502 503 504 505 506 507 508 509
  if (pKeepCfg->precision == TSDB_TIME_PRECISION_MILLI) {
    now = now * 1000;
  } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_MICRO) {
    now = now * 1000000l;
  } else if (pKeepCfg->precision == TSDB_TIME_PRECISION_NANO) {
    now = now * 1000000000l;
  } else {
    ASSERT(0);
  }

H
Hongze Cheng 已提交
510
  key = now - pKeepCfg->keep0 * tsTickPerMin[pKeepCfg->precision];
H
Hongze Cheng 已提交
511
  aFid[0] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision);
H
Hongze Cheng 已提交
512
  key = now - pKeepCfg->keep1 * tsTickPerMin[pKeepCfg->precision];
H
Hongze Cheng 已提交
513
  aFid[1] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision);
H
Hongze Cheng 已提交
514
  key = now - pKeepCfg->keep2 * tsTickPerMin[pKeepCfg->precision];
H
Hongze Cheng 已提交
515
  aFid[2] = tsdbKeyFid(key, pKeepCfg->days, pKeepCfg->precision);
H
Hongze Cheng 已提交
516 517 518 519 520 521 522 523 524 525 526

  if (fid >= aFid[0]) {
    return 0;
  } else if (fid >= aFid[1]) {
    return 1;
  } else if (fid >= aFid[2]) {
    return 2;
  } else {
    return -1;
  }
}
H
Hongze Cheng 已提交
527

H
Hongze Cheng 已提交
528 529
// TSDBROW ======================================================
void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
H
Hongze Cheng 已提交
530 531 532
  STColumn *pTColumn = &pTSchema->columns[iCol];
  SValue    value;

H
Hongze Cheng 已提交
533 534
  ASSERT(iCol > 0);

H
Hongze Cheng 已提交
535
  if (pRow->type == 0) {
H
Hongze Cheng 已提交
536
    tTSRowGetVal(pRow->pTSRow, pTSchema, iCol, pColVal);
H
Hongze Cheng 已提交
537
  } else if (pRow->type == 1) {
H
Hongze Cheng 已提交
538
    SColData *pColData;
H
Hongze Cheng 已提交
539

H
Hongze Cheng 已提交
540
    tBlockDataGetColData(pRow->pBlockData, pTColumn->colId, &pColData);
H
Hongze Cheng 已提交
541

H
Hongze Cheng 已提交
542
    if (pColData) {
H
Hongze Cheng 已提交
543
      tColDataGetValue(pColData, pRow->iRow, pColVal);
H
Hongze Cheng 已提交
544
    } else {
H
Hongze Cheng 已提交
545
      *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
H
Hongze Cheng 已提交
546 547 548 549
    }
  } else {
    ASSERT(0);
  }
H
Hongze Cheng 已提交
550 551
}

H
Hongze Cheng 已提交
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) {
  int32_t n = 0;

  n += tPutI64(p, pRow->version);
  if (p) memcpy(p + n, pRow->pTSRow, pRow->pTSRow->len);
  n += pRow->pTSRow->len;

  return n;
}

int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) {
  int32_t n = 0;

  n += tGetI64(p, &pRow->version);
  pRow->pTSRow = (STSRow *)(p + n);
  n += pRow->pTSRow->len;

  return n;
}

H
Hongze Cheng 已提交
572 573 574 575
int32_t tsdbRowCmprFn(const void *p1, const void *p2) {
  return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2));
}

H
Hongze Cheng 已提交
576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
// SRowIter ======================================================
void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) {
  pIter->pRow = pRow;
  if (pRow->type == 0) {
    ASSERT(pTSchema);
    pIter->pTSchema = pTSchema;
    pIter->i = 1;
  } else if (pRow->type == 1) {
    pIter->pTSchema = NULL;
    pIter->i = 0;
  } else {
    ASSERT(0);
  }
}

SColVal *tRowIterNext(SRowIter *pIter) {
  if (pIter->pRow->type == 0) {
    if (pIter->i < pIter->pTSchema->numOfCols) {
      tsdbRowGetColVal(pIter->pRow, pIter->pTSchema, pIter->i, &pIter->colVal);
      pIter->i++;

      return &pIter->colVal;
    }
  } else {
H
Hongze Cheng 已提交
600 601
    if (pIter->i < taosArrayGetSize(pIter->pRow->pBlockData->aIdx)) {
      SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i);
H
Hongze Cheng 已提交
602 603 604 605 606 607 608 609 610 611 612

      tColDataGetValue(pColData, pIter->pRow->iRow, &pIter->colVal);
      pIter->i++;

      return &pIter->colVal;
    }
  }

  return NULL;
}

H
Hongze Cheng 已提交
613
// SRowMerger ======================================================
614 615 616 617 618 619

int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) {
  int32_t   code = 0;
  TSDBKEY   key = TSDBROW_KEY(pRow);
  SColVal  *pColVal = &(SColVal){0};
  STColumn *pTColumn;
M
Minglei Jin 已提交
620
  int32_t   iCol, jCol = 0;
621 622 623 624 625 626 627 628 629 630 631

  pMerger->pTSchema = pResTSchema;
  pMerger->version = key.version;

  pMerger->pArray = taosArrayInit(pResTSchema->numOfCols, sizeof(SColVal));
  if (pMerger->pArray == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _exit;
  }

  // ts
M
Minglei Jin 已提交
632
  pTColumn = &pTSchema->columns[jCol++];
633 634 635 636 637 638 639 640 641 642

  ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);

  *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts});
  if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _exit;
  }

  // other
M
Minglei Jin 已提交
643
  for (iCol = 1; jCol < pTSchema->numOfCols && iCol < pResTSchema->numOfCols; ++iCol) {
644
    pTColumn = &pResTSchema->columns[iCol];
M
Minglei Jin 已提交
645 646 647 648 649
    if (pTSchema->columns[jCol].colId < pTColumn->colId) {
      ++jCol;
      --iCol;
      continue;
    } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
650 651 652 653
      taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
      continue;
    }

M
Minglei Jin 已提交
654
    tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
    if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _exit;
    }
  }

  for (; iCol < pResTSchema->numOfCols; ++iCol) {
    pTColumn = &pResTSchema->columns[iCol];
    taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
  }

_exit:
  return code;
}

int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
  int32_t   code = 0;
  TSDBKEY   key = TSDBROW_KEY(pRow);
  SColVal  *pColVal = &(SColVal){0};
  STColumn *pTColumn;
M
Minglei Jin 已提交
675
  int32_t   iCol, jCol = 1;
676 677 678

  ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts);

M
Minglei Jin 已提交
679
  for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
680
    pTColumn = &pMerger->pTSchema->columns[iCol];
M
Minglei Jin 已提交
681 682 683 684 685
    if (pTSchema->columns[jCol].colId < pTColumn->colId) {
      ++jCol;
      --iCol;
      continue;
    } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
686 687 688
      continue;
    }

M
Minglei Jin 已提交
689
    tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710

    if (key.version > pMerger->version) {
      if (!pColVal->isNone) {
        taosArraySet(pMerger->pArray, iCol, pColVal);
      }
    } else if (key.version < pMerger->version) {
      SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
      if (tColVal->isNone && !pColVal->isNone) {
        taosArraySet(pMerger->pArray, iCol, pColVal);
      }
    } else {
      ASSERT(0);
    }
  }

  pMerger->version = key.version;

_exit:
  return code;
}

H
Hongze Cheng 已提交
711 712
int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
  int32_t   code = 0;
H
Hongze Cheng 已提交
713
  TSDBKEY   key = TSDBROW_KEY(pRow);
H
Hongze Cheng 已提交
714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
  SColVal  *pColVal = &(SColVal){0};
  STColumn *pTColumn;

  pMerger->pTSchema = pTSchema;
  pMerger->version = key.version;

  pMerger->pArray = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal));
  if (pMerger->pArray == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _exit;
  }

  // ts
  pTColumn = &pTSchema->columns[0];

  ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);

  *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts});
  if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _exit;
  }

  // other
  for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
    tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
    if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _exit;
    }
  }

_exit:
  return code;
}

void tRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); }

int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
  int32_t  code = 0;
H
Hongze Cheng 已提交
754
  TSDBKEY  key = TSDBROW_KEY(pRow);
H
Hongze Cheng 已提交
755 756 757 758 759 760 761 762
  SColVal *pColVal = &(SColVal){0};

  ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts);

  for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
    tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal);

    if (key.version > pMerger->version) {
H
Hongze Cheng 已提交
763 764 765
      if (!pColVal->isNone) {
        taosArraySet(pMerger->pArray, iCol, pColVal);
      }
H
Hongze Cheng 已提交
766
    } else if (key.version < pMerger->version) {
H
Hongze Cheng 已提交
767 768 769 770
      SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
      if (tColVal->isNone && !pColVal->isNone) {
        taosArraySet(pMerger->pArray, iCol, pColVal);
      }
H
Hongze Cheng 已提交
771 772 773 774 775 776 777 778 779 780 781 782 783
    } else {
      ASSERT(0);
    }
  }

  pMerger->version = key.version;

_exit:
  return code;
}

int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow) {
  int32_t code = 0;
784 785 786

  code = tdSTSRowNew(pMerger->pArray, pMerger->pTSchema, ppRow);

H
Hongze Cheng 已提交
787 788 789
  return code;
}

H
Hongze Cheng 已提交
790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892
// delete skyline ======================================================
static int32_t tsdbMergeSkyline(SArray *aSkyline1, SArray *aSkyline2, SArray *aSkyline) {
  int32_t  code = 0;
  int32_t  i1 = 0;
  int32_t  n1 = taosArrayGetSize(aSkyline1);
  int32_t  i2 = 0;
  int32_t  n2 = taosArrayGetSize(aSkyline2);
  TSDBKEY *pSkyline1;
  TSDBKEY *pSkyline2;
  TSDBKEY  item;
  int64_t  version1 = 0;
  int64_t  version2 = 0;

  ASSERT(n1 > 0 && n2 > 0);

  taosArrayClear(aSkyline);

  while (i1 < n1 && i2 < n2) {
    pSkyline1 = (TSDBKEY *)taosArrayGet(aSkyline1, i1);
    pSkyline2 = (TSDBKEY *)taosArrayGet(aSkyline2, i2);

    if (pSkyline1->ts < pSkyline2->ts) {
      version1 = pSkyline1->version;
      i1++;
    } else if (pSkyline1->ts > pSkyline2->ts) {
      version2 = pSkyline2->version;
      i2++;
    } else {
      version1 = pSkyline1->version;
      version2 = pSkyline2->version;
      i1++;
      i2++;
    }

    item.ts = TMIN(pSkyline1->ts, pSkyline2->ts);
    item.version = TMAX(version1, version2);
    if (taosArrayPush(aSkyline, &item) == NULL) {
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _exit;
    }
  }

  while (i1 < n1) {
    pSkyline1 = (TSDBKEY *)taosArrayGet(aSkyline1, i1);
    item.ts = pSkyline1->ts;
    item.version = pSkyline1->version;
    if (taosArrayPush(aSkyline, &item) == NULL) {
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _exit;
    }
    i1++;
  }

  while (i2 < n2) {
    pSkyline2 = (TSDBKEY *)taosArrayGet(aSkyline2, i2);
    item.ts = pSkyline2->ts;
    item.version = pSkyline2->version;
    if (taosArrayPush(aSkyline, &item) == NULL) {
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _exit;
    }
    i2++;
  }

_exit:
  return code;
}
int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline) {
  int32_t   code = 0;
  SDelData *pDelData;
  int32_t   midx;

  taosArrayClear(aSkyline);
  if (sidx == eidx) {
    pDelData = (SDelData *)taosArrayGet(aDelData, sidx);
    taosArrayPush(aSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version});
    taosArrayPush(aSkyline, &(TSDBKEY){.ts = pDelData->eKey, .version = 0});
  } else {
    SArray *aSkyline1 = NULL;
    SArray *aSkyline2 = NULL;

    aSkyline1 = taosArrayInit(0, sizeof(TSDBKEY));
    aSkyline2 = taosArrayInit(0, sizeof(TSDBKEY));
    if (aSkyline1 == NULL || aSkyline2 == NULL) {
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _clear;
    }

    midx = (sidx + eidx) / 2;

    code = tsdbBuildDeleteSkyline(aDelData, sidx, midx, aSkyline1);
    if (code) goto _clear;

    code = tsdbBuildDeleteSkyline(aDelData, midx + 1, eidx, aSkyline2);
    if (code) goto _clear;

    code = tsdbMergeSkyline(aSkyline1, aSkyline2, aSkyline);

  _clear:
    taosArrayDestroy(aSkyline1);
    taosArrayDestroy(aSkyline2);
  }

H
Hongze Cheng 已提交
893 894 895
  return code;
}

H
Hongze Cheng 已提交
896
// SColData ========================================
H
Hongze Cheng 已提交
897
void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
H
Hongze Cheng 已提交
898 899
  pColData->cid = cid;
  pColData->type = type;
H
Hongze Cheng 已提交
900
  pColData->smaOn = smaOn;
H
Hongze Cheng 已提交
901 902 903 904
  tColDataReset(pColData);
}

void tColDataReset(SColData *pColData) {
H
Hongze Cheng 已提交
905 906
  pColData->nVal = 0;
  pColData->flag = 0;
H
Hongze Cheng 已提交
907
  pColData->nData = 0;
H
Hongze Cheng 已提交
908 909
}

H
Hongze Cheng 已提交
910 911 912
void tColDataClear(void *ph) {
  SColData *pColData = (SColData *)ph;

H
Hongze Cheng 已提交
913 914 915
  tFree(pColData->pBitMap);
  tFree((uint8_t *)pColData->aOffset);
  tFree(pColData->pData);
H
Hongze Cheng 已提交
916 917 918 919
}

int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
  int32_t code = 0;
H
Hongze Cheng 已提交
920 921 922
  int64_t size;
  SValue  value = {0};
  SValue *pValue = &value;
H
Hongze Cheng 已提交
923

H
Hongze Cheng 已提交
924 925 926 927 928
  ASSERT(pColVal->cid == pColData->cid);
  ASSERT(pColVal->type == pColData->type);

  // realloc bitmap
  size = BIT2_SIZE(pColData->nVal + 1);
H
Hongze Cheng 已提交
929
  code = tRealloc(&pColData->pBitMap, size);
H
Hongze Cheng 已提交
930
  if (code) goto _exit;
H
Hongze Cheng 已提交
931 932 933
  if ((pColData->nVal & 3) == 0) {
    pColData->pBitMap[pColData->nVal >> 2] = 0;
  }
H
Hongze Cheng 已提交
934 935

  // put value
H
Hongze Cheng 已提交
936
  if (pColVal->isNone) {
H
Hongze Cheng 已提交
937 938
    pColData->flag |= HAS_NONE;
    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
H
Hongze Cheng 已提交
939
  } else if (pColVal->isNull) {
H
Hongze Cheng 已提交
940 941
    pColData->flag |= HAS_NULL;
    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
H
Hongze Cheng 已提交
942
  } else {
H
Hongze Cheng 已提交
943 944 945 946 947
    pColData->flag |= HAS_VALUE;
    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
    pValue = &pColVal->value;
  }

H
Hongze Cheng 已提交
948 949
  if (IS_VAR_DATA_TYPE(pColData->type)) {
    // offset
H
Hongze Cheng 已提交
950
    code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1));
H
Hongze Cheng 已提交
951
    if (code) goto _exit;
H
Hongze Cheng 已提交
952
    pColData->aOffset[pColData->nVal] = pColData->nData;
H
Hongze Cheng 已提交
953

H
Hongze Cheng 已提交
954 955
    // value
    if ((!pColVal->isNone) && (!pColVal->isNull)) {
H
Hongze Cheng 已提交
956
      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
H
Hongze Cheng 已提交
957 958 959 960 961
      if (code) goto _exit;
      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
      pColData->nData += pColVal->value.nData;
    }
  } else {
H
Hongze Cheng 已提交
962
    code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type));
H
Hongze Cheng 已提交
963 964
    if (code) goto _exit;
    pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type);
H
Hongze Cheng 已提交
965 966
  }

H
Hongze Cheng 已提交
967 968 969
  pColData->nVal++;

_exit:
H
Hongze Cheng 已提交
970 971 972
  return code;
}

H
Hongze Cheng 已提交
973 974
int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
  int32_t code = 0;
H
Hongze Cheng 已提交
975
  int32_t size;
H
Hongze Cheng 已提交
976

H
Hongze Cheng 已提交
977
  ASSERT(pColDataSrc->nVal > 0);
H
Hongze Cheng 已提交
978 979
  ASSERT(pColDataDest->cid = pColDataSrc->cid);
  ASSERT(pColDataDest->type = pColDataSrc->type);
H
Hongze Cheng 已提交
980

H
Hongze Cheng 已提交
981
  pColDataDest->smaOn = pColDataSrc->smaOn;
H
Hongze Cheng 已提交
982 983 984
  pColDataDest->nVal = pColDataSrc->nVal;
  pColDataDest->flag = pColDataSrc->flag;

H
Hongze Cheng 已提交
985
  // bitmap
H
Hongze Cheng 已提交
986 987 988 989 990 991
  if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
    size = BIT2_SIZE(pColDataSrc->nVal);
    code = tRealloc(&pColDataDest->pBitMap, size);
    if (code) goto _exit;
    memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
  }
H
Hongze Cheng 已提交
992

H
Hongze Cheng 已提交
993
  // offset
H
Hongze Cheng 已提交
994 995
  if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
    size = sizeof(int32_t) * pColDataSrc->nVal;
H
Hongze Cheng 已提交
996

H
Hongze Cheng 已提交
997
    code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
H
Hongze Cheng 已提交
998 999
    if (code) goto _exit;

H
Hongze Cheng 已提交
1000
    memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
H
Hongze Cheng 已提交
1001 1002
  }

H
Hongze Cheng 已提交
1003 1004
  // value
  pColDataDest->nData = pColDataSrc->nData;
H
Hongze Cheng 已提交
1005
  code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
H
Hongze Cheng 已提交
1006 1007 1008
  if (code) goto _exit;
  memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);

H
Hongze Cheng 已提交
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
_exit:
  return code;
}

int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
  int32_t code = 0;

  ASSERT(iVal < pColData->nVal);
  ASSERT(pColData->flag);

  if (pColData->flag == HAS_NONE) {
    *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
    goto _exit;
  } else if (pColData->flag == HAS_NULL) {
    *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
    goto _exit;
  } else if (pColData->flag != HAS_VALUE) {
    uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
    if (v == 0) {
      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
      goto _exit;
    } else if (v == 1) {
      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
      goto _exit;
    }
  }

  // get value
  SValue value;
  if (IS_VAR_DATA_TYPE(pColData->type)) {
H
Hongze Cheng 已提交
1039 1040 1041 1042
    if (iVal + 1 < pColData->nVal) {
      value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
    } else {
      value.nData = pColData->nData - pColData->aOffset[iVal];
H
Hongze Cheng 已提交
1043
    }
H
Hongze Cheng 已提交
1044 1045

    value.pData = pColData->pData + pColData->aOffset[iVal];
H
Hongze Cheng 已提交
1046 1047 1048 1049 1050 1051 1052
  } else {
    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
  }
  *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);

_exit:
  return code;
H
Hongze Cheng 已提交
1053 1054
}

H
Hongze Cheng 已提交
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
int32_t tPutColData(uint8_t *p, SColData *pColData) {
  int32_t n = 0;

  n += tPutI16v(p ? p + n : p, pColData->cid);
  n += tPutI8(p ? p + n : p, pColData->type);
  n += tPutI8(p ? p + n : p, pColData->smaOn);
  n += tPutI32v(p ? p + n : p, pColData->nVal);
  n += tPutU8(p ? p + n : p, pColData->flag);

  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
  if (pColData->flag != HAS_VALUE) {
    // bitmap

    int32_t size = BIT2_SIZE(pColData->nVal);
    if (p) {
      memcpy(p + n, pColData->pBitMap, size);
    }
    n += size;
  }
  if (IS_VAR_DATA_TYPE(pColData->type)) {
    // offset

    int32_t size = sizeof(int32_t) * pColData->nVal;
    if (p) {
      memcpy(p + n, pColData->aOffset, size);
    }
    n += size;
  }
  n += tPutI32v(p ? p + n : p, pColData->nData);
  if (p) {
    memcpy(p + n, pColData->pData, pColData->nData);
  }
  n += pColData->nData;

_exit:
  return n;
}

int32_t tGetColData(uint8_t *p, SColData *pColData) {
  int32_t n = 0;

  n += tGetI16v(p + n, &pColData->cid);
  n += tGetI8(p + n, &pColData->type);
  n += tGetI8(p + n, &pColData->smaOn);
  n += tGetI32v(p + n, &pColData->nVal);
  n += tGetU8(p + n, &pColData->flag);

  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
  if (pColData->flag != HAS_VALUE) {
    // bitmap

    int32_t size = BIT2_SIZE(pColData->nVal);
    pColData->pBitMap = p + n;
    n += size;
  }
  if (IS_VAR_DATA_TYPE(pColData->type)) {
    // offset

    int32_t size = sizeof(int32_t) * pColData->nVal;
    pColData->aOffset = (int32_t *)(p + n);
    n += size;
  }
  n += tGetI32v(p + n, &pColData->nData);
  pColData->pData = p + n;
  n += pColData->nData;

_exit:
  return n;
}

H
Hongze Cheng 已提交
1125 1126 1127
static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
  SColData *pColData1 = (SColData *)p1;
  SColData *pColData2 = (SColData *)p2;
H
Hongze Cheng 已提交
1128 1129

  if (pColData1->cid < pColData2->cid) {
H
Hongze Cheng 已提交
1130
    return -1;
H
Hongze Cheng 已提交
1131
  } else if (pColData1->cid > pColData2->cid) {
H
Hongze Cheng 已提交
1132 1133 1134 1135 1136 1137
    return 1;
  }

  return 0;
}

H
Hongze Cheng 已提交
1138
// SBlockData ======================================================
H
Hongze Cheng 已提交
1139
int32_t tBlockDataCreate(SBlockData *pBlockData) {
H
Hongze Cheng 已提交
1140
  int32_t code = 0;
H
Hongze Cheng 已提交
1141

H
Hongze Cheng 已提交
1142 1143
  pBlockData->suid = 0;
  pBlockData->uid = 0;
H
Hongze Cheng 已提交
1144
  pBlockData->nRow = 0;
H
Hongze Cheng 已提交
1145
  pBlockData->aUid = NULL;
H
Hongze Cheng 已提交
1146 1147
  pBlockData->aVersion = NULL;
  pBlockData->aTSKEY = NULL;
H
Hongze Cheng 已提交
1148 1149
  pBlockData->aIdx = taosArrayInit(0, sizeof(int32_t));
  if (pBlockData->aIdx == NULL) {
H
Hongze Cheng 已提交
1150 1151 1152 1153 1154
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _exit;
  }
  pBlockData->aColData = taosArrayInit(0, sizeof(SColData));
  if (pBlockData->aColData == NULL) {
H
Hongze Cheng 已提交
1155
    taosArrayDestroy(pBlockData->aIdx);
H
Hongze Cheng 已提交
1156 1157 1158
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _exit;
  }
H
Hongze Cheng 已提交
1159 1160 1161 1162 1163

_exit:
  return code;
}

H
Hongze Cheng 已提交
1164
void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) {
H
Hongze Cheng 已提交
1165
  tFree((uint8_t *)pBlockData->aUid);
H
Hongze Cheng 已提交
1166 1167
  tFree((uint8_t *)pBlockData->aVersion);
  tFree((uint8_t *)pBlockData->aTSKEY);
H
Hongze Cheng 已提交
1168
  taosArrayDestroy(pBlockData->aIdx);
H
Hongze Cheng 已提交
1169
  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL);
H
Hongze Cheng 已提交
1170
  pBlockData->aUid = NULL;
H
Haojun Liao 已提交
1171
  pBlockData->aVersion = NULL;
H
Hongze Cheng 已提交
1172 1173 1174
  pBlockData->aTSKEY = NULL;
  pBlockData->aIdx = NULL;
  pBlockData->aColData = NULL;
H
Hongze Cheng 已提交
1175 1176
}

H
Hongze Cheng 已提交
1177 1178 1179 1180 1181 1182 1183
int32_t tBlockDataInit(SBlockData *pBlockData, int64_t suid, int64_t uid, STSchema *pTSchema) {
  int32_t code = 0;

  ASSERT(suid || uid);

  pBlockData->suid = suid;
  pBlockData->uid = uid;
H
Hongze Cheng 已提交
1184
  pBlockData->nRow = 0;
H
Hongze Cheng 已提交
1185

H
Hongze Cheng 已提交
1186
  taosArrayClear(pBlockData->aIdx);
H
Hongze Cheng 已提交
1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
  for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) {
    STColumn *pTColumn = &pTSchema->columns[iColumn];

    SColData *pColData;
    code = tBlockDataAddColData(pBlockData, iColumn - 1, &pColData);
    if (code) goto _exit;

    tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0);
  }

_exit:
  return code;
H
Hongze Cheng 已提交
1199 1200
}

H
Hongze Cheng 已提交
1201
int32_t tBlockDataInitEx(SBlockData *pBlockData, SBlockData *pBlockDataFrom) {
H
Hongze Cheng 已提交
1202 1203
  int32_t code = 0;

H
Hongze Cheng 已提交
1204
  ASSERT(pBlockDataFrom->suid || pBlockDataFrom->uid);
H
Hongze Cheng 已提交
1205

H
Hongze Cheng 已提交
1206 1207
  pBlockData->suid = pBlockDataFrom->suid;
  pBlockData->uid = pBlockDataFrom->uid;
H
Hongze Cheng 已提交
1208
  pBlockData->nRow = 0;
H
Hongze Cheng 已提交
1209

H
Hongze Cheng 已提交
1210
  taosArrayClear(pBlockData->aIdx);
H
Hongze Cheng 已提交
1211 1212
  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockDataFrom->aIdx); iColData++) {
    SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColData);
H
Hongze Cheng 已提交
1213

H
Hongze Cheng 已提交
1214 1215 1216
    SColData *pColData;
    code = tBlockDataAddColData(pBlockData, iColData, &pColData);
    if (code) goto _exit;
H
Hongze Cheng 已提交
1217

H
Hongze Cheng 已提交
1218
    tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn);
H
Hongze Cheng 已提交
1219 1220 1221 1222 1223 1224
  }

_exit:
  return code;
}

H
Hongze Cheng 已提交
1225 1226 1227 1228 1229 1230 1231 1232
void tBlockDataReset(SBlockData *pBlockData) {
  pBlockData->suid = 0;
  pBlockData->uid = 0;
  pBlockData->nRow = 0;
  taosArrayClear(pBlockData->aIdx);
}

void tBlockDataClear(SBlockData *pBlockData) {
H
Hongze Cheng 已提交
1233 1234
  ASSERT(pBlockData->suid || pBlockData->uid);

H
Hongze Cheng 已提交
1235
  pBlockData->nRow = 0;
H
Hongze Cheng 已提交
1236 1237
  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
H
Hongze Cheng 已提交
1238 1239 1240 1241
    tColDataReset(pColData);
  }
}

H
Hongze Cheng 已提交
1242 1243
int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData) {
  int32_t   code = 0;
H
Hongze Cheng 已提交
1244
  SColData *pColData = NULL;
H
Hongze Cheng 已提交
1245
  int32_t   idx = taosArrayGetSize(pBlockData->aIdx);
H
Hongze Cheng 已提交
1246 1247

  if (idx >= taosArrayGetSize(pBlockData->aColData)) {
H
Hongze Cheng 已提交
1248 1249 1250 1251
    if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) {
      code = TSDB_CODE_OUT_OF_MEMORY;
      goto _err;
    }
H
Hongze Cheng 已提交
1252 1253 1254
  }
  pColData = (SColData *)taosArrayGet(pBlockData->aColData, idx);

H
Hongze Cheng 已提交
1255
  if (taosArrayInsert(pBlockData->aIdx, iColData, &idx) == NULL) {
H
Hongze Cheng 已提交
1256 1257
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto _err;
H
Hongze Cheng 已提交
1258 1259
  }

H
Hongze Cheng 已提交
1260 1261 1262 1263 1264 1265
  *ppColData = pColData;
  return code;

_err:
  *ppColData = NULL;
  return code;
H
Hongze Cheng 已提交
1266
}
H
Hongze Cheng 已提交
1267

H
Hongze Cheng 已提交
1268
int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) {
H
Hongze Cheng 已提交
1269
  int32_t code = 0;
H
Hongze Cheng 已提交
1270

H
Hongze Cheng 已提交
1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
  ASSERT(pBlockData->suid || pBlockData->uid);

  // uid
  if (pBlockData->uid == 0) {
    ASSERT(uid);
    code = tRealloc((uint8_t **)&pBlockData->aUid, sizeof(int64_t) * (pBlockData->nRow + 1));
    if (code) goto _err;
    pBlockData->aUid[pBlockData->nRow] = uid;
  }
  // version
H
Hongze Cheng 已提交
1281
  code = tRealloc((uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * (pBlockData->nRow + 1));
H
Hongze Cheng 已提交
1282
  if (code) goto _err;
H
Hongze Cheng 已提交
1283 1284
  pBlockData->aVersion[pBlockData->nRow] = TSDBROW_VERSION(pRow);
  // timestamp
H
Hongze Cheng 已提交
1285
  code = tRealloc((uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * (pBlockData->nRow + 1));
H
Hongze Cheng 已提交
1286
  if (code) goto _err;
H
Hongze Cheng 已提交
1287
  pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow);
H
Hongze Cheng 已提交
1288

H
Hongze Cheng 已提交
1289
  // OTHER
H
Hongze Cheng 已提交
1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
  SRowIter rIter = {0};
  SColVal *pColVal;

  tRowIterInit(&rIter, pRow, pTSchema);
  pColVal = tRowIterNext(&rIter);
  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);

    while (pColVal && pColVal->cid < pColData->cid) {
      pColVal = tRowIterNext(&rIter);
    }

    if (pColVal == NULL || pColVal->cid > pColData->cid) {
H
Hongze Cheng 已提交
1303
      code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
H
Hongze Cheng 已提交
1304
      if (code) goto _err;
H
Hongze Cheng 已提交
1305 1306 1307 1308
    } else {
      code = tColDataAppendValue(pColData, pColVal);
      if (code) goto _err;
      pColVal = tRowIterNext(&rIter);
H
Hongze Cheng 已提交
1309
    }
H
Hongze Cheng 已提交
1310
  }
H
Hongze Cheng 已提交
1311

H
Hongze Cheng 已提交
1312
_exit:
H
Hongze Cheng 已提交
1313
  pBlockData->nRow++;
H
Hongze Cheng 已提交
1314
  return code;
H
Hongze Cheng 已提交
1315

H
Hongze Cheng 已提交
1316
_err:
H
Hongze Cheng 已提交
1317
  return code;
1318
}
H
Hongze Cheng 已提交
1319

H
Hongze Cheng 已提交
1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) {
  int32_t code = 0;

  int32_t iColData = 0;
  for (int32_t iColDataFrom = 0; iColDataFrom < taosArrayGetSize(pBlockDataFrom->aIdx); iColDataFrom++) {
    SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom);

    while (true) {
      SColData *pColData;
      if (iColData < taosArrayGetSize(pBlockData->aIdx)) {
        pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
      } else {
        pColData = NULL;
      }

      if (pColData == NULL || pColData->cid > pColDataFrom->cid) {
        code = tBlockDataAddColData(pBlockData, iColData, &pColData);
        if (code) goto _exit;

H
Hongze Cheng 已提交
1339
        tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn);
H
Hongze Cheng 已提交
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
        for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
          code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
          if (code) goto _exit;
        }

        iColData++;
        break;
      } else if (pColData->cid == pColDataFrom->cid) {
        iColData++;
        break;
      } else {
        iColData++;
      }
    }
  }

_exit:
  return code;
}

H
Hongze Cheng 已提交
1360 1361 1362
int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) {
  int32_t code = 0;

H
Hongze Cheng 已提交
1363 1364 1365 1366
  ASSERT(pBlockData->suid == pBlockData1->suid);
  ASSERT(pBlockData->uid == pBlockData1->uid);
  ASSERT(pBlockData1->nRow > 0);
  ASSERT(pBlockData2->nRow > 0);
H
Hongze Cheng 已提交
1367

H
Hongze Cheng 已提交
1368
  tBlockDataClear(pBlockData);
H
Hongze Cheng 已提交
1369

H
Hongze Cheng 已提交
1370 1371 1372 1373
  TSDBROW  row1 = tsdbRowFromBlockData(pBlockData1, 0);
  TSDBROW  row2 = tsdbRowFromBlockData(pBlockData2, 0);
  TSDBROW *pRow1 = &row1;
  TSDBROW *pRow2 = &row2;
H
Hongze Cheng 已提交
1374

H
Hongze Cheng 已提交
1375 1376
  while (pRow1 && pRow2) {
    int32_t c = tsdbRowCmprFn(pRow1, pRow2);
H
Hongze Cheng 已提交
1377 1378

    if (c < 0) {
H
Hongze Cheng 已提交
1379 1380
      code = tBlockDataAppendRow(pBlockData, pRow1, NULL,
                                 pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]);
H
Hongze Cheng 已提交
1381
      if (code) goto _exit;
H
Hongze Cheng 已提交
1382 1383 1384 1385 1386 1387 1388

      pRow1->iRow++;
      if (pRow1->iRow < pBlockData1->nRow) {
        *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow);
      } else {
        pRow1 = NULL;
      }
H
Hongze Cheng 已提交
1389
    } else if (c > 0) {
H
Hongze Cheng 已提交
1390 1391
      code = tBlockDataAppendRow(pBlockData, pRow2, NULL,
                                 pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]);
H
Hongze Cheng 已提交
1392
      if (code) goto _exit;
H
Hongze Cheng 已提交
1393 1394 1395 1396 1397 1398 1399

      pRow2->iRow++;
      if (pRow2->iRow < pBlockData2->nRow) {
        *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow);
      } else {
        pRow2 = NULL;
      }
H
Hongze Cheng 已提交
1400 1401 1402 1403 1404
    } else {
      ASSERT(0);
    }
  }

H
Hongze Cheng 已提交
1405 1406 1407
  while (pRow1) {
    code = tBlockDataAppendRow(pBlockData, pRow1, NULL,
                               pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]);
H
Hongze Cheng 已提交
1408
    if (code) goto _exit;
H
Hongze Cheng 已提交
1409 1410 1411 1412 1413 1414 1415

    pRow1->iRow++;
    if (pRow1->iRow < pBlockData1->nRow) {
      *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow);
    } else {
      pRow1 = NULL;
    }
H
Hongze Cheng 已提交
1416 1417
  }

H
Hongze Cheng 已提交
1418 1419 1420
  while (pRow2) {
    code = tBlockDataAppendRow(pBlockData, pRow2, NULL,
                               pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]);
H
Hongze Cheng 已提交
1421
    if (code) goto _exit;
H
Hongze Cheng 已提交
1422 1423 1424 1425 1426 1427 1428

    pRow2->iRow++;
    if (pRow2->iRow < pBlockData2->nRow) {
      *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow);
    } else {
      pRow2 = NULL;
    }
H
Hongze Cheng 已提交
1429 1430 1431 1432 1433 1434
  }

_exit:
  return code;
}

H
Hongze Cheng 已提交
1435 1436 1437 1438 1439 1440 1441 1442
int32_t tBlockDataCopy(SBlockData *pSrc, SBlockData *pDest) {
  int32_t code = 0;

  tBlockDataClear(pDest);

  ASSERT(pDest->suid == pSrc->suid);
  ASSERT(pDest->uid == pSrc->uid);
  ASSERT(taosArrayGetSize(pSrc->aIdx) == taosArrayGetSize(pDest->aIdx));
H
Hongze Cheng 已提交
1443

H
Hongze Cheng 已提交
1444
  pDest->nRow = pSrc->nRow;
H
Hongze Cheng 已提交
1445

H
Hongze Cheng 已提交
1446 1447 1448 1449 1450
  if (pSrc->uid == 0) {
    code = tRealloc((uint8_t **)&pDest->aUid, sizeof(int64_t) * pDest->nRow);
    if (code) goto _exit;
    memcpy(pDest->aUid, pSrc->aUid, sizeof(int64_t) * pDest->nRow);
  }
H
Hongze Cheng 已提交
1451

H
Hongze Cheng 已提交
1452
  code = tRealloc((uint8_t **)&pDest->aVersion, sizeof(int64_t) * pDest->nRow);
H
Hongze Cheng 已提交
1453
  if (code) goto _exit;
H
Hongze Cheng 已提交
1454 1455 1456
  memcpy(pDest->aVersion, pSrc->aVersion, sizeof(int64_t) * pDest->nRow);

  code = tRealloc((uint8_t **)&pDest->aTSKEY, sizeof(TSKEY) * pDest->nRow);
H
Hongze Cheng 已提交
1457
  if (code) goto _exit;
H
Hongze Cheng 已提交
1458
  memcpy(pDest->aTSKEY, pSrc->aTSKEY, sizeof(TSKEY) * pDest->nRow);
H
Hongze Cheng 已提交
1459

H
Hongze Cheng 已提交
1460 1461 1462 1463 1464 1465
  for (int32_t iColData = 0; iColData < taosArrayGetSize(pSrc->aIdx); iColData++) {
    SColData *pColSrc = tBlockDataGetColDataByIdx(pSrc, iColData);
    SColData *pColDest = tBlockDataGetColDataByIdx(pDest, iColData);

    ASSERT(pColSrc->cid == pColDest->cid);
    ASSERT(pColSrc->type == pColDest->type);
H
Hongze Cheng 已提交
1466

H
Hongze Cheng 已提交
1467
    code = tColDataCopy(pColSrc, pColDest);
H
Hongze Cheng 已提交
1468 1469 1470 1471 1472
    if (code) goto _exit;
  }

_exit:
  return code;
1473
}
H
Hongze Cheng 已提交
1474

H
Hongze Cheng 已提交
1475 1476 1477 1478 1479
SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) {
  ASSERT(idx >= 0 && idx < taosArrayGetSize(pBlockData->aIdx));
  return (SColData *)taosArrayGet(pBlockData->aColData, *(int32_t *)taosArrayGet(pBlockData->aIdx, idx));
}

H
Hongze Cheng 已提交
1480 1481
void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) {
  ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID);
H
Hongze Cheng 已提交
1482 1483
  int32_t lidx = 0;
  int32_t ridx = taosArrayGetSize(pBlockData->aIdx) - 1;
H
Hongze Cheng 已提交
1484

H
Hongze Cheng 已提交
1485
  while (lidx <= ridx) {
M
Minglei Jin 已提交
1486 1487 1488
    int32_t   midx = (lidx + ridx) / 2;
    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx);
    int32_t   c = tColDataCmprFn(pColData, &(SColData){.cid = cid});
H
Hongze Cheng 已提交
1489 1490 1491 1492 1493 1494 1495 1496 1497

    if (c == 0) {
      *ppColData = pColData;
      return;
    } else if (c < 0) {
      lidx = midx + 1;
    } else {
      ridx = midx - 1;
    }
H
Hongze Cheng 已提交
1498
  }
H
Hongze Cheng 已提交
1499 1500

  *ppColData = NULL;
H
Hongze Cheng 已提交
1501 1502
}

H
Hongze Cheng 已提交
1503
int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[]) {
H
Hongze Cheng 已提交
1504
  int32_t code = 0;
H
Hongze Cheng 已提交
1505

H
Hongze Cheng 已提交
1506 1507 1508 1509 1510 1511 1512 1513 1514
  SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT,
                      .fmtVer = 0,
                      .suid = pBlockData->suid,
                      .uid = pBlockData->uid,
                      .nRow = pBlockData->nRow,
                      .cmprAlg = cmprAlg};

  // encode =================
  // columns AND SBlockCol
H
Hongze Cheng 已提交
1515
  int32_t nBuf0 = 0;
H
Hongze Cheng 已提交
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);

    ASSERT(pColData->flag);

    if (pColData->flag == HAS_NONE) continue;

    SBlockCol blockCol = {.cid = pColData->cid,
                          .type = pColData->type,
                          .smaOn = pColData->smaOn,
                          .flag = pColData->flag,
                          .szOrigin = pColData->nData};

    if (pColData->flag != HAS_NULL) {
H
Hongze Cheng 已提交
1530
      code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], nBuf0, &aBuf[2]);
H
Hongze Cheng 已提交
1531 1532
      if (code) goto _exit;

H
Hongze Cheng 已提交
1533 1534
      blockCol.offset = nBuf0;
      nBuf0 = nBuf0 + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM);
H
Hongze Cheng 已提交
1535 1536
    }

H
Hongze Cheng 已提交
1537
    code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol));
H
Hongze Cheng 已提交
1538
    if (code) goto _exit;
H
Hongze Cheng 已提交
1539
    hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol);
H
Hongze Cheng 已提交
1540
  }
H
Hongze Cheng 已提交
1541

H
Hongze Cheng 已提交
1542
  int32_t nBuf1 = 0;
H
Hongze Cheng 已提交
1543
  if (hdr.szBlkCol > 0) {
H
Hongze Cheng 已提交
1544
    nBuf1 = hdr.szBlkCol + sizeof(TSCKSUM);
H
Hongze Cheng 已提交
1545

H
Hongze Cheng 已提交
1546
    code = tRealloc(&aBuf[1], nBuf1);
H
Hongze Cheng 已提交
1547 1548
    if (code) goto _exit;

H
Hongze Cheng 已提交
1549
    taosCalcChecksumAppend(0, aBuf[1], nBuf1);
H
Hongze Cheng 已提交
1550 1551
  }

H
Hongze Cheng 已提交
1552
  // uid + version + tskey
H
Hongze Cheng 已提交
1553
  int32_t nBuf2 = 0;
H
Hongze Cheng 已提交
1554 1555
  if (pBlockData->uid == 0) {
    code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg,
H
Hongze Cheng 已提交
1556
                        &aBuf[2], nBuf2, &hdr.szUid, &aBuf[3]);
H
Hongze Cheng 已提交
1557
    if (code) goto _exit;
H
Hongze Cheng 已提交
1558
  }
H
Hongze Cheng 已提交
1559
  nBuf2 += hdr.szUid;
H
Hongze Cheng 已提交
1560

H
Hongze Cheng 已提交
1561
  code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT,
H
Hongze Cheng 已提交
1562
                      cmprAlg, &aBuf[2], nBuf2, &hdr.szVer, &aBuf[3]);
H
Hongze Cheng 已提交
1563
  if (code) goto _exit;
H
Hongze Cheng 已提交
1564
  nBuf2 += hdr.szVer;
H
Hongze Cheng 已提交
1565

H
Hongze Cheng 已提交
1566
  code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP,
H
Hongze Cheng 已提交
1567
                      cmprAlg, &aBuf[2], nBuf2, &hdr.szKey, &aBuf[3]);
H
Hongze Cheng 已提交
1568
  if (code) goto _exit;
H
Hongze Cheng 已提交
1569
  nBuf2 += hdr.szKey;
H
Hongze Cheng 已提交
1570

H
Hongze Cheng 已提交
1571 1572
  nBuf2 += sizeof(TSCKSUM);
  code = tRealloc(&aBuf[2], nBuf2);
H
Hongze Cheng 已提交
1573
  if (code) goto _exit;
H
Hongze Cheng 已提交
1574

H
Hongze Cheng 已提交
1575
  // hdr
H
Hongze Cheng 已提交
1576 1577
  int32_t nBuf3 = tPutDiskDataHdr(NULL, &hdr);
  code = tRealloc(&aBuf[3], nBuf3);
H
Hongze Cheng 已提交
1578
  if (code) goto _exit;
H
Hongze Cheng 已提交
1579 1580
  tPutDiskDataHdr(aBuf[3], &hdr);
  taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], nBuf3), aBuf[2], nBuf2);
H
Hongze Cheng 已提交
1581

H
Hongze Cheng 已提交
1582 1583
  // aggragate
  if (ppOut) {
H
Hongze Cheng 已提交
1584
    *szOut = nBuf0 + nBuf1 + nBuf2 + nBuf3;
H
Hongze Cheng 已提交
1585 1586
    code = tRealloc(ppOut, *szOut);
    if (code) goto _exit;
H
Hongze Cheng 已提交
1587

H
Hongze Cheng 已提交
1588 1589
    memcpy(*ppOut, aBuf[3], nBuf3);
    memcpy(*ppOut + nBuf3, aBuf[2], nBuf2);
H
Hongze Cheng 已提交
1590
    if (nBuf1) {
H
Hongze Cheng 已提交
1591 1592 1593 1594
      memcpy(*ppOut + nBuf3 + nBuf2, aBuf[1], nBuf1);
    }
    if (nBuf0) {
      memcpy(*ppOut + nBuf3 + nBuf2 + nBuf1, aBuf[0], nBuf0);
H
Hongze Cheng 已提交
1595
    }
H
Hongze Cheng 已提交
1596 1597
  }

H
Hongze Cheng 已提交
1598 1599 1600 1601 1602 1603 1604 1605
_exit:
  return code;
}

int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData) {
  int32_t code = 0;
  // TODO
  return code;
H
Hongze Cheng 已提交
1606 1607
}

H
Hongze Cheng 已提交
1608 1609 1610 1611 1612 1613
// SDiskDataHdr ==============================
int32_t tPutDiskDataHdr(uint8_t *p, void *ph) {
  int32_t       n = 0;
  SDiskDataHdr *pHdr = (SDiskDataHdr *)ph;

  n += tPutU32(p ? p + n : p, pHdr->delimiter);
H
Hongze Cheng 已提交
1614
  n += tPutU32v(p ? p + n : p, pHdr->fmtVer);
H
Hongze Cheng 已提交
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
  n += tPutI64(p ? p + n : p, pHdr->suid);
  n += tPutI64(p ? p + n : p, pHdr->uid);
  n += tPutI32v(p ? p + n : p, pHdr->szUid);
  n += tPutI32v(p ? p + n : p, pHdr->szVer);
  n += tPutI32v(p ? p + n : p, pHdr->szKey);
  n += tPutI32v(p ? p + n : p, pHdr->szBlkCol);
  n += tPutI32v(p ? p + n : p, pHdr->nRow);
  n += tPutI8(p ? p + n : p, pHdr->cmprAlg);

  return n;
}

int32_t tGetDiskDataHdr(uint8_t *p, void *ph) {
  int32_t       n = 0;
  SDiskDataHdr *pHdr = (SDiskDataHdr *)ph;

  n += tGetU32(p + n, &pHdr->delimiter);
H
Hongze Cheng 已提交
1632
  n += tGetU32v(p + n, &pHdr->fmtVer);
H
Hongze Cheng 已提交
1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644
  n += tGetI64(p + n, &pHdr->suid);
  n += tGetI64(p + n, &pHdr->uid);
  n += tGetI32v(p + n, &pHdr->szUid);
  n += tGetI32v(p + n, &pHdr->szVer);
  n += tGetI32v(p + n, &pHdr->szKey);
  n += tGetI32v(p + n, &pHdr->szBlkCol);
  n += tGetI32v(p + n, &pHdr->nRow);
  n += tGetI8(p + n, &pHdr->cmprAlg);

  return n;
}

H
Hongze Cheng 已提交
1645
// ALGORITHM ==============================
H
Hongze Cheng 已提交
1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
  int32_t n = 0;

  n += tPutI16v(p ? p + n : p, pColAgg->colId);
  n += tPutI16v(p ? p + n : p, pColAgg->numOfNull);
  n += tPutI64(p ? p + n : p, pColAgg->sum);
  n += tPutI64(p ? p + n : p, pColAgg->max);
  n += tPutI64(p ? p + n : p, pColAgg->min);

  return n;
}

int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
  int32_t n = 0;

  n += tGetI16v(p + n, &pColAgg->colId);
  n += tGetI16v(p + n, &pColAgg->numOfNull);
  n += tGetI64(p + n, &pColAgg->sum);
  n += tGetI64(p + n, &pColAgg->max);
  n += tGetI64(p + n, &pColAgg->min);

  return n;
}

H
Hongze Cheng 已提交
1670 1671 1672 1673
void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) {
  SColVal  colVal;
  SColVal *pColVal = &colVal;

1674 1675 1676 1677
  memset(pColAgg, 0, sizeof(*pColAgg));
  bool minAssigned = false;
  bool maxAssigned = false;

H
Hongze Cheng 已提交
1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
  *pColAgg = (SColumnDataAgg){.colId = pColData->cid};
  for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
    tColDataGetValue(pColData, iVal, pColVal);

    if (pColVal->isNone || pColVal->isNull) {
      pColAgg->numOfNull++;
    } else {
      switch (pColData->type) {
        case TSDB_DATA_TYPE_NULL:
          break;
        case TSDB_DATA_TYPE_BOOL:
          break;
1690
        case TSDB_DATA_TYPE_TINYINT: {
1691
          pColAgg->sum += colVal.value.i8;
1692
          if (!minAssigned || pColAgg->min > colVal.value.i8) {
1693
            pColAgg->min = colVal.value.i8;
1694
            minAssigned = true;
1695
          }
1696
          if (!maxAssigned || pColAgg->max < colVal.value.i8) {
1697
            pColAgg->max = colVal.value.i8;
1698
            maxAssigned = true;
1699
          }
H
Hongze Cheng 已提交
1700
          break;
1701
        }
1702
        case TSDB_DATA_TYPE_SMALLINT: {
1703
          pColAgg->sum += colVal.value.i16;
1704
          if (!minAssigned || pColAgg->min > colVal.value.i16) {
1705
            pColAgg->min = colVal.value.i16;
1706
            minAssigned = true;
1707
          }
1708
          if (!maxAssigned || pColAgg->max < colVal.value.i16) {
1709
            pColAgg->max = colVal.value.i16;
1710
            maxAssigned = true;
1711
          }
H
Hongze Cheng 已提交
1712
          break;
1713
        }
1714 1715
        case TSDB_DATA_TYPE_INT: {
          pColAgg->sum += colVal.value.i32;
1716
          if (!minAssigned || pColAgg->min > colVal.value.i32) {
1717
            pColAgg->min = colVal.value.i32;
1718
            minAssigned = true;
1719
          }
1720
          if (!maxAssigned || pColAgg->max < colVal.value.i32) {
1721
            pColAgg->max = colVal.value.i32;
1722
            maxAssigned = true;
1723
          }
H
Hongze Cheng 已提交
1724
          break;
1725 1726 1727
        }
        case TSDB_DATA_TYPE_BIGINT: {
          pColAgg->sum += colVal.value.i64;
1728
          if (!minAssigned || pColAgg->min > colVal.value.i64) {
1729
            pColAgg->min = colVal.value.i64;
1730
            minAssigned = true;
1731
          }
1732
          if (!maxAssigned || pColAgg->max < colVal.value.i64) {
1733
            pColAgg->max = colVal.value.i64;
1734
            maxAssigned = true;
1735
          }
H
Hongze Cheng 已提交
1736
          break;
1737
        }
1738
        case TSDB_DATA_TYPE_FLOAT: {
H
Hongze Cheng 已提交
1739 1740 1741
          *(double *)(&pColAgg->sum) += colVal.value.f;
          if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) {
            *(double *)(&pColAgg->min) = colVal.value.f;
1742
            minAssigned = true;
1743
          }
H
Hongze Cheng 已提交
1744 1745
          if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) {
            *(double *)(&pColAgg->max) = colVal.value.f;
1746
            maxAssigned = true;
1747
          }
H
Hongze Cheng 已提交
1748
          break;
1749
        }
1750
        case TSDB_DATA_TYPE_DOUBLE: {
H
Hongze Cheng 已提交
1751 1752 1753
          *(double *)(&pColAgg->sum) += colVal.value.d;
          if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) {
            *(double *)(&pColAgg->min) = colVal.value.d;
1754
            minAssigned = true;
1755
          }
H
Hongze Cheng 已提交
1756 1757
          if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) {
            *(double *)(&pColAgg->max) = colVal.value.d;
1758
            maxAssigned = true;
1759
          }
H
Hongze Cheng 已提交
1760
          break;
1761
        }
H
Hongze Cheng 已提交
1762 1763
        case TSDB_DATA_TYPE_VARCHAR:
          break;
1764
        case TSDB_DATA_TYPE_TIMESTAMP: {
1765
          if (!minAssigned || pColAgg->min > colVal.value.i64) {
1766
            pColAgg->min = colVal.value.i64;
1767
            minAssigned = true;
1768
          }
1769
          if (!maxAssigned || pColAgg->max < colVal.value.i64) {
1770
            pColAgg->max = colVal.value.i64;
1771
            maxAssigned = true;
1772
          }
H
Hongze Cheng 已提交
1773
          break;
1774
        }
H
Hongze Cheng 已提交
1775 1776
        case TSDB_DATA_TYPE_NCHAR:
          break;
1777
        case TSDB_DATA_TYPE_UTINYINT: {
1778
          pColAgg->sum += colVal.value.u8;
1779
          if (!minAssigned || pColAgg->min > colVal.value.u8) {
1780
            pColAgg->min = colVal.value.u8;
1781
            minAssigned = true;
1782
          }
1783
          if (!maxAssigned || pColAgg->max < colVal.value.u8) {
1784
            pColAgg->max = colVal.value.u8;
1785
            maxAssigned = true;
1786
          }
H
Hongze Cheng 已提交
1787
          break;
1788
        }
1789
        case TSDB_DATA_TYPE_USMALLINT: {
1790
          pColAgg->sum += colVal.value.u16;
1791
          if (!minAssigned || pColAgg->min > colVal.value.u16) {
1792
            pColAgg->min = colVal.value.u16;
1793
            minAssigned = true;
1794
          }
1795
          if (!maxAssigned || pColAgg->max < colVal.value.u16) {
1796
            pColAgg->max = colVal.value.u16;
1797
            maxAssigned = true;
1798
          }
H
Hongze Cheng 已提交
1799
          break;
1800
        }
1801
        case TSDB_DATA_TYPE_UINT: {
1802
          pColAgg->sum += colVal.value.u32;
1803
          if (!minAssigned || pColAgg->min > colVal.value.u32) {
1804
            pColAgg->min = colVal.value.u32;
1805
            minAssigned = true;
1806
          }
1807
          if (!minAssigned || pColAgg->max < colVal.value.u32) {
1808
            pColAgg->max = colVal.value.u32;
1809
            maxAssigned = true;
1810
          }
H
Hongze Cheng 已提交
1811
          break;
1812
        }
1813
        case TSDB_DATA_TYPE_UBIGINT: {
1814
          pColAgg->sum += colVal.value.u64;
1815
          if (!minAssigned || pColAgg->min > colVal.value.u64) {
1816
            pColAgg->min = colVal.value.u64;
1817
            minAssigned = true;
1818
          }
1819
          if (!maxAssigned || pColAgg->max < colVal.value.u64) {
1820
            pColAgg->max = colVal.value.u64;
1821
            maxAssigned = true;
1822
          }
H
Hongze Cheng 已提交
1823
          break;
1824
        }
H
Hongze Cheng 已提交
1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840
        case TSDB_DATA_TYPE_JSON:
          break;
        case TSDB_DATA_TYPE_VARBINARY:
          break;
        case TSDB_DATA_TYPE_DECIMAL:
          break;
        case TSDB_DATA_TYPE_BLOB:
          break;
        case TSDB_DATA_TYPE_MEDIUMBLOB:
          break;
        default:
          ASSERT(0);
      }
    }
  }
}
H
Hongze Cheng 已提交
1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877

int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut,
                     int32_t *szOut, uint8_t **ppBuf) {
  int32_t code = 0;

  ASSERT(szIn > 0 && ppOut);

  if (cmprAlg == NO_COMPRESSION) {
    code = tRealloc(ppOut, nOut + szIn);
    if (code) goto _exit;

    memcpy(*ppOut + nOut, pIn, szIn);
    *szOut = szIn;
  } else {
    int32_t size = szIn + COMP_OVERFLOW_BYTES;

    code = tRealloc(ppOut, nOut + size);
    if (code) goto _exit;

    if (cmprAlg == TWO_STAGE_COMP) {
      ASSERT(ppBuf);
      code = tRealloc(ppBuf, size);
      if (code) goto _exit;
    }

    *szOut =
        tDataTypes[type].compFunc(pIn, szIn, szIn / tDataTypes[type].bytes, *ppOut + nOut, size, cmprAlg, *ppBuf, size);
    if (*szOut <= 0) {
      code = TSDB_CODE_COMPRESS_ERROR;
      goto _exit;
    }
  }

_exit:
  return code;
}

H
Hongze Cheng 已提交
1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907
int32_t tsdbDecmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t szOut,
                       uint8_t **ppBuf) {
  int32_t code = 0;

  code = tRealloc(ppOut, szOut);
  if (code) goto _exit;

  if (cmprAlg == NO_COMPRESSION) {
    ASSERT(szIn == szOut);
    memcpy(*ppOut, pIn, szOut);
  } else {
    if (cmprAlg == TWO_STAGE_COMP) {
      code = tRealloc(ppBuf, szOut + COMP_OVERFLOW_BYTES);
      if (code) goto _exit;
    }

    int32_t size = tDataTypes[type].decompFunc(pIn, szIn, szOut / tDataTypes[type].bytes, *ppOut, szOut, cmprAlg,
                                               *ppBuf, szOut + COMP_OVERFLOW_BYTES);
    if (size <= 0) {
      code = TSDB_CODE_COMPRESS_ERROR;
      goto _exit;
    }

    ASSERT(size == szOut);
  }

_exit:
  return code;
}

H
Hongze Cheng 已提交
1908
int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol, uint8_t **ppOut, int32_t nOut,
H
Hongze Cheng 已提交
1909
                        uint8_t **ppBuf) {
H
Hongze Cheng 已提交
1910 1911
  int32_t code = 0;

H
Hongze Cheng 已提交
1912 1913
  ASSERT(pColData->flag && (pColData->flag != HAS_NONE) && (pColData->flag != HAS_NULL));

H
Hongze Cheng 已提交
1914 1915 1916 1917 1918
  pBlockCol->szBitmap = 0;
  pBlockCol->szOffset = 0;
  pBlockCol->szValue = 0;

  int32_t size = 0;
H
Hongze Cheng 已提交
1919 1920
  // bitmap
  if (pColData->flag != HAS_VALUE) {
H
Hongze Cheng 已提交
1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956
    uint8_t *pBitMap = pColData->pBitMap;
    int32_t  szBitMap = BIT2_SIZE(pColData->nVal);

    // BIT2 to BIT1
    if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
      szBitMap = BIT1_SIZE(pColData->nVal);
      pBitMap = taosMemoryCalloc(1, szBitMap);
      if (pBitMap == NULL) {
        code = TSDB_CODE_OUT_OF_MEMORY;
        goto _exit;
      }

      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
        uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
        switch (pColData->flag) {
          case (HAS_NULL | HAS_NONE):
            SET_BIT1(pBitMap, iVal, v);
            break;
          case (HAS_VALUE | HAS_NONE):
            if (v) {
              SET_BIT1(pBitMap, iVal, 1);
            } else {
              SET_BIT1(pBitMap, iVal, 0);
            }
            break;
          case (HAS_VALUE | HAS_NULL):
            SET_BIT1(pBitMap, iVal, v - 1);
            break;
          default:
            ASSERT(0);
        }
      }
    }

    code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap,
                        ppBuf);
H
Hongze Cheng 已提交
1957
    if (code) goto _exit;
H
Hongze Cheng 已提交
1958 1959 1960 1961

    if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
      taosMemoryFree(pBitMap);
    }
H
Hongze Cheng 已提交
1962
  }
H
Hongze Cheng 已提交
1963
  size += pBlockCol->szBitmap;
H
Hongze Cheng 已提交
1964 1965 1966 1967

  // offset
  if (IS_VAR_DATA_TYPE(pColData->type)) {
    code = tsdbCmprData((uint8_t *)pColData->aOffset, sizeof(int32_t) * pColData->nVal, TSDB_DATA_TYPE_INT, cmprAlg,
H
Hongze Cheng 已提交
1968
                        ppOut, nOut + size, &pBlockCol->szOffset, ppBuf);
H
Hongze Cheng 已提交
1969 1970
    if (code) goto _exit;
  }
H
Hongze Cheng 已提交
1971
  size += pBlockCol->szOffset;
H
Hongze Cheng 已提交
1972 1973 1974

  // value
  if (pColData->flag != (HAS_NULL | HAS_NONE)) {
H
Hongze Cheng 已提交
1975
    code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, ppOut, nOut + size,
H
Hongze Cheng 已提交
1976 1977 1978
                        &pBlockCol->szValue, ppBuf);
    if (code) goto _exit;
  }
H
Hongze Cheng 已提交
1979
  size += pBlockCol->szValue;
H
Hongze Cheng 已提交
1980 1981

  // checksum
H
Hongze Cheng 已提交
1982 1983 1984 1985
  size += sizeof(TSCKSUM);
  code = tRealloc(ppOut, nOut + size);
  if (code) goto _exit;
  taosCalcChecksumAppend(0, *ppOut + nOut, size);
H
Hongze Cheng 已提交
1986 1987 1988 1989

_exit:
  return code;
}
H
Hongze Cheng 已提交
1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000

int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData,
                          uint8_t **ppBuf) {
  int32_t code = 0;

  int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM);
  if (!taosCheckChecksumWhole(pIn, size)) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _exit;
  }

H
Hongze Cheng 已提交
2001 2002
  ASSERT(pColData->cid == pBlockCol->cid);
  ASSERT(pColData->type == pBlockCol->type);
H
Hongze Cheng 已提交
2003 2004 2005 2006 2007 2008 2009 2010
  pColData->smaOn = pBlockCol->smaOn;
  pColData->flag = pBlockCol->flag;
  pColData->nVal = nVal;
  pColData->nData = pBlockCol->szOrigin;

  uint8_t *p = pIn;
  // bitmap
  if (pBlockCol->szBitmap) {
H
Hongze Cheng 已提交
2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
    if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
      uint8_t *pBitMap = NULL;
      code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap,
                            BIT1_SIZE(pColData->nVal), ppBuf);
      if (code) goto _exit;

      code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal));
      if (code) {
        tFree(pBitMap);
        goto _exit;
      }

      // BIT1 to BIT2
      for (int32_t iVal = 0; iVal < nVal; iVal++) {
        uint8_t v = GET_BIT1(pBitMap, iVal);
        switch (pBlockCol->flag) {
          case (HAS_NULL | HAS_NONE):
            SET_BIT2(pColData->pBitMap, iVal, v);
            break;
          case (HAS_VALUE | HAS_NONE):
            if (v) {
              SET_BIT2(pColData->pBitMap, iVal, 2);
            } else {
              SET_BIT2(pColData->pBitMap, iVal, 0);
            }
            break;
          case (HAS_VALUE | HAS_NULL):
            SET_BIT2(pColData->pBitMap, iVal, v + 1);
            break;
          default:
            ASSERT(0);
        }
      }

      tFree(pBitMap);
    } else {
      code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap,
                            BIT2_SIZE(pColData->nVal), ppBuf);
      if (code) goto _exit;
    }
H
Hongze Cheng 已提交
2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
  }
  p += pBlockCol->szBitmap;

  // offset
  if (pBlockCol->szOffset) {
    code = tsdbDecmprData(p, pBlockCol->szOffset, TSDB_DATA_TYPE_INT, cmprAlg, (uint8_t **)&pColData->aOffset,
                          sizeof(int32_t) * pColData->nVal, ppBuf);
    if (code) goto _exit;
  }
  p += pBlockCol->szOffset;

  // value
  if (pBlockCol->szValue) {
    code = tsdbDecmprData(p, pBlockCol->szValue, pColData->type, cmprAlg, &pColData->pData, pColData->nData, ppBuf);
    if (code) goto _exit;
  }
  p += pBlockCol->szValue;

H
Hongze Cheng 已提交
2069 2070 2071 2072
_exit:
  return code;
}

H
Hongze Cheng 已提交
2073
int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) {
H
Hongze Cheng 已提交
2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
  int32_t code = 0;

  // alloc
  code = tRealloc(ppOut, size);
  if (code) goto _exit;

  // seek
  int64_t n = taosLSeekFile(pFD, offset, SEEK_SET);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _exit;
  }

  // read
  n = taosReadFile(pFD, *ppOut, size);
  if (n < 0) {
    code = TAOS_SYSTEM_ERROR(errno);
    goto _exit;
  } else if (n < size) {
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _exit;
  }

  // check
H
Hongze Cheng 已提交
2098
  if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) {
H
Hongze Cheng 已提交
2099 2100 2101 2102
    code = TSDB_CODE_FILE_CORRUPTED;
    goto _exit;
  }

H
Hongze Cheng 已提交
2103 2104
_exit:
  return code;
2105
}