tdataformat.h 30.0 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
S
Shengliang Guan 已提交
15

16 17
#ifndef _TD_COMMON_DATA_FORMAT_H_
#define _TD_COMMON_DATA_FORMAT_H_
H
more  
Hongze Cheng 已提交
18

S
TD-4088  
Shengliang Guan 已提交
19
#include "os.h"
H
Hongze Cheng 已提交
20
#include "talgo.h"
H
Haojun Liao 已提交
21
#include "ttypes.h"
H
TD-166  
hzcheng 已提交
22
#include "tutil.h"
H
hzcheng 已提交
23

H
more  
hzcheng 已提交
24 25 26
#ifdef __cplusplus
extern "C" {
#endif
H
hzcheng 已提交
27

C
Cary Xu 已提交
28 29
// Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
#define TD_SUPPORT_BITMAP
C
Cary Xu 已提交
30
#define TD_SUPPORT_READ2
K
Kaili Xu 已提交
31 32 33
#define TD_SUPPORT_BACK2  // suppport back compatibility of 2.0

#define TASSERT(x) ASSERT(x)
C
Cary Xu 已提交
34

35 36 37 38 39
#define STR_TO_VARSTR(x, str)                     \
  do {                                            \
    VarDataLenT __len = (VarDataLenT)strlen(str); \
    *(VarDataLenT *)(x) = __len;                  \
    memcpy(varDataVal(x), (str), __len);          \
H
Hongze Cheng 已提交
40 41
  } while (0);

H
Hongze Cheng 已提交
42 43 44 45 46 47
#define STR_TO_NET_VARSTR(x, str)                 \
  do {                                            \
    VarDataLenT __len = (VarDataLenT)strlen(str); \
    *(VarDataLenT *)(x) = htons(__len);           \
    memcpy(varDataVal(x), (str), __len);          \
  } while (0);
D
dapan1121 已提交
48

49 50
#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs)                         \
  do {                                                                    \
H
Hui Li 已提交
51
    char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \
52
    varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE));                      \
H
Hongze Cheng 已提交
53 54
  } while (0)

55 56 57 58
#define STR_WITH_SIZE_TO_VARSTR(x, str, _size)  \
  do {                                          \
    *(VarDataLenT *)(x) = (VarDataLenT)(_size); \
    memcpy(varDataVal(x), (str), (_size));      \
H
Hongze Cheng 已提交
59
  } while (0);
H
hjxilinx 已提交
60

61
// ----------------- TSDB COLUMN DEFINITION
62
#pragma pack(push, 1)
63
typedef struct {
64 65
  col_id_t colId;        // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1))
  int32_t  type : 8;     // column type
C
Cary Xu 已提交
66
  int32_t  bytes : 24;   // column bytes (0~16M)
67 68
  int32_t  sma : 8;      // block SMA: 0, no SMA, 1, sum/min/max, 2, ...
  int32_t  offset : 24;  // point offset in STpRow after the header part.
69
} STColumn;
70
#pragma pack(pop)
71

S
Shengliang Guan 已提交
72 73 74
#define colType(col)   ((col)->type)
#define colColId(col)  ((col)->colId)
#define colBytes(col)  ((col)->bytes)
75 76
#define colOffset(col) ((col)->offset)

S
Shengliang Guan 已提交
77
#define colSetType(col, t)   (colType(col) = (t))
78
#define colSetColId(col, id) (colColId(col) = (id))
S
Shengliang Guan 已提交
79
#define colSetBytes(col, b)  (colBytes(col) = (b))
80 81 82 83
#define colSetOffset(col, o) (colOffset(col) = (o))

// ----------------- TSDB SCHEMA DEFINITION
typedef struct {
C
Cary Xu 已提交
84 85 86 87 88 89
  int32_t      numOfCols;  // Number of columns appended
  schema_ver_t version;    // schema version
  uint16_t     flen;       // First part length in a STpRow after the header part
  int32_t      vlen;       // pure value part length, excluded the overhead (bytes only)
  int32_t      tlen;       // maximum length of a STpRow without the header part
                           // (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
90 91 92
  STColumn columns[];
} STSchema;

S
Shengliang Guan 已提交
93 94 95 96 97
#define schemaNCols(s)    ((s)->numOfCols)
#define schemaVersion(s)  ((s)->version)
#define schemaTLen(s)     ((s)->tlen)
#define schemaFLen(s)     ((s)->flen)
#define schemaVLen(s)     ((s)->vlen)
98
#define schemaColAt(s, i) ((s)->columns + i)
wafwerar's avatar
wafwerar 已提交
99
#define tdFreeSchema(s)   taosMemoryFreeClear((s))
100

H
Hongze Cheng 已提交
101
STSchema *tdDupSchema(const STSchema *pSchema);
S
Shengliang Guan 已提交
102 103
int32_t   tdEncodeSchema(void **buf, STSchema *pSchema);
void     *tdDecodeSchema(void *buf, STSchema **pRSchema);
104

S
Shengliang Guan 已提交
105
static FORCE_INLINE int32_t comparColId(const void *key1, const void *key2) {
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
  if (*(int16_t *)key1 > ((STColumn *)key2)->colId) {
    return 1;
  } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) {
    return -1;
  } else {
    return 0;
  }
}

static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) {
  void *ptr = bsearch(&colId, (void *)pSchema->columns, schemaNCols(pSchema), sizeof(STColumn), comparColId);
  if (ptr == NULL) return NULL;
  return (STColumn *)ptr;
}

// ----------------- SCHEMA BUILDER DEFINITION
typedef struct {
C
Cary Xu 已提交
123 124 125 126 127 128 129
  int32_t      tCols;
  int32_t      nCols;
  schema_ver_t version;
  uint16_t     flen;
  int32_t      vlen;
  int32_t      tlen;
  STColumn    *columns;
130 131
} STSchemaBuilder;

S
Shengliang Guan 已提交
132
#define TD_VTYPE_BITS  2  // val type
C
Cary Xu 已提交
133
#define TD_VTYPE_PARTS 4  // 8 bits / TD_VTYPE_BITS = 4
S
Shengliang Guan 已提交
134
#define TD_VTYPE_OPTR  3  // TD_VTYPE_PARTS - 1, utilize to get remainder
C
Cary Xu 已提交
135

C
Cary Xu 已提交
136
#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS))
C
update  
Cary Xu 已提交
137
#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8))
C
Cary Xu 已提交
138

C
Cary Xu 已提交
139
int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
140
void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
C
Cary Xu 已提交
141
void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
142
int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes);
143
STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
H
Hongze Cheng 已提交
144

H
TD-1548  
Hongze Cheng 已提交
145
// ----------------- Semantic timestamp key definition
C
update  
Cary Xu 已提交
146
#ifdef TD_2_0
C
Cary Xu 已提交
147

H
TD-1548  
Hongze Cheng 已提交
148 149
typedef uint64_t TKEY;

S
Shengliang Guan 已提交
150 151
#define TKEY_INVALID       UINT64_MAX
#define TKEY_NULL          TKEY_INVALID
H
Hongze Cheng 已提交
152
#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
S
Shengliang Guan 已提交
153 154
#define TKEY_DELETE_FLAG   (((TKEY)1) << 62)
#define TKEY_VALUE_FILTER  (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG))
H
TD-1548  
Hongze Cheng 已提交
155 156

#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
S
Shengliang Guan 已提交
157
#define TKEY_IS_DELETED(tkey)  (((tkey)&TKEY_DELETE_FLAG) != 0)
H
TD-1548  
Hongze Cheng 已提交
158
#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
S
Shengliang Guan 已提交
159 160
#define tdGetTKEY(key)         (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
#define tdGetKey(tkey)         (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
H
TD-1548  
Hongze Cheng 已提交
161

D
dapan1121 已提交
162 163 164 165 166
#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff)

#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))

C
Cary Xu 已提交
167 168
#else

C
Cary Xu 已提交
169 170
// typedef uint64_t TKEY;
#define TKEY TSKEY
C
Cary Xu 已提交
171

S
Shengliang Guan 已提交
172 173
#define TKEY_INVALID       UINT64_MAX
#define TKEY_NULL          TKEY_INVALID
C
Cary Xu 已提交
174
#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
S
Shengliang Guan 已提交
175
#define TKEY_VALUE_FILTER  (~(TKEY_NEGATIVE_FLAG))
C
Cary Xu 已提交
176 177

#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
S
Shengliang Guan 已提交
178
#define TKEY_IS_DELETED(tkey)  (false)
C
Cary Xu 已提交
179

S
Shengliang Guan 已提交
180
#define tdGetTKEY(key)  (key)
C
Cary Xu 已提交
181
#define tdGetKey(tskey) (tskey)
C
Cary Xu 已提交
182 183 184 185 186 187 188 189

#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
#define MAX_TS_KEY ((TSKEY)0x7fffffffffffffff)

#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))

#endif

D
dapan1121 已提交
190 191 192 193 194 195 196 197 198 199 200
static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
  TSKEY lkey = key;
  if (key > MAX_TS_KEY) {
    lkey = MAX_TS_KEY;
  } else if (key < MIN_TS_KEY) {
    lkey = MIN_TS_KEY;
  }

  return tdGetTKEY(lkey);
}

S
Shengliang Guan 已提交
201
static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) {
H
TD-1548  
Hongze Cheng 已提交
202 203 204 205 206 207 208 209 210 211 212
  TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
  TSKEY key2 = tdGetKey(*(TKEY *)tkey2);

  if (key1 < key2) {
    return -1;
  } else if (key1 > key2) {
    return 1;
  } else {
    return 0;
  }
}
H
hzcheng 已提交
213

C
Cary Xu 已提交
214
#if 0
C
Cary Xu 已提交
215 216 217
// ----------------- Data row structure

/* A data row, the format is like below:
K
kailixu 已提交
218
 * |<------------------------------------------------ len ---------------------------------->|
C
Cary Xu 已提交
219 220 221 222 223 224 225 226 227 228 229
 * |<--     Head      -->|<---------   flen -------------->|                                 |
 * +---------------------+---------------------------------+---------------------------------+
 * | uint16_t |  int16_t |                                 |                                 |
 * +----------+----------+---------------------------------+---------------------------------+
 * |   len    | sversion |           First part            |             Second part         |
 * +----------+----------+---------------------------------+---------------------------------+
 *
 * NOTE: timestamp in this row structure is TKEY instead of TSKEY
 */
typedef void *SDataRow;

B
Bomin Zhang 已提交
230
#define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
H
hzcheng 已提交
231

S
Shengliang Guan 已提交
232 233 234 235 236 237 238 239 240
#define dataRowLen(r)                (*(TDRowLenT *)(r))  // 0~65535
#define dataRowEnd(r)                POINTER_SHIFT(r, dataRowLen(r))
#define dataRowVersion(r)            (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
#define dataRowTuple(r)              POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
#define dataRowTKey(r)               (*(TKEY *)(dataRowTuple(r)))
#define dataRowKey(r)                tdGetKey(dataRowTKey(r))
#define dataRowSetLen(r, l)          (dataRowLen(r) = (l))
#define dataRowSetVersion(r, v)      (dataRowVersion(r) = (v))
#define dataRowCpy(dst, r)           memcpy((dst), (r), dataRowLen(r))
H
TD-166  
hzcheng 已提交
241
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
S
Shengliang Guan 已提交
242
#define dataRowDeleted(r)            TKEY_IS_DELETED(dataRowTKey(r))
H
hzcheng 已提交
243

C
Cary Xu 已提交
244 245
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
void     tdFreeDataRow(SDataRow row);
H
TD-166  
hzcheng 已提交
246
void     tdInitDataRow(SDataRow row, STSchema *pSchema);
C
Cary Xu 已提交
247
SDataRow tdDataRowDup(SDataRow row);
H
more  
Hongze Cheng 已提交
248

249
// offset here not include dataRow header length
S
Shengliang Guan 已提交
250
static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
251
                                           int32_t offset) {
252
  assert(value != NULL);
253 254
  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;

H
TD-1548  
Hongze Cheng 已提交
255 256
  if (IS_VAR_DATA_TYPE(type)) {
    *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
257 258 259
    if (isCopyVarData) {
      memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
    }
H
TD-1548  
Hongze Cheng 已提交
260 261 262
    dataRowLen(row) += varDataTLen(value);
  } else {
    if (offset == 0) {
263
      assert(type == TSDB_DATA_TYPE_TIMESTAMP);
H
TD-1548  
Hongze Cheng 已提交
264
      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
C
Cary Xu 已提交
265
      memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]);
H
TD-1548  
Hongze Cheng 已提交
266
    } else {
267
      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
H
TD-1548  
Hongze Cheng 已提交
268
    }
269 270 271 272 273
  }

  return 0;
}

274
// offset here not include dataRow header length
S
Shengliang Guan 已提交
275
static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
276 277 278
  return tdAppendDataColVal(row, value, true, type, offset);
}

C
Cary Xu 已提交
279
// NOTE: offset here including the header size
C
Cary Xu 已提交
280
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
C
Cary Xu 已提交
281 282 283 284 285 286 287
  if (IS_VAR_DATA_TYPE(type)) {
    return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
  } else {
    return POINTER_SHIFT(row, offset);
  }
}

S
Shengliang Guan 已提交
288
static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) {
289 290 291
  return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset);
}

S
Shengliang Guan 已提交
292
static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
293
  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
H
Hongze Cheng 已提交
294
  int8_t  type = pSchema->columns[idx].type;
295 296 297 298

  return tdGetRowDataOfCol(row, type, offset);
}

S
Shengliang Guan 已提交
299
static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
300
  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
H
Hongze Cheng 已提交
301
  int8_t  type = pSchema->columns[idx].type;
302 303 304 305

  return isNull(tdGetRowDataOfCol(row, type, offset), type);
}

S
Shengliang Guan 已提交
306
static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) {
307
  int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset;
H
Hongze Cheng 已提交
308
  int8_t  type = pSchema->columns[idx].type;
309 310 311 312 313
  int16_t bytes = pSchema->columns[idx].bytes;

  setNull(tdGetRowDataOfCol(row, type, offset), type, bytes);
}

S
Shengliang Guan 已提交
314 315
static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src,
                                                STSchema *pSrcSchema, int32_t srcIdx) {
316
  int8_t type = pDstSchema->columns[dstIdx].type;
317
  assert(type == pSrcSchema->columns[srcIdx].type);
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
  void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx);
  void *value = tdGetPtrToCol(src, pSrcSchema, srcIdx);

  switch (type) {
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      *(VarDataOffsetT *)pData = *(VarDataOffsetT *)value;
      pData = POINTER_SHIFT(dst, *(VarDataOffsetT *)pData);
      value = POINTER_SHIFT(src, *(VarDataOffsetT *)value);
      memcpy(pData, value, varDataTLen(value));
      break;
    case TSDB_DATA_TYPE_NULL:
    case TSDB_DATA_TYPE_BOOL:
    case TSDB_DATA_TYPE_TINYINT:
    case TSDB_DATA_TYPE_UTINYINT:
      *(uint8_t *)pData = *(uint8_t *)value;
      break;
    case TSDB_DATA_TYPE_SMALLINT:
    case TSDB_DATA_TYPE_USMALLINT:
      *(uint16_t *)pData = *(uint16_t *)value;
      break;
    case TSDB_DATA_TYPE_INT:
    case TSDB_DATA_TYPE_UINT:
      *(uint32_t *)pData = *(uint32_t *)value;
      break;
    case TSDB_DATA_TYPE_BIGINT:
    case TSDB_DATA_TYPE_UBIGINT:
      *(uint64_t *)pData = *(uint64_t *)value;
      break;
    case TSDB_DATA_TYPE_FLOAT:
      SET_FLOAT_PTR(pData, value);
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      SET_DOUBLE_PTR(pData, value);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
354
      if (pSrcSchema->columns[srcIdx].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
355 356 357 358 359 360 361 362 363
        *(TSKEY *)pData = tdGetKey(*(TKEY *)value);
      } else {
        *(TSKEY *)pData = *(TSKEY *)value;
      }
      break;
    default:
      memcpy(pData, value, pSrcSchema->columns[srcIdx].bytes);
  }
}
C
Cary Xu 已提交
364
#endif
H
TD-34  
hzcheng 已提交
365 366
// ----------------- Data column structure
typedef struct SDataCol {
C
Cary Xu 已提交
367 368 369
  int8_t          type;        // column type
  uint8_t         bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
  uint8_t         reserve : 7;
H
TD-166  
hzcheng 已提交
370
  int16_t         colId;      // column ID
S
Shengliang Guan 已提交
371 372 373 374
  int32_t         bytes;      // column data bytes defined
  int32_t         offset;     // data offset in a SDataRow (including the header size)
  int32_t         spaceSize;  // Total space size for this column
  int32_t         len;        // column data length
H
TD-166  
hzcheng 已提交
375
  VarDataOffsetT *dataOff;    // For binary and nchar data, the offset in the data column
S
Shengliang Guan 已提交
376 377
  void           *pData;      // Actual data pointer
  void           *pBitmap;    // Bitmap pointer
L
lichuang 已提交
378
  TSKEY           ts;         // only used in last NULL column
H
TD-34  
hzcheng 已提交
379 380
} SDataCol;

K
kailixu 已提交
381
#define isAllRowsNull(pCol) ((pCol)->len == 0)
K
Kaili Xu 已提交
382
#define isAllRowsNone(pCol) ((pCol)->len == 0)
H
TD-166  
hzcheng 已提交
383 384
static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }

S
Shengliang Guan 已提交
385
int32_t tdAllocMemForCol(SDataCol *pCol, int32_t maxPoints);
386

S
Shengliang Guan 已提交
387 388 389
void    dataColInit(SDataCol *pDataCol, STColumn *pCol, int32_t maxPoints);
int32_t dataColAppendVal(SDataCol *pCol, const void *value, int32_t numOfRows, int32_t maxPoints);
void   *dataColSetOffset(SDataCol *pCol, int32_t nEle);
H
TD-166  
hzcheng 已提交
390

S
Shengliang Guan 已提交
391
bool isNEleNull(SDataCol *pCol, int32_t nEle);
H
TD-166  
hzcheng 已提交
392

C
Cary Xu 已提交
393
#if 0
H
TD-166  
hzcheng 已提交
394
// Get the data pointer from a column-wised data
S
Shengliang Guan 已提交
395
static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) {
K
kailixu 已提交
396
  if (isAllRowsNull(pCol)) {
C
Cary Xu 已提交
397
    return getNullValue(pCol->type);
K
kailixu 已提交
398
  }
H
Hongze Cheng 已提交
399 400 401 402
  if (IS_VAR_DATA_TYPE(pCol->type)) {
    return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
  } else {
    return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
H
TD-166  
hzcheng 已提交
403 404 405
  }
}

S
Shengliang Guan 已提交
406
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) {
407
  assert(rows > 0);
H
TD-166  
hzcheng 已提交
408

H
Hongze Cheng 已提交
409 410 411 412
  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
    return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
  } else {
    return TYPE_BYTES[pDataCol->type] * rows;
H
TD-166  
hzcheng 已提交
413 414
  }
}
C
Cary Xu 已提交
415
#endif
H
TD-34  
hzcheng 已提交
416
typedef struct {
C
update  
Cary Xu 已提交
417 418
  col_id_t  maxCols;    // max number of columns
  col_id_t  numOfCols;  // Total number of cols
S
Shengliang Guan 已提交
419 420 421
  int32_t   maxPoints;  // max number of points
  int32_t   numOfRows;
  int32_t   sversion;  // TODO: set sversion
H
Hongze Cheng 已提交
422
  SDataCol *cols;
H
TD-34  
hzcheng 已提交
423 424
} SDataCols;

S
Shengliang Guan 已提交
425
#define keyCol(pCols)              (&((pCols)->cols[0]))                    // Key column
C
Cary Xu 已提交
426
#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]  // the idx row of column-wised data
S
Shengliang Guan 已提交
427
#define dataColsKeyAt(pCols, idx)  tdGetKey(dataColsTKeyAt(pCols, idx))
428 429 430 431 432 433 434 435
static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) {
  if (pCols->numOfRows) {
    return dataColsTKeyAt(pCols, 0);
  } else {
    return TKEY_INVALID;
  }
}

S
Shengliang Guan 已提交
436
static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int32_t row) {
437
  assert(row < pCols->numOfRows);
438 439 440
  return dataColsKeyAt(pCols, row);
}

441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
static FORCE_INLINE TSKEY dataColsKeyFirst(SDataCols *pCols) {
  if (pCols->numOfRows) {
    return dataColsKeyAt(pCols, 0);
  } else {
    return TSDB_DATA_TIMESTAMP_NULL;
  }
}

static FORCE_INLINE TKEY dataColsTKeyLast(SDataCols *pCols) {
  if (pCols->numOfRows) {
    return dataColsTKeyAt(pCols, pCols->numOfRows - 1);
  } else {
    return TKEY_INVALID;
  }
}

static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) {
  if (pCols->numOfRows) {
    return dataColsKeyAt(pCols, pCols->numOfRows - 1);
  } else {
    return TSDB_DATA_TIMESTAMP_NULL;
  }
}
H
TD-34  
hzcheng 已提交
464

S
Shengliang Guan 已提交
465
SDataCols *tdNewDataCols(int32_t maxCols, int32_t maxRows);
H
TD-34  
hzcheng 已提交
466
void       tdResetDataCols(SDataCols *pCols);
S
Shengliang Guan 已提交
467
int32_t    tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
H
TD-100  
hzcheng 已提交
468
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
H
Hongze Cheng 已提交
469
SDataCols *tdFreeDataCols(SDataCols *pCols);
S
Shengliang Guan 已提交
470
int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull);
H
more  
Hongze Cheng 已提交
471

H
Hongze Cheng 已提交
472
// ----------------- K-V data row structure
C
Cary Xu 已提交
473 474
/* |<-------------------------------------- len -------------------------------------------->|
 * |<----- header  ----->|<--------------------------- body -------------------------------->|
H
Hongze Cheng 已提交
475
 * +----------+----------+---------------------------------+---------------------------------+
C
Cary Xu 已提交
476
 * | uint16_t |  int16_t |                                 |                                 |
H
Hongze Cheng 已提交
477 478 479 480
 * +----------+----------+---------------------------------+---------------------------------+
 * |    len   |   ncols  |           cols index            |             data part           |
 * +----------+----------+---------------------------------+---------------------------------+
 */
H
Hongze Cheng 已提交
481
typedef void *SKVRow;
H
Hongze Cheng 已提交
482 483

typedef struct {
C
Cary Xu 已提交
484 485
  int16_t  colId;
  uint16_t offset;
H
Hongze Cheng 已提交
486 487
} SColIdx;

C
Cary Xu 已提交
488
#define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
H
Hongze Cheng 已提交
489

S
Shengliang Guan 已提交
490 491 492 493 494 495 496
#define kvRowLen(r)            (*(TDRowLenT *)(r))
#define kvRowNCols(r)          (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
#define kvRowSetLen(r, len)    kvRowLen(r) = (len)
#define kvRowSetNCols(r, n)    kvRowNCols(r) = (n)
#define kvRowColIdx(r)         (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE)
#define kvRowValues(r)         POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r))
#define kvRowCpy(dst, r)       memcpy((dst), (r), kvRowLen(r))
H
Hongze Cheng 已提交
497
#define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
S
Shengliang Guan 已提交
498
#define kvRowColIdxAt(r, i)    (kvRowColIdx(r) + (i))
wafwerar's avatar
wafwerar 已提交
499
#define kvRowFree(r)           taosMemoryFreeClear(r)
S
Shengliang Guan 已提交
500 501 502 503 504 505 506 507 508 509 510 511 512 513
#define kvRowEnd(r)            POINTER_SHIFT(r, kvRowLen(r))
#define kvRowValLen(r)         (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r))
#define kvRowTKey(r)           (*(TKEY *)(kvRowValues(r)))
#define kvRowKey(r)            tdGetKey(kvRowTKey(r))
#define kvRowKeys(r)           POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t)))
#define kvRowDeleted(r)        TKEY_IS_DELETED(kvRowTKey(r))

SKVRow  tdKVRowDup(SKVRow row);
int32_t tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value);
int32_t tdEncodeKVRow(void **buf, SKVRow row);
void   *tdDecodeKVRow(void *buf, SKVRow *row);
void    tdSortKVRowByColIdx(SKVRow row);

static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) {
H
Hongze Cheng 已提交
514 515 516 517 518 519 520 521 522
  if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) {
    return 1;
  } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) {
    return -1;
  } else {
    return 0;
  }
}

H
Hongze Cheng 已提交
523
static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) {
H
Hongze Cheng 已提交
524
  void *ret = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
H
Hongze Cheng 已提交
525
  if (ret == NULL) return NULL;
H
Hongze Cheng 已提交
526
  return kvRowColVal(row, (SColIdx *)ret);
H
Hongze Cheng 已提交
527 528
}

L
liuyq-617 已提交
529 530 531 532
static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
  return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
}

C
Cary Xu 已提交
533
#if 0
C
Cary Xu 已提交
534
// offset here not include kvRow header length
S
Shengliang Guan 已提交
535
static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
536
                                         int32_t offset) {
537
  assert(value != NULL);
538
  int32_t  toffset = offset + TD_KV_ROW_HEAD_SIZE;
C
Cary Xu 已提交
539 540 541 542
  SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset);
  char *   ptr = (char *)POINTER_SHIFT(row, kvRowLen(row));

  pColIdx->colId = colId;
C
Cary Xu 已提交
543
  pColIdx->offset = kvRowLen(row);  // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE
C
Cary Xu 已提交
544 545

  if (IS_VAR_DATA_TYPE(type)) {
546 547 548
    if (isCopyValData) {
      memcpy(ptr, value, varDataTLen(value));
    }
C
Cary Xu 已提交
549 550
    kvRowLen(row) += varDataTLen(value);
  } else {
551
    if (offset == 0) {
552
      assert(type == TSDB_DATA_TYPE_TIMESTAMP);
C
Cary Xu 已提交
553 554 555 556 557 558 559 560 561 562
      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
      memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]);
    } else {
      memcpy(ptr, value, TYPE_BYTES[type]);
    }
    kvRowLen(row) += TYPE_BYTES[type];
  }

  return 0;
}
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
// NOTE: offset here including the header size
static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); }

static FORCE_INLINE void *tdGetKVRowValOfColEx(SKVRow row, int16_t colId, int32_t *nIdx) {
  while (*nIdx < kvRowNCols(row)) {
    SColIdx *pColIdx = kvRowColIdxAt(row, *nIdx);
    if (pColIdx->colId == colId) {
      ++(*nIdx);
      return tdGetKvRowDataOfCol(row, pColIdx->offset);
    } else if (pColIdx->colId > colId) {
      return NULL;
    } else {
      ++(*nIdx);
    }
  }
  return NULL;
}
C
Cary Xu 已提交
580
#endif
H
Hongze Cheng 已提交
581 582 583 584 585
// ----------------- K-V data row builder
typedef struct {
  int16_t  tCols;
  int16_t  nCols;
  SColIdx *pColIdx;
C
Cary Xu 已提交
586 587
  uint16_t alloc;
  uint16_t size;
S
Shengliang Guan 已提交
588
  void    *buf;
H
Hongze Cheng 已提交
589
} SKVRowBuilder;
H
Hongze Cheng 已提交
590

S
Shengliang Guan 已提交
591 592 593 594
int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder);
void    tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
void    tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
SKVRow  tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
H
Hongze Cheng 已提交
595

596
static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) {
H
Hongze Cheng 已提交
597 598
  if (pBuilder->nCols >= pBuilder->tCols) {
    pBuilder->tCols *= 2;
wafwerar's avatar
wafwerar 已提交
599
    SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
T
tickduan 已提交
600 601
    if (pColIdx == NULL) return -1;
    pBuilder->pColIdx = pColIdx;
H
Hongze Cheng 已提交
602 603 604 605 606 607 608
  }

  pBuilder->pColIdx[pBuilder->nCols].colId = colId;
  pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size;

  pBuilder->nCols++;

S
Shengliang Guan 已提交
609
  int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
H
Hongze Cheng 已提交
610 611 612 613
  if (tlen > pBuilder->alloc - pBuilder->size) {
    while (tlen > pBuilder->alloc - pBuilder->size) {
      pBuilder->alloc *= 2;
    }
wafwerar's avatar
wafwerar 已提交
614
    void *buf = taosMemoryRealloc(pBuilder->buf, pBuilder->alloc);
T
tickduan 已提交
615 616
    if (buf == NULL) return -1;
    pBuilder->buf = buf;
H
Hongze Cheng 已提交
617 618 619 620 621 622 623
  }

  memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen);
  pBuilder->size += tlen;

  return 0;
}
C
Cary Xu 已提交
624
#if 0
C
update  
Cary Xu 已提交
625
// ----------------- SMemRow appended with tuple row structure
C
Cary Xu 已提交
626
/*
C
Cary Xu 已提交
627
 * |---------|------------------------------------------------- len ---------------------------------->|
C
Cary Xu 已提交
628 629 630 631 632 633
 * |<--------     Head      ------>|<---------   flen -------------->|                                 |
 * |---------+---------------------+---------------------------------+---------------------------------+
 * | uint8_t | uint16_t |  int16_t |                                 |                                 |
 * |---------+----------+----------+---------------------------------+---------------------------------+
 * |  flag   |   len    | sversion |           First part            |             Second part         |
 * +---------+----------+----------+---------------------------------+---------------------------------+
C
Cary Xu 已提交
634 635 636
 *
 * NOTE: timestamp in this row structure is TKEY instead of TSKEY
 */
C
Cary Xu 已提交
637

C
update  
Cary Xu 已提交
638
// ----------------- SMemRow appended with extended K-V data row structure
C
Cary Xu 已提交
639 640
/* |--------------------|------------------------------------------------  len ---------------------------------->|
 * |<-------------     Head      ------------>|<---------   flen -------------->|                                 |
C
Cary Xu 已提交
641 642 643 644 645
 * |--------------------+----------+--------------------------------------------+---------------------------------+
 * | uint8_t | int16_t  | uint16_t |  int16_t |                                 |                                 |
 * |---------+----------+----------+----------+---------------------------------+---------------------------------+
 * |   flag  | sversion |   len    |   ncols  |           cols index            |             data part           |
 * |---------+----------+----------+----------+---------------------------------+---------------------------------+
C
Cary Xu 已提交
646 647
 */

C
Cary Xu 已提交
648 649
typedef void *SMemRow;

S
Shengliang Guan 已提交
650 651
#define TD_MEM_ROW_TYPE_SIZE        sizeof(uint8_t)
#define TD_MEM_ROW_KV_VER_SIZE      sizeof(int16_t)
C
Cary Xu 已提交
652
#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE)
S
Shengliang Guan 已提交
653 654
#define TD_MEM_ROW_DATA_HEAD_SIZE   (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE)
#define TD_MEM_ROW_KV_HEAD_SIZE     (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE)
C
Cary Xu 已提交
655

S
Shengliang Guan 已提交
656 657
#define SMEM_ROW_DATA 0x0U   // SDataRow
#define SMEM_ROW_KV   0x01U  // SKVRow
658 659 660

#define KVRatioConvert (0.9f)

C
Cary Xu 已提交
661
#define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
C
Cary Xu 已提交
662

S
Shengliang Guan 已提交
663 664 665 666 667
#define memRowSetType(r, t)  ((*(uint8_t *)(r)) = (t))  // set the total byte in case of dirty memory
#define isDataRowT(t)        (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
#define isDataRow(r)         (SMEM_ROW_DATA == memRowType(r))
#define isKvRowT(t)          (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
#define isKvRow(r)           (SMEM_ROW_KV == memRowType(r))
C
Cary Xu 已提交
668
#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert))
C
Cary Xu 已提交
669

C
Cary Xu 已提交
670 671
#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)  // section after flag
#define memRowKvBody(r) \
C
Cary Xu 已提交
672 673 674
  POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE)  // section after flag + sversion as to reuse SKVRow

#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r))  //  0~65535
S
Shengliang Guan 已提交
675
#define memRowKvLen(r)   (*(TDRowLenT *)memRowKvBody(r))    //  0~65535
C
Cary Xu 已提交
676

K
kailixu 已提交
677
#define memRowDataTLen(r) \
C
Cary Xu 已提交
678
  ((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE))  // using uint32_t/int32_t to store the TLen
C
Cary Xu 已提交
679

C
Cary Xu 已提交
680
#define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE))
C
Cary Xu 已提交
681

S
Shengliang Guan 已提交
682
#define memRowLen(r)  (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
C
Cary Xu 已提交
683
#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r))  // using uint32_t/int32_t to store the TLen
C
Cary Xu 已提交
684

685 686 687 688 689 690 691 692
static FORCE_INLINE char *memRowEnd(SMemRow row) {
  if (isDataRow(row)) {
    return (char *)dataRowEnd(memRowDataBody(row));
  } else {
    return (char *)kvRowEnd(memRowKvBody(row));
  }
}

S
Shengliang Guan 已提交
693 694 695
#define memRowDataVersion(r)     dataRowVersion(memRowDataBody(r))
#define memRowKvVersion(r)       (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
#define memRowVersion(r)         (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r))  // schema version
C
Cary Xu 已提交
696
#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v))
S
Shengliang Guan 已提交
697
#define memRowTuple(r)           (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r)))
C
Cary Xu 已提交
698

C
Cary Xu 已提交
699
#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r)))
S
Shengliang Guan 已提交
700
#define memRowKey(r)  (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r)))
H
Hongze Cheng 已提交
701
#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r)))
C
Cary Xu 已提交
702 703 704 705 706 707 708 709
#define memRowSetTKey(r, k)                 \
  do {                                      \
    if (isDataRow(r)) {                     \
      dataRowTKey(memRowDataBody(r)) = (k); \
    } else {                                \
      kvRowTKey(memRowKvBody(r)) = (k);     \
    }                                       \
  } while (0)
C
Cary Xu 已提交
710

S
Shengliang Guan 已提交
711 712 713
#define memRowSetLen(r, l)          (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l))
#define memRowSetVersion(r, v)      (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v))
#define memRowCpy(dst, r)           memcpy((dst), (r), memRowTLen(r))
C
Cary Xu 已提交
714
#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE)
S
Shengliang Guan 已提交
715
#define memRowDeleted(r)            TKEY_IS_DELETED(memRowTKey(r))
C
Cary Xu 已提交
716

C
Cary Xu 已提交
717
SMemRow tdMemRowDup(SMemRow row);
718 719
void    tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull);

C
Cary Xu 已提交
720
// NOTE: offset here including the header size
721
static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int16_t colId, int8_t colType, uint16_t offset) {
C
Cary Xu 已提交
722
  if (isDataRow(row)) {
723
    return tdGetRowDataOfCol(memRowDataBody(row), colType, offset);
C
Cary Xu 已提交
724
  } else {
725
    return tdGetKVRowValOfCol(memRowKvBody(row), colId);
C
Cary Xu 已提交
726 727 728
  }
}

729 730 731 732 733 734 735 736 737 738 739 740 741 742
/**
 * NOTE:
 *  1. Applicable to scan columns one by one
 *  2. offset here including the header size
 */
static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_t colType, int32_t offset,
                                                 int32_t *kvNIdx) {
  if (isDataRow(row)) {
    return tdGetRowDataOfCol(memRowDataBody(row), colType, offset);
  } else {
    return tdGetKVRowValOfColEx(memRowKvBody(row), colId, kvNIdx);
  }
}

S
Shengliang Guan 已提交
743
static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
744
                                             int8_t type, int32_t offset) {
745
  if (isDataRow(row)) {
746
    tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset);
747
  } else {
748
    tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset);
749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
  }
  return 0;
}

// make sure schema->flen appended for SDataRow
static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value, int8_t colType) {
  int32_t len = 0;
  if (IS_VAR_DATA_TYPE(colType)) {
    len += varDataTLen(value);
    if (rowType == SMEM_ROW_KV) {
      len += sizeof(SColIdx);
    }
  } else {
    if (rowType == SMEM_ROW_KV) {
      len += TYPE_BYTES[colType];
      len += sizeof(SColIdx);
    }
  }
  return len;
}

typedef struct {
  int16_t colId;
  uint8_t colType;
H
Hongze Cheng 已提交
773
  char *  colVal;
774 775
} SColInfo;

H
Hongze Cheng 已提交
776
static FORCE_INLINE void setSColInfo(SColInfo *colInfo, int16_t colId, uint8_t colType, char *colVal) {
777 778 779 780 781 782
  colInfo->colId = colId;
  colInfo->colType = colType;
  colInfo->colVal = colVal;
}

SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
C
Cary Xu 已提交
783
#endif
784

H
more  
hzcheng 已提交
785 786 787 788
#ifdef __cplusplus
}
#endif

H
Hongze Cheng 已提交
789
#endif /*_TD_COMMON_DATA_FORMAT_H_*/