tdataformat.h 12.6 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
hzcheng 已提交
15
#ifndef _TD_DATA_FORMAT_H_
H
more  
Hongze Cheng 已提交
16 17 18
#define _TD_DATA_FORMAT_H_

#include <stdint.h>
H
hzcheng 已提交
19
#include <stdlib.h>
H
hzcheng 已提交
20
#include <string.h>
H
more  
Hongze Cheng 已提交
21

H
Hongze Cheng 已提交
22
#include "talgo.h"
H
hzcheng 已提交
23
#include "taosdef.h"
H
TD-166  
hzcheng 已提交
24
#include "tutil.h"
H
hzcheng 已提交
25

H
more  
hzcheng 已提交
26 27 28
#ifdef __cplusplus
extern "C" {
#endif
H
hzcheng 已提交
29

H
Hongze Cheng 已提交
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
#define STR_TO_VARSTR(x, str)             \
  do {                                    \
    VarDataLenT __len = strlen(str);      \
    *(VarDataLenT *)(x) = __len;          \
    strncpy(varDataVal(x), (str), __len); \
  } while (0);

#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs)      \
  do {                                                 \
    char *_e = stpncpy(varDataVal(x), (str), (_maxs)); \
    varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE));   \
  } while (0)

#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \
  do {                                         \
    *(VarDataLenT *)(x) = (_size);             \
    strncpy(varDataVal(x), (str), (_size));    \
  } while (0);
H
hjxilinx 已提交
48

H
hzcheng 已提交
49 50 51 52 53
// ----------------- TSDB COLUMN DEFINITION
typedef struct {
  int8_t  type;    // Column type
  int16_t colId;   // column ID
  int32_t bytes;   // column bytes
H
TD-166  
hzcheng 已提交
54
  int32_t offset;  // point offset in SDataRow after the header part
H
hzcheng 已提交
55 56 57 58 59 60 61 62 63 64 65 66 67 68
} STColumn;

#define colType(col) ((col)->type)
#define colColId(col) ((col)->colId)
#define colBytes(col) ((col)->bytes)
#define colOffset(col) ((col)->offset)

#define colSetType(col, t) (colType(col) = (t))
#define colSetColId(col, id) (colColId(col) = (id))
#define colSetBytes(col, b) (colBytes(col) = (b))
#define colSetOffset(col, o) (colOffset(col) = (o))

// ----------------- TSDB SCHEMA DEFINITION
typedef struct {
H
TD-166  
hzcheng 已提交
69
  int      totalCols;  // Total columns allocated
H
hzcheng 已提交
70
  int      numOfCols;  // Number of columns appended
H
TD-166  
hzcheng 已提交
71 72
  int      tlen;       // maximum length of a SDataRow without the header part
  int      flen;       // First part length in a SDataRow after the header part
H
hzcheng 已提交
73 74 75 76
  STColumn columns[];
} STSchema;

#define schemaNCols(s) ((s)->numOfCols)
H
TD-166  
hzcheng 已提交
77 78 79
#define schemaTotalCols(s) ((s)->totalCols)
#define schemaTLen(s) ((s)->tlen)
#define schemaFLen(s) ((s)->flen)
H
hzcheng 已提交
80
#define schemaColAt(s, i) ((s)->columns + i)
H
Hongze Cheng 已提交
81
#define tdFreeSchema(s) tfree((s))
H
hzcheng 已提交
82 83

STSchema *tdNewSchema(int32_t nCols);
H
TD-166  
hzcheng 已提交
84
int       tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
H
hzcheng 已提交
85
STSchema *tdDupSchema(STSchema *pSchema);
H
TD-27  
hzcheng 已提交
86 87 88
int       tdGetSchemaEncodeSize(STSchema *pSchema);
void *    tdEncodeSchema(void *dst, STSchema *pSchema);
STSchema *tdDecodeSchema(void **psrc);
H
hzcheng 已提交
89

H
more  
Hongze Cheng 已提交
90 91
// ----------------- Data row structure

H
hzcheng 已提交
92
/* A data row, the format is like below:
H
TD-166  
hzcheng 已提交
93 94 95 96 97 98 99
 * |<------------------------------------- len ---------------------------------->|
 * |<--Head ->|<---------   flen -------------->|                                 |
 * +----------+---------------------------------+---------------------------------+
 * | int32_t  |                                 |                                 |
 * +----------+---------------------------------+---------------------------------+
 * |   len    |           First part            |             Second part         |
 * +----------+---------------------------------+---------------------------------+
H
more  
Hongze Cheng 已提交
100
 */
H
hzcheng 已提交
101 102
typedef void *SDataRow;

H
TD-166  
hzcheng 已提交
103
#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
H
hzcheng 已提交
104

H
hzcheng 已提交
105
#define dataRowLen(r) (*(int32_t *)(r))
H
hzcheng 已提交
106
#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
H
TD-34  
hzcheng 已提交
107
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
H
hzcheng 已提交
108
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
H
hzcheng 已提交
109
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
H
TD-166  
hzcheng 已提交
110
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
H
hzcheng 已提交
111

H
hzcheng 已提交
112
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
H
hzcheng 已提交
113
void     tdFreeDataRow(SDataRow row);
H
TD-166  
hzcheng 已提交
114
void     tdInitDataRow(SDataRow row, STSchema *pSchema);
H
hzcheng 已提交
115
SDataRow tdDataRowDup(SDataRow row);
H
more  
Hongze Cheng 已提交
116

117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
  ASSERT(value != NULL);
  int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
  char *  ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));

  switch (type) {
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
      memcpy(ptr, value, varDataTLen(value));
      dataRowLen(row) += varDataTLen(value);
      break;
    default:
      memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
      break;
  }

  return 0;
}

H
TD-166  
hzcheng 已提交
137
// NOTE: offset here including the header size
H
TD-166  
hzcheng 已提交
138 139 140 141
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
  switch (type) {
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
H
hzcheng 已提交
142
      return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset));
H
TD-166  
hzcheng 已提交
143
    default:
H
hzcheng 已提交
144
      return POINTER_SHIFT(row, offset);
H
TD-166  
hzcheng 已提交
145 146 147
  }
}

H
TD-34  
hzcheng 已提交
148 149
// ----------------- Data column structure
typedef struct SDataCol {
H
TD-166  
hzcheng 已提交
150 151 152
  int8_t          type;       // column type
  int16_t         colId;      // column ID
  int             bytes;      // column data bytes defined
H
TD-166  
hzcheng 已提交
153
  int             offset;     // data offset in a SDataRow (including the header size)
H
TD-166  
hzcheng 已提交
154 155 156 157
  int             spaceSize;  // Total space size for this column
  int             len;        // column data length
  VarDataOffsetT *dataOff;    // For binary and nchar data, the offset in the data column
  void *          pData;      // Actual data pointer
H
TD-34  
hzcheng 已提交
158 159
} SDataCol;

H
TD-166  
hzcheng 已提交
160 161 162
static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }

void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
H
Haojun Liao 已提交
163 164
void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows);
H
TD-166  
hzcheng 已提交
165 166
void dataColSetOffset(SDataCol *pCol, int nEle);

H
TD-166  
hzcheng 已提交
167 168
bool isNEleNull(SDataCol *pCol, int nEle);
void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
H
TD-166  
hzcheng 已提交
169 170 171

// Get the data pointer from a column-wised data
static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
H
TD-166  
hzcheng 已提交
172 173 174
  switch (pCol->type) {
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
H
hzcheng 已提交
175
      return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
H
TD-166  
hzcheng 已提交
176 177 178
      break;

    default:
H
hzcheng 已提交
179
      return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
H
TD-166  
hzcheng 已提交
180
      break;
H
TD-166  
hzcheng 已提交
181 182 183
  }
}

H
TD-166  
hzcheng 已提交
184
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
H
TD-166  
hzcheng 已提交
185 186
  ASSERT(rows > 0);

H
TD-166  
hzcheng 已提交
187 188 189
  switch (pDataCol->type) {
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
H
TD-166  
hzcheng 已提交
190
      return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
H
TD-166  
hzcheng 已提交
191 192
      break;
    default:
H
TD-166  
hzcheng 已提交
193
      return TYPE_BYTES[pDataCol->type] * rows;
H
TD-166  
hzcheng 已提交
194 195 196
  }
}

H
TD-34  
hzcheng 已提交
197
typedef struct {
H
Hongze Cheng 已提交
198 199 200 201
  int maxRowSize;
  int maxCols;    // max number of columns
  int maxPoints;  // max number of points
  int bufSize;
H
TD-166  
hzcheng 已提交
202

H
Haojun Liao 已提交
203
  int      numOfRows;
H
TD-34  
hzcheng 已提交
204
  int      numOfCols;  // Total number of cols
H
TD-34  
hzcheng 已提交
205
  int      sversion;   // TODO: set sversion
H
TD-34  
hzcheng 已提交
206 207 208 209
  void *   buf;
  SDataCol cols[];
} SDataCols;

H
TD-34  
hzcheng 已提交
210
#define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
H
TD-166  
hzcheng 已提交
211
#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
H
TD-34  
hzcheng 已提交
212
#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
H
Haojun Liao 已提交
213
#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1))
H
TD-34  
hzcheng 已提交
214

H
TD-166  
hzcheng 已提交
215
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
H
TD-34  
hzcheng 已提交
216
void       tdResetDataCols(SDataCols *pCols);
H
TD-34  
hzcheng 已提交
217
void       tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
H
TD-100  
hzcheng 已提交
218
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
H
TD-34  
hzcheng 已提交
219
void       tdFreeDataCols(SDataCols *pCols);
H
TD-34  
hzcheng 已提交
220
void       tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols);
H
Hongze Cheng 已提交
221
void       tdPopDataColsPoints(SDataCols *pCols, int pointsToPop);  //!!!!
H
hzcheng 已提交
222
int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
H
TD-100  
hzcheng 已提交
223
void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows);
H
more  
Hongze Cheng 已提交
224

H
Hongze Cheng 已提交
225 226 227 228 229 230 231 232
// ----------------- K-V data row structure
/*
 * +----------+----------+---------------------------------+---------------------------------+
 * |  int16_t |  int16_t |                                 |                                 |
 * +----------+----------+---------------------------------+---------------------------------+
 * |    len   |   ncols  |           cols index            |             data part           |
 * +----------+----------+---------------------------------+---------------------------------+
 */
H
Hongze Cheng 已提交
233
typedef void *SKVRow;
H
Hongze Cheng 已提交
234 235 236 237 238 239

typedef struct {
  int16_t colId;
  int16_t offset;
} SColIdx;

H
Hongze Cheng 已提交
240
#define TD_KV_ROW_HEAD_SIZE 2 * sizeof(int16_t)
H
Hongze Cheng 已提交
241

H
Hongze Cheng 已提交
242 243 244 245 246 247 248 249 250
#define kvRowLen(r) (*(int16_t *)(r))
#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE)
#define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r))
#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r))
#define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset)
#define kvRowSetLen(r, len) kvRowLen(r) = (len)
#define kvRowSetNCols(r, n) kvRowNCols(r) = (n)
#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i))
H
Hongze Cheng 已提交
251

H
Hongze Cheng 已提交
252 253 254 255
SKVRow tdKVRowDup(SKVRow row);
SKVRow tdSetKVRowDataOfCol(SKVRow row, int16_t colId, int8_t type, void *value);
void * tdEncodeKVRow(void *buf, SKVRow row);
void * tdDecodeKVRow(void *buf, SKVRow *row);
H
Hongze Cheng 已提交
256 257 258 259 260 261 262 263 264 265 266

static FORCE_INLINE int comparTagId(const void *key1, const void *key2) {
  if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) {
    return 1;
  } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) {
    return -1;
  } else {
    return 0;
  }
}

H
Hongze Cheng 已提交
267 268
static FORCE_INLINE void *tdGetKVRowDataOfCol(SKVRow row, int16_t colId) {
  void *ret = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
H
Hongze Cheng 已提交
269
  if (ret == NULL) return NULL;
H
Hongze Cheng 已提交
270
  return kvRowColVal(row, (SColIdx *)ret);
H
Hongze Cheng 已提交
271 272
}

H
Hongze Cheng 已提交
273 274 275 276 277 278 279 280
// ----------------- K-V data row builder
typedef struct {
  int16_t  tCols;
  int16_t  nCols;
  SColIdx *pColIdx;
  int16_t  alloc;
  int16_t  size;
  void *   buf;
H
Hongze Cheng 已提交
281
} SKVRowBuilder;
H
Hongze Cheng 已提交
282

H
Hongze Cheng 已提交
283 284 285 286 287
int    tdInitKVRowBuilder(SKVRowBuilder *pBuilder);
void   tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
void   tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
int    tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value);
H
Hongze Cheng 已提交
288

T
Tao Liu 已提交
289 290 291
// ----------------- Tag row structure

/* A tag row, the format is like below:
T
Tao Liu 已提交
292
+----------+----------------------------------------------------------------+
H
Hongze Cheng 已提交
293
| STagRow  | STagCol | STagCol | STagCol | STagCol | ...| STagCol | STagCol |
T
Tao Liu 已提交
294 295 296 297 298 299 300
+----------+----------------------------------------------------------------+

pData
+----------+----------------------------------------------------------------+
| value 1     | value 2 |  value 3     | value 4       | ....|value n       |
+----------+----------------------------------------------------------------+

T
Tao Liu 已提交
301
 */
T
Tao Liu 已提交
302

T
Tao Liu 已提交
303 304 305 306 307 308 309 310 311 312
#define TD_TAG_ROW_HEAD_SIZE sizeof(int16_t)

#define tagRowNum(r) (*(int16_t *)(r))
#define tagRowArray(r) POINTER_SHIFT(r, TD_TAG_ROW_HEAD_SIZE)
//#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
//#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
//#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
//#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)

typedef struct {
H
Hongze Cheng 已提交
313 314 315
  int16_t  colId;  // column ID
  int16_t  colType;
  uint16_t offset;  // to store value for numeric col or offset for binary/Nchar
T
Tao Liu 已提交
316 317 318
} STagCol;

typedef struct {
H
Hongze Cheng 已提交
319 320 321 322 323
  int32_t  len;
  void *   pData;  // Space to store the tag value
  uint16_t dataLen;
  int16_t  ncols;  // Total columns allocated
  STagCol  tagCols[];
T
Tao Liu 已提交
324 325
} STagRow;

T
Tao Liu 已提交
326 327
#define tagColSize(r) (sizeof(STagCol) + r.colLen)

H
Hongze Cheng 已提交
328 329 330 331 332 333 334 335 336 337
int       tdSetTagCol(SDataRow row, void *value, int16_t len, int8_t type,
                      int16_t colId);                   // insert tag value and update all the information
int       tdDeleteTagCol(SDataRow row, int16_t colId);  // delete tag value and update all the information
void *    tdQueryTagByID(SDataRow row, int16_t colId, int16_t *type);  // if find tag, 0, else return -1;
int       tdAppendTagColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int16_t colId);
SDataRow  tdTagRowDup(SDataRow row);
void      tdFreeTagRow(SDataRow row);
SDataRow  tdTagRowDecode(SDataRow row);
int       tdTagRowCpy(SDataRow dst, SDataRow src);
void *    tdNewTagRowFromSchema(STSchema *pSchema, int16_t numofTags);
T
Tao Liu 已提交
338
STSchema *tdGetSchemaFromData(SDataRow *row);
T
Tao Liu 已提交
339

H
more  
hzcheng 已提交
340 341 342 343
#ifdef __cplusplus
}
#endif

H
hzcheng 已提交
344
#endif  // _TD_DATA_FORMAT_H_