tdataformat.h 7.5 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
hzcheng 已提交
15
#ifndef _TD_DATA_FORMAT_H_
H
more  
Hongze Cheng 已提交
16 17 18
#define _TD_DATA_FORMAT_H_

#include <stdint.h>
H
hzcheng 已提交
19
#include <stdlib.h>
H
hzcheng 已提交
20
#include <string.h>
H
more  
Hongze Cheng 已提交
21

H
hzcheng 已提交
22
#include "taosdef.h"
H
TD-166  
hzcheng 已提交
23
#include "tutil.h"
H
hzcheng 已提交
24

H
more  
hzcheng 已提交
25 26 27
#ifdef __cplusplus
extern "C" {
#endif
H
hzcheng 已提交
28

29 30
#define STR_TO_VARSTR(x, str) do {VarDataLenT __len = strlen(str); \
  *(VarDataLenT*)(x) = __len; \
H
hjxilinx 已提交
31
  strncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), __len);} while(0);
32 33 34 35 36 37 38 39 40

#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) do {\
  char* _e = stpncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_maxs));\
  *(VarDataLenT*)(x) = _e - (x);\
} while(0)

#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) do {\
  *(VarDataLenT*)(x) = (_size); \
  strncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_size));\
H
hjxilinx 已提交
41 42
} while(0);

H
hzcheng 已提交
43 44 45 46 47
// ----------------- TSDB COLUMN DEFINITION
typedef struct {
  int8_t  type;    // Column type
  int16_t colId;   // column ID
  int32_t bytes;   // column bytes
H
TD-166  
hzcheng 已提交
48
  int32_t offset;  // point offset in SDataRow after the header part
H
hzcheng 已提交
49 50 51 52 53 54 55 56 57 58 59 60 61 62
} STColumn;

#define colType(col) ((col)->type)
#define colColId(col) ((col)->colId)
#define colBytes(col) ((col)->bytes)
#define colOffset(col) ((col)->offset)

#define colSetType(col, t) (colType(col) = (t))
#define colSetColId(col, id) (colColId(col) = (id))
#define colSetBytes(col, b) (colBytes(col) = (b))
#define colSetOffset(col, o) (colOffset(col) = (o))

// ----------------- TSDB SCHEMA DEFINITION
typedef struct {
H
TD-166  
hzcheng 已提交
63
  int      totalCols;  // Total columns allocated
H
hzcheng 已提交
64
  int      numOfCols;  // Number of columns appended
H
TD-166  
hzcheng 已提交
65 66
  int      tlen;       // maximum length of a SDataRow without the header part
  int      flen;       // First part length in a SDataRow after the header part
H
hzcheng 已提交
67 68 69 70
  STColumn columns[];
} STSchema;

#define schemaNCols(s) ((s)->numOfCols)
H
TD-166  
hzcheng 已提交
71 72 73
#define schemaTotalCols(s) ((s)->totalCols)
#define schemaTLen(s) ((s)->tlen)
#define schemaFLen(s) ((s)->flen)
H
hzcheng 已提交
74 75 76
#define schemaColAt(s, i) ((s)->columns + i)

STSchema *tdNewSchema(int32_t nCols);
H
TD-166  
hzcheng 已提交
77 78
#define   tdFreeSchema(s) tfree((s))
int       tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
H
hzcheng 已提交
79
STSchema *tdDupSchema(STSchema *pSchema);
H
TD-27  
hzcheng 已提交
80 81 82
int       tdGetSchemaEncodeSize(STSchema *pSchema);
void *    tdEncodeSchema(void *dst, STSchema *pSchema);
STSchema *tdDecodeSchema(void **psrc);
H
hzcheng 已提交
83

H
more  
Hongze Cheng 已提交
84 85
// ----------------- Data row structure

H
hzcheng 已提交
86
/* A data row, the format is like below:
H
TD-166  
hzcheng 已提交
87 88 89 90 91 92 93
 * |<------------------------------------- len ---------------------------------->|
 * |<--Head ->|<---------   flen -------------->|                                 |
 * +----------+---------------------------------+---------------------------------+
 * | int32_t  |                                 |                                 |
 * +----------+---------------------------------+---------------------------------+
 * |   len    |           First part            |             Second part         |
 * +----------+---------------------------------+---------------------------------+
H
more  
Hongze Cheng 已提交
94
 */
H
hzcheng 已提交
95 96
typedef void *SDataRow;

H
TD-166  
hzcheng 已提交
97
#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
H
hzcheng 已提交
98

H
hzcheng 已提交
99
#define dataRowLen(r) (*(int32_t *)(r))
H
TD-166  
hzcheng 已提交
100
#define dataRowTuple(r) POINTER_DRIFT(r, TD_DATA_ROW_HEAD_SIZE)
H
TD-34  
hzcheng 已提交
101
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
H
hzcheng 已提交
102
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
H
hzcheng 已提交
103
#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
H
TD-166  
hzcheng 已提交
104
#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
H
hzcheng 已提交
105

H
hzcheng 已提交
106
SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
H
hzcheng 已提交
107
void     tdFreeDataRow(SDataRow row);
H
TD-166  
hzcheng 已提交
108
void     tdInitDataRow(SDataRow row, STSchema *pSchema);
H
TD-166  
hzcheng 已提交
109
int      tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset);
H
hzcheng 已提交
110
SDataRow tdDataRowDup(SDataRow row);
H
more  
Hongze Cheng 已提交
111

H
TD-166  
hzcheng 已提交
112
// NOTE: offset here including the header size
H
TD-166  
hzcheng 已提交
113 114 115 116
static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
  switch (type) {
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
H
TD-166  
hzcheng 已提交
117
      return POINTER_DRIFT(row, *(VarDataOffsetT *)POINTER_DRIFT(row, offset));
H
TD-166  
hzcheng 已提交
118 119
      break;
    default:
H
TD-166  
hzcheng 已提交
120
      return POINTER_DRIFT(row, offset);
H
TD-166  
hzcheng 已提交
121 122 123 124
      break;
  }
}

H
TD-34  
hzcheng 已提交
125 126
// ----------------- Data column structure
typedef struct SDataCol {
H
TD-166  
hzcheng 已提交
127 128 129
  int8_t          type;       // column type
  int16_t         colId;      // column ID
  int             bytes;      // column data bytes defined
H
TD-166  
hzcheng 已提交
130
  int             offset;     // data offset in a SDataRow (including the header size)
H
TD-166  
hzcheng 已提交
131 132 133 134
  int             spaceSize;  // Total space size for this column
  int             len;        // column data length
  VarDataOffsetT *dataOff;    // For binary and nchar data, the offset in the data column
  void *          pData;      // Actual data pointer
H
TD-34  
hzcheng 已提交
135 136
} SDataCol;

H
TD-166  
hzcheng 已提交
137 138 139
static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }

void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
H
TD-166  
hzcheng 已提交
140
void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints);
H
TD-166  
hzcheng 已提交
141 142 143
void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints);
void dataColSetOffset(SDataCol *pCol, int nEle);

H
TD-166  
hzcheng 已提交
144 145
bool isNEleNull(SDataCol *pCol, int nEle);
void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
H
TD-166  
hzcheng 已提交
146 147 148

// Get the data pointer from a column-wised data
static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
H
TD-166  
hzcheng 已提交
149 150 151 152 153 154 155 156 157
  switch (pCol->type) {
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      return POINTER_DRIFT(pCol->pData, pCol->dataOff[row]);
      break;

    default:
      return POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
      break;
H
TD-166  
hzcheng 已提交
158 159 160
  }
}

H
TD-166  
hzcheng 已提交
161
static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
H
TD-166  
hzcheng 已提交
162 163
  ASSERT(rows > 0);

H
TD-166  
hzcheng 已提交
164 165 166
  switch (pDataCol->type) {
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
H
TD-166  
hzcheng 已提交
167
      return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
H
TD-166  
hzcheng 已提交
168 169
      break;
    default:
H
TD-166  
hzcheng 已提交
170
      return TYPE_BYTES[pDataCol->type] * rows;
H
TD-166  
hzcheng 已提交
171 172 173 174
  }
}


H
TD-34  
hzcheng 已提交
175
typedef struct {
H
TD-34  
hzcheng 已提交
176 177 178
  int      maxRowSize;
  int      maxCols;    // max number of columns
  int      maxPoints;  // max number of points
H
TD-166  
hzcheng 已提交
179
  int      bufSize;
H
TD-166  
hzcheng 已提交
180

H
TD-34  
hzcheng 已提交
181
  int      numOfPoints;
H
TD-34  
hzcheng 已提交
182
  int      numOfCols;  // Total number of cols
H
TD-34  
hzcheng 已提交
183
  int      sversion;   // TODO: set sversion
H
TD-34  
hzcheng 已提交
184 185 186 187
  void *   buf;
  SDataCol cols[];
} SDataCols;

H
TD-34  
hzcheng 已提交
188
#define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
H
TD-166  
hzcheng 已提交
189
#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
H
TD-34  
hzcheng 已提交
190
#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
H
hzcheng 已提交
191
#define dataColsKeyLast(pCols) ((pCols->numOfPoints == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfPoints - 1))
H
TD-34  
hzcheng 已提交
192

H
TD-166  
hzcheng 已提交
193
SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
H
TD-34  
hzcheng 已提交
194
void       tdResetDataCols(SDataCols *pCols);
H
TD-34  
hzcheng 已提交
195
void       tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
H
TD-100  
hzcheng 已提交
196
SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
H
TD-34  
hzcheng 已提交
197
void       tdFreeDataCols(SDataCols *pCols);
H
TD-34  
hzcheng 已提交
198
void       tdAppendDataRowToDataCol(SDataRow row, SDataCols *pCols);
H
hjxilinx 已提交
199
void       tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!!
H
hzcheng 已提交
200
int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
H
TD-100  
hzcheng 已提交
201
void       tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, SDataCols *src2, int *iter2, int tRows);
H
more  
Hongze Cheng 已提交
202

H
more  
hzcheng 已提交
203 204 205 206
#ifdef __cplusplus
}
#endif

H
hzcheng 已提交
207
#endif  // _TD_DATA_FORMAT_H_