tdataformat.h 7.8 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
S
Shengliang Guan 已提交
15

16 17
#ifndef _TD_COMMON_DATA_FORMAT_H_
#define _TD_COMMON_DATA_FORMAT_H_
H
more  
Hongze Cheng 已提交
18

S
TD-4088  
Shengliang Guan 已提交
19
#include "os.h"
H
Hongze Cheng 已提交
20
#include "talgo.h"
H
Hongze Cheng 已提交
21
#include "tarray.h"
H
Hongze Cheng 已提交
22
#include "tencode.h"
H
Haojun Liao 已提交
23
#include "ttypes.h"
H
TD-166  
hzcheng 已提交
24
#include "tutil.h"
H
hzcheng 已提交
25

H
more  
hzcheng 已提交
26 27 28
#ifdef __cplusplus
extern "C" {
#endif
H
hzcheng 已提交
29

H
Hongze Cheng 已提交
30 31 32 33 34 35 36 37 38 39 40
typedef struct SBuffer  SBuffer;
typedef struct SSchema  SSchema;
typedef struct STColumn STColumn;
typedef struct STSchema STSchema;
typedef struct SValue   SValue;
typedef struct SColVal  SColVal;
typedef struct SRow     SRow;
typedef struct SRowIter SRowIter;
typedef struct STagVal  STagVal;
typedef struct STag     STag;
typedef struct SColData SColData;
H
Hongze Cheng 已提交
41

H
Hongze Cheng 已提交
42 43 44
#define HAS_NONE  ((uint8_t)0x1)
#define HAS_NULL  ((uint8_t)0x2)
#define HAS_VALUE ((uint8_t)0x4)
H
refact  
Hongze Cheng 已提交
45 46

// bitmap ================================
H
Hongze Cheng 已提交
47
const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0},
H
fix  
Hongze Cheng 已提交
48 49 50
                                       {0b00000000, 0b00000100, 0b00001000, 2},
                                       {0b00000000, 0b00010000, 0b00100000, 4},
                                       {0b00000000, 0b01000000, 0b10000000, 6}};
H
Hongze Cheng 已提交
51

H
Hongze Cheng 已提交
52
#define N1(n)             ((((uint8_t)1) << (n)) - 1)
H
Hongze Cheng 已提交
53 54
#define BIT1_SIZE(n)      ((((n)-1) >> 3) + 1)
#define BIT2_SIZE(n)      ((((n)-1) >> 2) + 1)
H
Hongze Cheng 已提交
55
#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7)))
H
more  
Hongze Cheng 已提交
56
#define GET_BIT1(p, i)    (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1))
H
Hongze Cheng 已提交
57
#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)])
H
more  
Hongze Cheng 已提交
58
#define GET_BIT2(p, i)    (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3))
H
Hongze Cheng 已提交
59

H
more  
Hongze Cheng 已提交
60 61 62 63 64 65 66 67 68 69 70
// SBuffer ================================
struct SBuffer {
  int64_t  nBuf;
  uint8_t *pBuf;
};

#define tBufferCreate() \
  (SBuffer) { .nBuf = 0, .pBuf = NULL }
void    tBufferDestroy(SBuffer *pBuffer);
int32_t tBufferInit(SBuffer *pBuffer, int64_t size);
int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData);
H
Hongze Cheng 已提交
71
int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData);
H
more  
Hongze Cheng 已提交
72

H
refact  
Hongze Cheng 已提交
73
// SColVal ================================
H
Hongze Cheng 已提交
74 75 76 77 78 79
#define CV_FLAG_VALUE ((int8_t)0x0)
#define CV_FLAG_NONE  ((int8_t)0x1)
#define CV_FLAG_NULL  ((int8_t)0x2)

#define COL_VAL_NONE(CID, TYPE)     ((SColVal){.cid = (CID), .type = (TYPE), .flag = CV_FLAG_NONE})
#define COL_VAL_NULL(CID, TYPE)     ((SColVal){.cid = (CID), .type = (TYPE), .flag = CV_FLAG_NULL})
H
Hongze Cheng 已提交
80
#define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)})
H
more  
Hongze Cheng 已提交
81

H
Hongze Cheng 已提交
82 83 84 85
#define COL_VAL_IS_NONE(CV)  ((CV)->flag == CV_FLAG_NONE)
#define COL_VAL_IS_NULL(CV)  ((CV)->flag == CV_FLAG_NULL)
#define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE)

H
Hongze Cheng 已提交
86
// SRow ================================
H
Hongze Cheng 已提交
87
int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow);
H
Hongze Cheng 已提交
88
void    tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
H
Hongze Cheng 已提交
89
void    tRowDestroy(SRow *pRow);
H
Hongze Cheng 已提交
90
int32_t tRowMergeSort(SArray *aRowP, STSchema *pTSchema, int8_t flag);
H
Hongze Cheng 已提交
91 92

// SRowIter ================================
H
Hongze Cheng 已提交
93 94 95
int32_t  tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);
void     tRowIterClose(SRowIter **ppIter);
SColVal *tRowIterNext(SRowIter *pIter);
H
Hongze Cheng 已提交
96
int32_t  tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SArray *aColData, int32_t nColData);
H
Hongze Cheng 已提交
97

H
refact  
Hongze Cheng 已提交
98
// STag ================================
H
Hongze Cheng 已提交
99
int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag);
H
Hongze Cheng 已提交
100
void    tTagFree(STag *pTag);
wmmhello's avatar
wmmhello 已提交
101 102
bool    tTagIsJson(const void *pTag);
bool    tTagIsJsonNull(void *tagVal);
C
Cary Xu 已提交
103
bool    tTagGet(const STag *pTag, STagVal *pTagVal);
H
more  
Hongze Cheng 已提交
104
char   *tTagValToData(const STagVal *pTagVal, bool isJson);
H
more  
Hongze Cheng 已提交
105
int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
H
Hongze Cheng 已提交
106
int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
C
Cary Xu 已提交
107
int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
wmmhello's avatar
wmmhello 已提交
108
void    tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid);
H
more  
Hongze Cheng 已提交
109
void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);  // TODO: remove
H
Hongze Cheng 已提交
110
int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf);
H
Hongze Cheng 已提交
111

H
refact  
Hongze Cheng 已提交
112
// SColData ================================
H
Hongze Cheng 已提交
113
void    tColDataDestroy(void *ph);
H
refact  
Hongze Cheng 已提交
114
void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
H
Hongze Cheng 已提交
115
void    tColDataClear(SColData *pColData);
H
refact  
Hongze Cheng 已提交
116
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
H
Hongze Cheng 已提交
117
void    tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
H
Haojun Liao 已提交
118
uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
H
refact  
Hongze Cheng 已提交
119
int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
H
Hongze Cheng 已提交
120
extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull);
H
refact  
Hongze Cheng 已提交
121

H
Hongze Cheng 已提交
122 123 124 125
// for stmt bind
int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind);
int32_t tColDataSortMerge(SColData *aColData);

H
refact  
Hongze Cheng 已提交
126
// STRUCT ================================
H
Hongze Cheng 已提交
127 128 129 130 131 132 133 134 135
struct STColumn {
  col_id_t colId;
  int8_t   type;
  int8_t   flags;
  int32_t  bytes;
  int32_t  offset;
};

struct STSchema {
H
Hongze Cheng 已提交
136 137 138 139 140
  int32_t  numOfCols;
  int32_t  version;
  int32_t  flen;
  int32_t  tlen;
  STColumn columns[];
H
Hongze Cheng 已提交
141 142
};

H
Hongze Cheng 已提交
143 144 145 146 147
struct SRow {
  uint8_t  flag;
  uint8_t  rsv;
  uint16_t sver;
  uint32_t len;
H
Hongze Cheng 已提交
148
  TSKEY    ts;
H
Hongze Cheng 已提交
149
  uint8_t  data[];
H
Hongze Cheng 已提交
150 151
};

H
Hongze Cheng 已提交
152 153
struct SValue {
  union {
H
Hongze Cheng 已提交
154
    int64_t val;
H
Hongze Cheng 已提交
155 156 157 158 159 160 161
    struct {
      uint32_t nData;
      uint8_t *pData;
    };
  };
};

H
Hongze Cheng 已提交
162
struct SColVal {
H
Hongze Cheng 已提交
163
  int16_t cid;
H
Hongze Cheng 已提交
164
  int8_t  type;
H
Hongze Cheng 已提交
165
  int8_t  flag;
H
Hongze Cheng 已提交
166
  SValue  value;
H
Hongze Cheng 已提交
167 168
};

H
refact  
Hongze Cheng 已提交
169 170 171 172 173 174 175 176 177 178 179 180
struct SColData {
  int16_t  cid;
  int8_t   type;
  int8_t   smaOn;
  int32_t  nVal;
  uint8_t  flag;
  uint8_t *pBitMap;
  int32_t *aOffset;
  int32_t  nData;
  uint8_t *pData;
};

wmmhello's avatar
wmmhello 已提交
181
#pragma pack(push, 1)
H
Hongze Cheng 已提交
182
struct STagVal {
H
Hongze Cheng 已提交
183
  //  char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta
H
Hongze Cheng 已提交
184 185 186 187
  union {
    int16_t cid;
    char   *pKey;
  };
H
Hongze Cheng 已提交
188 189
  int8_t type;
  union {
H
more  
Hongze Cheng 已提交
190
    int64_t i64;
H
Hongze Cheng 已提交
191 192 193 194 195
    struct {
      uint32_t nData;
      uint8_t *pData;
    };
  };
H
Hongze Cheng 已提交
196 197
};

H
more  
Hongze Cheng 已提交
198
#define TD_TAG_JSON  ((int8_t)0x40)  // distinguish JSON string and JSON value with the highest bit
wmmhello's avatar
wmmhello 已提交
199
#define TD_TAG_LARGE ((int8_t)0x20)
C
Cary Xu 已提交
200
struct STag {
C
Cary Xu 已提交
201
  int8_t  flags;
C
Cary Xu 已提交
202 203 204
  int16_t len;
  int16_t nTag;
  int32_t ver;
C
Cary Xu 已提交
205
  int8_t  idx[];
C
Cary Xu 已提交
206 207 208
};
#pragma pack(pop)

H
Hongze Cheng 已提交
209
#if 1  //================================================================================================================================================
C
Cary Xu 已提交
210 211
// Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
#define TD_SUPPORT_BITMAP
K
Kaili Xu 已提交
212

213 214 215 216 217
#define STR_TO_VARSTR(x, str)                     \
  do {                                            \
    VarDataLenT __len = (VarDataLenT)strlen(str); \
    *(VarDataLenT *)(x) = __len;                  \
    memcpy(varDataVal(x), (str), __len);          \
H
Hongze Cheng 已提交
218 219
  } while (0);

220 221
#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs)                         \
  do {                                                                    \
H
Hui Li 已提交
222
    char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \
223
    varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE));                      \
H
Hongze Cheng 已提交
224 225
  } while (0)

226 227 228 229
#define STR_WITH_SIZE_TO_VARSTR(x, str, _size)  \
  do {                                          \
    *(VarDataLenT *)(x) = (VarDataLenT)(_size); \
    memcpy(varDataVal(x), (str), (_size));      \
H
Hongze Cheng 已提交
230
  } while (0);
H
hjxilinx 已提交
231

H
Hongze Cheng 已提交
232
// STSchema ================================
H
Hongze Cheng 已提交
233
STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version);
H
Hongze Cheng 已提交
234
void      tDestroyTSchema(STSchema *pTSchema);
H
Hongze Cheng 已提交
235

H
Hongze Cheng 已提交
236
#endif
237

H
more  
hzcheng 已提交
238 239 240 241
#ifdef __cplusplus
}
#endif

H
Hongze Cheng 已提交
242
#endif /*_TD_COMMON_DATA_FORMAT_H_*/