tsqlfunction.h 10.7 KB
Newer Older
H
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef TDENGINE_TSQLFUNCTION_H
#define TDENGINE_TSQLFUNCTION_H

#ifdef __cplusplus
extern "C" {
#endif

23
#include "os.h"
H
hzcheng 已提交
24

25
#include "../../common/inc/name.h"
H
hzcheng 已提交
26
#include "taosdef.h"
27
#include "trpc.h"
28
#include "tvariant.h"
H
hzcheng 已提交
29 30 31 32 33 34 35 36 37 38 39 40 41

#define TSDB_FUNC_INVALID_ID  -1
#define TSDB_FUNC_COUNT        0
#define TSDB_FUNC_SUM          1
#define TSDB_FUNC_AVG          2
#define TSDB_FUNC_MIN          3
#define TSDB_FUNC_MAX          4
#define TSDB_FUNC_STDDEV       5
#define TSDB_FUNC_PERCT        6
#define TSDB_FUNC_APERCT       7
#define TSDB_FUNC_FIRST        8
#define TSDB_FUNC_LAST         9
#define TSDB_FUNC_LAST_ROW     10
S
slguan 已提交
42 43 44 45 46 47
#define TSDB_FUNC_TOP          11
#define TSDB_FUNC_BOTTOM       12
#define TSDB_FUNC_SPREAD       13
#define TSDB_FUNC_TWA          14
#define TSDB_FUNC_LEASTSQR     15

H
hzcheng 已提交
48 49
#define TSDB_FUNC_TS           16
#define TSDB_FUNC_TS_DUMMY     17
S
slguan 已提交
50 51
#define TSDB_FUNC_TAG_DUMMY    18
#define TSDB_FUNC_TS_COMP      19
H
hzcheng 已提交
52

S
slguan 已提交
53 54
#define TSDB_FUNC_TAG          20
#define TSDB_FUNC_PRJ          21
H
hzcheng 已提交
55

S
slguan 已提交
56 57 58
#define TSDB_FUNC_TAGPRJ       22
#define TSDB_FUNC_ARITHM       23
#define TSDB_FUNC_DIFF         24
H
hzcheng 已提交
59

S
slguan 已提交
60 61 62
#define TSDB_FUNC_FIRST_DST    25
#define TSDB_FUNC_LAST_DST     26
#define TSDB_FUNC_INTERP       27
H
hzcheng 已提交
63

L
lihui 已提交
64 65 66 67 68 69 70
#define TSDB_FUNC_RATE         28
#define TSDB_FUNC_IRATE        29
#define TSDB_FUNC_SUM_RATE     30
#define TSDB_FUNC_SUM_IRATE    31
#define TSDB_FUNC_AVG_RATE     32
#define TSDB_FUNC_AVG_IRATE    33

S
slguan 已提交
71 72 73 74 75 76 77
#define TSDB_FUNCSTATE_SO           0x1U    // single output
#define TSDB_FUNCSTATE_MO           0x2U    // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
#define TSDB_FUNCSTATE_STREAM       0x4U    // function avail for stream
#define TSDB_FUNCSTATE_METRIC       0x8U    // function avail for metric
#define TSDB_FUNCSTATE_OF           0x10U   // outer forward
#define TSDB_FUNCSTATE_NEED_TS      0x20U   // timestamp is required during query processing
#define TSDB_FUNCSTATE_SELECTIVITY  0x40U   // selectivity functions, can exists along with tag columns
H
hzcheng 已提交
78 79 80 81

#define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF
#define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_METRIC | TSDB_FUNCSTATE_OF

S
slguan 已提交
82 83 84 85
#define TSDB_PATTERN_MATCH            0
#define TSDB_PATTERN_NOMATCH          1
#define TSDB_PATTERN_NOWILDCARDMATCH  2
#define TSDB_PATTERN_STRING_MAX_LEN   20
H
hzcheng 已提交
86

S
slguan 已提交
87
#define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16
H
hzcheng 已提交
88 89 90 91 92 93
#define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50

#define PATTERN_COMPARE_INFO_INITIALIZER \
  { '%', '_' }

#define DATA_SET_FLAG ','  // to denote the output area has data, not null value
S
slguan 已提交
94 95 96 97
#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG)

#define QUERY_COND_REL_PREFIX_IN "IN|"
#define QUERY_COND_REL_PREFIX_LIKE "LIKE|"
H
hzcheng 已提交
98

S
slguan 已提交
99 100 101 102
#define QUERY_COND_REL_PREFIX_IN_LEN 3
#define QUERY_COND_REL_PREFIX_LIKE_LEN 5

#define QUERY_ASC_FORWARD_STEP 1
H
hzcheng 已提交
103
#define QUERY_DESC_FORWARD_STEP -1
S
slguan 已提交
104

H
hzcheng 已提交
105 106
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSQL_SO_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)

S
slguan 已提交
107 108 109
#define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000
#define TOP_BOTTOM_QUERY_LIMIT 100

H
hzcheng 已提交
110
enum {
H
hjxilinx 已提交
111 112 113 114 115
  MASTER_SCAN           = 0x0u,
  SUPPLEMENTARY_SCAN    = 0x1u,
  REPEAT_SCAN           = 0x2u,  //repeat scan belongs to the master scan
  FIRST_STAGE_MERGE     = 0x10u,
  SECONDARY_STAGE_MERGE = 0x20u,
H
hzcheng 已提交
116 117
};

S
slguan 已提交
118 119 120 121 122 123
#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
#define QUERY_IS_JOIN_QUERY(type) (((type)&TSDB_QUERY_TYPE_JOIN_QUERY) != 0)
#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0)
#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0)

typedef struct SArithmeticSupport {
H
hzcheng 已提交
124 125 126 127 128 129 130 131 132
  SSqlFunctionExpr *pExpr;
  int32_t           elemSize[TSDB_MAX_COLUMNS];
  int32_t           numOfCols;
  int32_t           offset;
  char *            data[TSDB_MAX_COLUMNS];
} SArithmeticSupport;

typedef struct SQLPreAggVal {
  bool    isSet;
133 134
  int32_t size;
  SDataStatis statis;
H
hzcheng 已提交
135 136
} SQLPreAggVal;

S
slguan 已提交
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
typedef struct SInterpInfoDetail {
  TSKEY  ts;  // interp specified timestamp
  int8_t hasResult;
  int8_t type;
  int8_t primaryCol;
} SInterpInfoDetail;

typedef struct SInterpInfo { SInterpInfoDetail *pInterpDetail; } SInterpInfo;

typedef struct SResultInfo {
  int8_t  hasResult;       // result generated, not NULL value
  bool    initialized;     // output buffer has been initialized
  bool    complete;        // query has completed
  bool    superTableQ;     // is super table query
  int32_t numOfRes;        // num of output result in current buffer
  int32_t bufLen;          // buffer size
  void *  interResultBuf;  // output result buffer
} SResultInfo;

struct SQLFunctionCtx;

/**
 * for selectivity query, the corresponding tag value is assigned if the data is qualified
 */
typedef struct SExtTagsInfo {
  int16_t                 tagsLen;      // keep the tags data for top/bottom query result
  int16_t                 numOfTagCols;
  struct SQLFunctionCtx **pTagCtxList;
} SExtTagsInfo;

// sql function runtime context
H
hzcheng 已提交
168
typedef struct SQLFunctionCtx {
169 170 171
  int32_t  startOffset;
  int32_t  size;      // number of rows
  uint32_t order;     // asc|desc
H
hjxilinx 已提交
172
  uint32_t scanFlag;  // TODO merge with currentStage
H
hzcheng 已提交
173 174 175 176

  int16_t inputType;
  int16_t inputBytes;

S
slguan 已提交
177 178 179 180 181 182 183 184 185
  int16_t  outputType;
  int16_t  outputBytes;  // size of results, determined by function and input column data type
  bool     hasNull;      // null value exist in current block
  int16_t  functionId;   // function id
  int32_t  blockStatus;  // Indicate if data is loaded, it is first/last/internal block. Only for file blocks
  void *   aInputElemBuf;
  char *   aOutputBuf;            // final result output buffer, point to sdata->data
  uint8_t  currentStage;          // record current running step, default: 0
  int64_t  nStartQueryTimestamp;  // timestamp range of current query when function is executed on a specific data block
H
hzcheng 已提交
186
  int32_t  numOfParams;
S
slguan 已提交
187 188 189
  tVariant param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param */
  int64_t *ptsList;       // corresponding timestamp array list
  void *   ptsOutputBuf;  // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
H
hzcheng 已提交
190
  SQLPreAggVal preAggVals;
S
slguan 已提交
191 192 193 194
  tVariant     tag;
  SResultInfo *resultInfo;

  SExtTagsInfo tagInfo;
H
hzcheng 已提交
195 196 197 198 199
} SQLFunctionCtx;

typedef struct SQLAggFuncElem {
  char aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH];

S
slguan 已提交
200 201
  uint8_t  nAggIdx;       // index of function in aAggs
  int8_t   stableFuncId;  // transfer function for super table query
H
hzcheng 已提交
202 203
  uint16_t nStatus;

S
slguan 已提交
204
  bool (*init)(SQLFunctionCtx *pCtx);  // setup the execute environment
H
hzcheng 已提交
205

S
slguan 已提交
206 207
  void (*xFunction)(SQLFunctionCtx *pCtx);                     // blocks version function
  void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position);  // single-row function version
H
hzcheng 已提交
208

S
slguan 已提交
209 210
  // some sql function require scan data twice or more, e.g.,stddev
  void (*xNextStep)(SQLFunctionCtx *pCtx);
H
hzcheng 已提交
211 212 213

  /*
   * finalizer must be called after all xFunction has been executed to
S
slguan 已提交
214
   * generated final result. Otherwise, the value in aOutputBuf is a intern result.
H
hzcheng 已提交
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
   */
  void (*xFinalize)(SQLFunctionCtx *pCtx);

  void (*distMergeFunc)(SQLFunctionCtx *pCtx);

  void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx);

  int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, int32_t blockStatus);
} SQLAggFuncElem;

typedef struct SPatternCompareInfo {
  char matchAll;  // symbol for match all wildcard, default: '%'
  char matchOne;  // symbol for match one wildcard, default: '_'
} SPatternCompareInfo;

S
slguan 已提交
230 231 232 233
#define GET_RES_INFO(ctx) ((ctx)->resultInfo)

int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
                          int16_t *len, int16_t *interResBytes, int16_t extLength, bool isSuperTable);
H
hzcheng 已提交
234 235 236

int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo);

S
slguan 已提交
237
int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo);
H
hzcheng 已提交
238

S
slguan 已提交
239 240 241 242
#define IS_STREAM_QUERY_VALID(x)  (((x)&TSDB_FUNCSTATE_STREAM) != 0)
#define IS_MULTIOUTPUT(x)         (((x)&TSDB_FUNCSTATE_MO) != 0)
#define IS_SINGLEOUTPUT(x)        (((x)&TSDB_FUNCSTATE_SO) != 0)
#define IS_OUTER_FORWARD(x)       (((x)&TSDB_FUNCSTATE_OF) != 0)
H
hzcheng 已提交
243 244 245

/*
 * the status of one block, used in metric query. all blocks are mixed together,
S
slguan 已提交
246
 * we need the status to decide if one block is a first/end/inter block of one meter
H
hzcheng 已提交
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
 */
enum {
  BLK_FILE_BLOCK = 0x1,
  BLK_BLOCK_LOADED = 0x2,
  BLK_CACHE_BLOCK = 0x4,  // in case of cache block, block must be loaded
};

/* determine the real data need to calculated the result */
enum {
  BLK_DATA_NO_NEEDED = 0x0,
  BLK_DATA_FILEDS_NEEDED = 0x1,
  BLK_DATA_ALL_NEEDED = 0x3,
};

#define IS_FILE_BLOCK(x) (((x)&BLK_FILE_BLOCK) != 0)

#define SET_FILE_BLOCK_FLAG(x) \
  do {                         \
    (x) &= (~BLK_CACHE_BLOCK); \
    (x) |= BLK_FILE_BLOCK;     \
  } while (0);

#define SET_CACHE_BLOCK_FLAG(x) ((x) = BLK_CACHE_BLOCK | BLK_BLOCK_LOADED);

#define SET_DATA_BLOCK_NOT_LOADED(x) ((x) &= (~BLK_BLOCK_LOADED));

#define SET_DATA_BLOCK_LOADED(x) ((x) |= BLK_BLOCK_LOADED);
#define IS_DATA_BLOCK_LOADED(x) (((x)&BLK_BLOCK_LOADED) != 0)

S
slguan 已提交
276 277 278 279 280 281
typedef struct STwaInfo {
  TSKEY   lastKey;
  int8_t  hasResult;  // flag to denote has value
  int16_t type;       // source data type
  TSKEY   SKey;
  TSKEY   EKey;
H
hzcheng 已提交
282 283 284 285 286 287 288 289 290 291

  union {
    double  dOutput;
    int64_t iOutput;
  };

  union {
    double  dLastValue;
    int64_t iLastValue;
  };
S
slguan 已提交
292
} STwaInfo;
H
hzcheng 已提交
293 294

/* global sql function array */
L
lihui 已提交
295
extern struct SQLAggFuncElem aAggs[];
H
hzcheng 已提交
296 297

/* compatible check array list */
L
lihui 已提交
298
extern int32_t funcCompatDefList[];
H
hzcheng 已提交
299 300

void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, int32_t type, int64_t *min, int64_t *max,
S
slguan 已提交
301
                   int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull);
H
hzcheng 已提交
302 303 304

bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval);

H
hjxilinx 已提交
305 306 307
bool stableQueryFunctChanged(int32_t funcId);


S
slguan 已提交
308 309 310 311
void resetResultInfo(SResultInfo *pResInfo);
void initResultInfo(SResultInfo *pResInfo);
void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable);

H
hzcheng 已提交
312 313 314 315 316
#ifdef __cplusplus
}
#endif

#endif  // TDENGINE_TSQLFUNCTION_H