Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 8d413f09 编写于 作者: D dapan1121
浏览文件
操作

feature/qnode

上级 6d04b954
显示空白变更内容
内联 并排
Showing with 737 addition and 751 deletion
source/libs/scalar/inc/tsclfunc.h → include/libs/scalar/filter.h
浏览文件 @ 8d413f09
......@@ -12,34 +12,25 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCALARFUNCTION_H
#define TDENGINE_TSCALARFUNCTION_H
#ifndef TDENGINE_SCALAR_H
#define TDENGINE_SCALAR_H
#ifdef __cplusplus
extern "C" {
#endif
#include "function.h"
#include "scalar.h"
#include "nodes.h"
typedef struct SScalarFunctionSupport {
struct SExprInfo *pExprInfo;
int32_t numOfCols;
SColumnInfo *colList;
void *exprList; // client side used
int32_t offset;
char** data;
} SScalarFunctionSupport;
typedef struct SFilterInfo SFilterInfo;
extern struct SScalarFunctionInfo scalarFunc[8];
int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput,
void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t));
int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSCALARFUNCTION_H
#endif // TDENGINE_SCALAR_H
source/libs/scalar/inc/filter.h → source/libs/scalar/inc/filterInt.h
浏览文件 @ 8d413f09
......@@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QFILTER_H
#define TDENGINE_QFILTER_H
#ifndef TDENGINE_FILTER_INT_H
#define TDENGINE_FILTER_INT_H
#ifdef __cplusplus
extern "C" {
......@@ -85,6 +85,12 @@ enum {
RANGE_TYPE_MR_CTX = 3,
};
enum {
FI_ACTION_NO_NEED = 1,
FI_ACTION_CONTINUE,
FI_ACTION_STOP,
};
typedef struct OptrStr {
uint16_t optr;
char *str;
......@@ -97,17 +103,11 @@ typedef struct SFilterRange {
char eflag;
} SFilterRange;
typedef struct SFilterColRange {
uint16_t idx; //column field idx
bool isNull;
bool notNull;
bool isRange;
SFilterRange ra;
} SFilterColRange;
typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t);
typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **);
typedef struct SFilterRangeCompare {
int64_t s;
......@@ -155,37 +155,39 @@ typedef struct SFilterField {
} SFilterField;
typedef struct SFilterFields {
uint16_t size;
uint16_t num;
uint32_t size;
uint32_t num;
SFilterField *fields;
} SFilterFields;
typedef struct SFilterFieldId {
uint16_t type;
uint16_t idx;
uint32_t idx;
} SFilterFieldId;
typedef struct SFilterGroup {
uint16_t unitSize;
uint16_t unitNum;
uint16_t *unitIdxs;
uint32_t unitSize;
uint32_t unitNum;
uint32_t *unitIdxs;
uint8_t *unitFlags; // !unit result
} SFilterGroup;
typedef struct SFilterColInfo {
uint8_t type;
int32_t dataType;
uint8_t optr; // for equal operation in the relation of RELATION_IN
int64_t value; // for equal operation in the relation of RELATION_IN
void *info;
} SFilterColInfo;
typedef struct SFilterGroupCtx {
uint16_t colNum;
uint16_t *colIdx;
uint32_t colNum;
uint32_t *colIdx;
SFilterColInfo *colInfo;
} SFilterGroupCtx;
typedef struct SFilterColCtx {
uint16_t colIdx;
uint32_t colIdx;
void* ctx;
} SFilterColCtx;
......@@ -219,16 +221,31 @@ typedef struct SFilterPCtx {
SHashObj *unitHash;
} SFilterPCtx;
typedef struct SFltTreeStat {
int32_t code;
} SFltTreeStat;
typedef struct SFltScalarCtx {
SNode *node;
} SFltScalarCtx;
typedef struct SFltBuildGroupCtx {
SFilterInfo *info;
SArray *group;
int32_t code;
} SFltBuildGroupCtx;
typedef struct SFilterInfo {
bool scalarMode;
SFltScalarCtx sclCtx;
uint32_t options;
uint32_t status;
uint16_t unitSize;
uint16_t unitNum;
uint16_t groupNum;
uint16_t colRangeNum;
uint32_t unitSize;
uint32_t unitNum;
uint32_t groupNum;
uint32_t colRangeNum;
SFilterFields fields[FLD_TYPE_MAX];
SFilterGroup *groups;
uint16_t *cgroups;
SFilterUnit *units;
SFilterComUnit *cunits;
uint8_t *unitRes; // result
......@@ -236,16 +253,15 @@ typedef struct SFilterInfo {
SFilterRangeCtx **colRange;
filter_exec_func func;
uint8_t blkFlag;
uint16_t blkGroupNum;
uint16_t *blkUnits;
uint32_t blkGroupNum;
uint32_t *blkUnits;
int8_t *blkUnitRes;
void *pTable;
SFilterPCtx pctx;
} SFilterInfo;
#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t))
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR)
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON)
#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
......@@ -261,7 +277,7 @@ typedef struct SFilterInfo {
#define FILTER_CLR_FLAG(st, f) st &= (~f)
#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src)
#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0)
#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint32_t *)(_t + 1) = idx1; *(uint32_t *)(_t + 3) = idx2; } while (0)
#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE))))
#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0)
......@@ -271,15 +287,20 @@ typedef struct SFilterInfo {
#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
#define CHK_RETV(c) do { if (c) { return; } } while (0)
#define CHK_RET(c, r) do { if (c) { return r; } } while (0)
#define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0)
#define fltFatal(...) qFatal(__VA_ARGS__)
#define fltError(...) qError(__VA_ARGS__)
#define fltWarn(...) qWarn(__VA_ARGS__)
#define fltInfo(...) qInfo(__VA_ARGS__)
#define fltDebug(...) qDebug(__VA_ARGS__)
#define fltTrace(...) qTrace(__VA_ARGS__)
#define FLT_CHK_JMP(c) do { if (c) { goto _return; } } while (0)
#define FLT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define FLT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define FLT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
......@@ -290,6 +311,7 @@ typedef struct SFilterInfo {
#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
......@@ -302,6 +324,7 @@ typedef struct SFilterInfo {
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
......@@ -316,7 +339,7 @@ typedef struct SFilterInfo {
#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0)
#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0)
#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0)
#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0)
#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0)
......@@ -325,9 +348,10 @@ typedef struct SFilterInfo {
#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
#if 0
extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock);
extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp);
extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
......@@ -344,4 +368,4 @@ extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
}
#endif
#endif // TDENGINE_QFILTER_H
#endif // TDENGINE_FILTER_INT_H
\ No newline at end of file
source/libs/scalar/inc/tfilter.h 已删除 100644 → 0
浏览文件 @ 6d04b954
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QFILTER_H
#define TDENGINE_QFILTER_H
#ifdef __cplusplus
extern "C" {
#endif
#include "thash.h"
#include "tname.h"
#include "common.h"
#include "scalar.h"
#include "querynodes.h"
#define FILTER_DEFAULT_GROUP_SIZE 4
#define FILTER_DEFAULT_UNIT_SIZE 4
#define FILTER_DEFAULT_FIELD_SIZE 4
#define FILTER_DEFAULT_VALUE_SIZE 4
#define FILTER_DEFAULT_GROUP_UNIT_SIZE 2
#define FILTER_DUMMY_EMPTY_OPTR 127
#define MAX_NUM_STR_SIZE 40
#define FILTER_RM_UNIT_MIN_ROWS 100
enum {
FLD_TYPE_COLUMN = 1,
FLD_TYPE_VALUE = 2,
FLD_TYPE_MAX = 3,
FLD_DESC_NO_FREE = 4,
FLD_DATA_NO_FREE = 8,
FLD_DATA_IS_HASH = 16,
};
enum {
MR_ST_START = 1,
MR_ST_FIN = 2,
MR_ST_ALL = 4,
MR_ST_EMPTY = 8,
};
enum {
RANGE_FLG_EXCLUDE = 1,
RANGE_FLG_INCLUDE = 2,
RANGE_FLG_NULL = 4,
};
enum {
FI_OPTION_NO_REWRITE = 1,
FI_OPTION_TIMESTAMP = 2,
FI_OPTION_NEED_UNIQE = 4,
};
enum {
FI_STATUS_ALL = 1,
FI_STATUS_EMPTY = 2,
FI_STATUS_REWRITE = 4,
FI_STATUS_CLONED = 8,
};
enum {
FI_STATUS_BLK_ALL = 1,
FI_STATUS_BLK_EMPTY = 2,
FI_STATUS_BLK_ACTIVE = 4,
};
enum {
RANGE_TYPE_UNIT = 1,
RANGE_TYPE_VAR_HASH = 2,
RANGE_TYPE_MR_CTX = 3,
};
typedef struct OptrStr {
uint16_t optr;
char *str;
} OptrStr;
typedef struct SFilterRange {
int64_t s;
int64_t e;
char sflag;
char eflag;
} SFilterRange;
typedef struct SFilterColRange {
uint16_t idx; //column field idx
bool isNull;
bool notNull;
bool isRange;
SFilterRange ra;
} SFilterColRange;
typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t);
typedef struct SFilterRangeCompare {
int64_t s;
int64_t e;
rangeCompFunc func;
} SFilterRangeCompare;
typedef struct SFilterRangeNode {
struct SFilterRangeNode* prev;
struct SFilterRangeNode* next;
union {
SFilterRange ra;
SFilterRangeCompare rc;
};
} SFilterRangeNode;
typedef struct SFilterRangeCtx {
int32_t type;
int32_t options;
int8_t status;
bool isnull;
bool notnull;
bool isrange;
int16_t colId;
__compar_fn_t pCompareFunc;
SFilterRangeNode *rf; //freed
SFilterRangeNode *rs;
} SFilterRangeCtx ;
typedef struct SFilterVarCtx {
int32_t type;
int32_t options;
int8_t status;
bool isnull;
bool notnull;
bool isrange;
SHashObj *wild;
SHashObj *value;
} SFilterVarCtx;
typedef struct SFilterField {
uint16_t flag;
void* desc;
void* data;
} SFilterField;
typedef struct SFilterFields {
uint16_t size;
uint16_t num;
SFilterField *fields;
} SFilterFields;
typedef struct SFilterFieldId {
uint16_t type;
uint16_t idx;
} SFilterFieldId;
typedef struct SFilterGroup {
uint16_t unitSize;
uint16_t unitNum;
uint16_t *unitIdxs;
uint8_t *unitFlags; // !unit result
} SFilterGroup;
typedef struct SFilterColInfo {
uint8_t type;
int32_t dataType;
void *info;
} SFilterColInfo;
typedef struct SFilterGroupCtx {
uint16_t colNum;
uint16_t *colIdx;
SFilterColInfo *colInfo;
} SFilterGroupCtx;
typedef struct SFilterColCtx {
uint16_t colIdx;
void* ctx;
} SFilterColCtx;
typedef struct SFilterCompare {
uint8_t type;
uint8_t optr;
uint8_t optr2;
} SFilterCompare;
typedef struct SFilterUnit {
SFilterCompare compare;
SFilterFieldId left;
SFilterFieldId right;
SFilterFieldId right2;
} SFilterUnit;
typedef struct SFilterComUnit {
void *colData;
void *valData;
void *valData2;
uint16_t colId;
uint16_t dataSize;
uint8_t dataType;
uint8_t optr;
int8_t func;
int8_t rfunc;
} SFilterComUnit;
typedef struct SFilterPCtx {
SHashObj *valHash;
SHashObj *unitHash;
} SFilterPCtx;
typedef struct SFilterInfo {
uint32_t options;
uint32_t status;
uint16_t unitSize;
uint16_t unitNum;
uint16_t groupNum;
uint16_t colRangeNum;
SFilterFields fields[FLD_TYPE_MAX];
SFilterGroup *groups;
uint16_t *cgroups;
SFilterUnit *units;
SFilterComUnit *cunits;
uint8_t *unitRes; // result
uint8_t *unitFlags; // got result
SFilterRangeCtx **colRange;
filter_exec_func func;
uint8_t blkFlag;
uint16_t blkGroupNum;
uint16_t *blkUnits;
int8_t *blkUnitRes;
SFilterPCtx pctx;
} SFilterInfo;
#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t))
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR)
#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0)
#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0)
#define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true)
#define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true)))
#define FILTER_GET_FLAG(st, f) (st & f)
#define FILTER_SET_FLAG(st, f) st |= (f)
#define FILTER_CLR_FLAG(st, f) st &= (~f)
#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src)
#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0)
#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE))))
#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0)
#define RESET_RANGE(ctx, r) do { (r)->next = (ctx)->rf; (ctx)->rf = r; } while (0)
#define FREE_RANGE(ctx, r) do { if ((r)->prev) { (r)->prev->next = (r)->next; } else { (ctx)->rs = (r)->next;} if ((r)->next) { (r)->next->prev = (r)->prev; } RESET_RANGE(ctx, r); } while (0)
#define FREE_FROM_RANGE(ctx, r) do { SFilterRangeNode *_r = r; if ((_r)->prev) { (_r)->prev->next = NULL; } else { (ctx)->rs = NULL;} while (_r) {SFilterRangeNode *n = (_r)->next; RESET_RANGE(ctx, _r); _r = n; } } while (0)
#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
#define CHK_RETV(c) do { if (c) { return; } } while (0)
#define CHK_RET(c, r) do { if (c) { return r; } } while (0)
#define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0)
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes)
#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc))
#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
#define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left)
#define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right)
#define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2)
#define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type)
#define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri)
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
#define FILTER_UNIT_CLR_F(i) memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags))
#define FILTER_UNIT_SET_F(i, idx) (i)->unitFlags[idx] = 1
#define FILTER_UNIT_GET_F(i, idx) ((i)->unitFlags[idx])
#define FILTER_UNIT_GET_R(i, idx) ((i)->unitRes[idx])
#define FILTER_UNIT_SET_R(i, idx, v) (i)->unitRes[idx] = (v)
#define FILTER_PUSH_UNIT(colInfo, u) do { (colInfo).type = RANGE_TYPE_UNIT; (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u);taosArrayPush((SArray *)((colInfo).info), &u);} while (0)
#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0)
#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0)
#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0)
#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0)
#define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL)
#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
#if 0
extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
#else
//REMOVE THESE!!!!!!!!!!!!!!!!!!!!
#include "function.h"
#endif
extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QFILTER_H
source/libs/scalar/src/filter.c
浏览文件 @ 8d413f09
......@@ -17,7 +17,7 @@
#include "thash.h"
//#include "queryLog.h"
#include "tcompare.h"
#include "filter.h"
#include "filterInt.h"
OptrStr gOptrStr[] = {
{TSDB_RELATION_INVALID, "invalid"},
......@@ -28,21 +28,22 @@ OptrStr gOptrStr[] = {
{TSDB_RELATION_GREATER_EQUAL, ">="},
{TSDB_RELATION_NOT_EQUAL, "!="},
{TSDB_RELATION_LIKE, "like"},
{TSDB_RELATION_MATCH, "match"},
{TSDB_RELATION_MATCH, "nmatch"},
{TSDB_RELATION_ISNULL, "is null"},
{TSDB_RELATION_NOTNULL, "not null"},
{TSDB_RELATION_IN, "in"},
{TSDB_RELATION_AND, "and"},
{TSDB_RELATION_OR, "or"},
{TSDB_RELATION_NOT, "not"}
{TSDB_RELATION_NOT, "not"},
{TSDB_RELATION_MATCH, "match"},
{TSDB_RELATION_NMATCH, "nmatch"},
{TSDB_RELATION_CONTAINS, "contains"},
};
static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) {
const SSchema *sch1 = desc1;
const SSchema *sch2 = desc2;
return sch1->colId != sch2->colId;
return !(strcmp(sch1->name, sch2->name) == 0 && sch1->colId == sch2->colId);
}
static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) {
......@@ -60,16 +61,24 @@ filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = {
};
bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(maxv, minr) >= 0;
int32_t result = cfunc(maxv, minr);
if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result >= 0;
}
bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(maxv, minr) > 0;
int32_t result = cfunc(maxv, minr);
if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result > 0;
}
bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(minv, maxr) <= 0;
int32_t result = cfunc(minv, maxr);
if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result <= 0;
}
bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(minv, maxr) < 0;
int32_t result = cfunc(minv, maxr);
if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result < 0;
}
bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0;
......@@ -291,7 +300,7 @@ int32_t filterInitUnitsFields(SFilterInfo *info) {
info->fields[FLD_TYPE_COLUMN].num = 0;
info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE;
info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE);
info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField));
info->fields[FLD_TYPE_VALUE].num = 0;
info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE;
info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField));
......@@ -318,7 +327,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt
void* filterInitRangeCtx(int32_t type, int32_t options) {
if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
//qError("not supported range type:%d", type);
qError("not supported range type:%d", type);
return NULL;
}
......@@ -685,7 +694,7 @@ int32_t filterGetRangeRes(void* h, SFilterRange *ra) {
}
if (num == 0) {
//qError("no range result");
qError("no range result");
return TSDB_CODE_QRY_APP_ERROR;
}
......@@ -775,8 +784,15 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) {
int32_t leftSize = (int32_t)taosArrayGetSize(left);
int32_t rightSize = (int32_t)taosArrayGetSize(right);
// CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group");
// CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group");
if (taosArrayGetSize(left) <= 0) {
fltDebug("empty group");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (taosArrayGetSize(right) <= 0) {
fltDebug("empty group");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
for (int32_t l = 0; l < leftSize; ++l) {
SFilterGroup *gp1 = taosArrayGet(left, l);
......@@ -793,7 +809,7 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) {
}
int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) {
for (uint16_t i = 0; i < fields->num; ++i) {
for (uint32_t i = 0; i < fields->num; ++i) {
if (0 == gDescCompare[type](fields->fields[i].desc, v)) {
return i;
}
......@@ -806,7 +822,7 @@ int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) {
int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) {
if (type == FLD_TYPE_VALUE) {
if (info->pctx.valHash == false) {
//qError("value hash is empty");
qError("value hash is empty");
return -1;
}
......@@ -819,10 +835,11 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d
return -1;
}
// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0
// Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better.
int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) {
int32_t idx = -1;
uint16_t *num;
uint32_t *num;
num = &info->fields[type].num;
......@@ -885,8 +902,14 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte
int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) {
// CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node");
// CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR);
if (node == NULL) {
fltError("empty node");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE) {
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
int32_t type;
void *v;
......@@ -906,7 +929,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI
return TSDB_CODE_SUCCESS;
}
int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) {
int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) {
if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) {
if (info->pctx.unitHash == NULL) {
info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false);
......@@ -915,14 +938,14 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1);
void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v));
if (hu) {
*uidx = *(uint16_t *)hu;
*uidx = *(uint32_t *)hu;
return TSDB_CODE_SUCCESS;
}
}
}
if (info->unitNum >= info->unitSize) {
uint16_t psize = info->unitSize;
uint32_t psize = info->unitSize;
info->unitSize += FILTER_DEFAULT_UNIT_SIZE;
info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit));
memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE);
......@@ -940,7 +963,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u);
assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE));
} else {
assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR);
if(optr != TSDB_RELATION_ISNULL && optr != TSDB_RELATION_NOTNULL && optr != FILTER_DUMMY_EMPTY_OPTR){
return -1;
}
}
SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u);
......@@ -963,7 +988,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) {
int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) {
if (group->unitNum >= group->unitSize) {
group->unitSize += FILTER_DEFAULT_UNIT_SIZE;
group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs));
......@@ -974,12 +999,14 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) {
return TSDB_CODE_SUCCESS;
}
int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) {
int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType, bool tolower) {
SBufferReader br = tbufInitReader(buf, len, false);
uint32_t sType = tbufReadUint32(&br);
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false);
int32_t code = 0;
taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType));
int dummy = -1;
SVariant tmpVar = {0};
size_t t = 0;
......@@ -1065,47 +1092,53 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
switch (tType) {
case TSDB_DATA_TYPE_BOOL:
if (sType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(sType)) {
goto _return;
}
if (tmpVar.i64 > 1 ||tmpVar.i64 < 0) {
goto _return;
}
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: {
// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
// if (converted) {
// taosVariantDestroy(&tmpVar);
// memset(&tmpVar, 0, sizeof(tmpVar));
// continue;
// }
//
// goto _return;
// }
if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
if (converted) {
tVariantDestroy(&tmpVar);
memset(&tmpVar, 0, sizeof(tmpVar));
continue;
}
goto _return;
}
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: {
// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
// if (converted) {
// taosVariantDestroy(&tmpVar);
// memset(&tmpVar, 0, sizeof(tmpVar));
// continue;
// }
//
// goto _return;
// }
if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
if (converted) {
tVariantDestroy(&tmpVar);
memset(&tmpVar, 0, sizeof(tmpVar));
continue;
}
goto _return;
}
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_INT: {
// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
// if (converted) {
// taosVariantDestroy(&tmpVar);
// memset(&tmpVar, 0, sizeof(tmpVar));
// continue;
// }
//
// goto _return;
// }
if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
if (converted) {
tVariantDestroy(&tmpVar);
memset(&tmpVar, 0, sizeof(tmpVar));
continue;
}
goto _return;
}
pvar = &val;
t = sizeof(val);
break;
......@@ -1129,15 +1162,15 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
break;
}
case TSDB_DATA_TYPE_FLOAT: {
// if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
// if (converted) {
// taosVariantDestroy(&tmpVar);
// memset(&tmpVar, 0, sizeof(tmpVar));
// continue;
// }
// goto _return;
// }
if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
if (converted) {
tVariantDestroy(&tmpVar);
memset(&tmpVar, 0, sizeof(tmpVar));
continue;
}
goto _return;
}
pvar = &val;
t = sizeof(val);
break;
......@@ -1148,6 +1181,10 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
}
t = varDataLen(tmp);
pvar = varDataVal(tmp);
if (tolower) {
strntolower_s(pvar, pvar, (int32_t)t);
}
break;
}
case TSDB_DATA_TYPE_NCHAR: {
......@@ -1178,22 +1215,51 @@ _return:
return code;
}
int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) {
static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pLeft) {
if((*pLeft)->nodeType == TSQL_NODE_EXPR && (*pLeft)->_node.optr == TSDB_RELATION_ARROW){ // json tag -> operation
assert(info->pTable != NULL);
SSchema* schema = (*pLeft)->_node.pLeft->pSchema;
if((*pLeft)->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
jsonKeyMd5((*pLeft)->_node.pRight->pVal->pz, (*pLeft)->_node.pRight->pVal->nLen, keyMd5);
memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
(*pLeft) = (*pLeft)->_node.pLeft; // -> operation use left as input
}else if(((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) &&
(tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL)){
SSchema* schema = (*pLeft)->pSchema;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
uint32_t nullData = TSDB_DATA_JSON_NULL;
jsonKeyMd5(&nullData, INT_BYTES, keyMd5);
memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
}else if(tree->_node.optr == TSDB_RELATION_CONTAINS){
SSchema* schema = (*pLeft)->pSchema;
if(tree->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
jsonKeyMd5(tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen, keyMd5);
memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
}
return TSDB_CODE_SUCCESS;
}
int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
tExprNode* pLeft = tree->_node.pLeft;
int32_t ret = TSDB_CODE_SUCCESS;
if((ret = filterDealJson(info, tree, &pLeft)) != TSDB_CODE_SUCCESS) return ret;
SFilterFieldId left = {0}, right = {0};
filterAddFieldFromNode(info, tree->_node.pLeft, &left);
SVariant* var = tree->_node.pRight->pVal;
int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
size_t len = 0;
uint16_t uidx = 0;
filterAddFieldFromNode(info, pLeft, &left);
uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
int32_t len = 0;
uint32_t uidx = 0;
if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) {
void *data = NULL;
filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type);
// CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param");
SVariant* var = tree->_node.pRight->pVal;
filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type, false);
if (data == NULL) {
fltError("failed to convert in param");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (taosHashGetSize((SHashObj *)data) <= 0) {
filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx);
......@@ -1209,22 +1275,12 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
void *p = taosHashIterate((SHashObj *)data, NULL);
while(p) {
void* key = NULL;
len = 0;
taosHashGetKey(p, &key, &len);
void *key = taosHashGetDataKey((SHashObj *)data, p);
void *fdata = NULL;
if (IS_VAR_DATA_TYPE(type)) {
fdata = malloc(len + VARSTR_HEADER_SIZE);
varDataLen(fdata) = len;
memcpy(varDataVal(fdata), key, len);
len += VARSTR_HEADER_SIZE;
} else {
fdata = malloc(sizeof(int64_t));
SIMPLE_COPY_VALUES(fdata, key);
len = tDataTypes[type].bytes;
}
filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true);
......@@ -1242,7 +1298,8 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
} else {
filterAddFieldFromNode(info, tree->_node.pRight, &right);
filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);
ret = filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);
CHK_LRET(ret != TSDB_CODE_SUCCESS, TSDB_CODE_QRY_APP_ERROR, "invalid where condition");
SFilterGroup fgroup = {0};
filterAddUnitToGroup(&fgroup, uidx);
......@@ -1254,7 +1311,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
}
int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) {
int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) {
SFilterFieldId left, right, *pright = &right;
int32_t type = FILTER_UNIT_DATA_TYPE(u);
uint16_t flag = FLD_DESC_NO_FREE;
......@@ -1267,7 +1324,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
void *data = FILTER_UNIT_VAL_DATA(src, u);
if (IS_VAR_DATA_TYPE(type)) {
if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) {
filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false);
filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right.
t = FILTER_GET_FIELD(dst, right);
......@@ -1289,7 +1346,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx);
}
int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) {
int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) {
SFilterUnit *u = &info->units[uidx];
u->compare.optr2 = optr;
......@@ -1299,9 +1356,9 @@ int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *righ
}
int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) {
int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) {
SFilterFieldId left, right, right2;
uint16_t uidx = 0;
uint32_t uidx = 0;
SFilterField *col = FILTER_GET_COL_FIELD(src, cidx);
......@@ -1479,61 +1536,80 @@ static void filterFreeGroup(void *pItem) {
}
int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) {
EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* leftGroup = NULL;
SArray* rightGroup = NULL;
SArray* preGroup = NULL;
SArray* newGroup = NULL;
SArray* resGroup = NULL;
ENodeType nType = nodeType(nType);
SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext;
if (tree->nodeType != TEXPR_BINARYEXPR_NODE) {
//qError("invalid nodeType:%d", tree->nodeType);
return TSDB_CODE_QRY_APP_ERROR;
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) {
SLogicConditionNode *node = (SLogicConditionNode *)pNode;
if (LOGIC_COND_TYPE_AND == node->condType) {
SListCell *cell = node->pParameterList->pHead;
for (int32_t i = 0; i < node->pParameterList->length; ++i) {
newGroup = taosArrayInit(4, sizeof(SFilterGroup));
resGroup = taosArrayInit(4, sizeof(SFilterGroup));
SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup};
nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx);
FLT_ERR_JRET(tctx.code);
FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup));
taosArrayDestroyEx(newGroup, filterFreeGroup);
taosArrayDestroyEx(preGroup, filterFreeGroup);
preGroup = resGroup;
}
if (tree->_node.optr == TSDB_RELATION_AND) {
leftGroup = taosArrayInit(4, sizeof(SFilterGroup));
rightGroup = taosArrayInit(4, sizeof(SFilterGroup));
ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup));
ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup));
taosArrayAddAll(ctx->group, resGroup);
ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup));
taosArrayDestroyEx(newGroup, filterFreeGroup);
taosArrayDestroyEx(preGroup, filterFreeGroup);
taosArrayDestroyEx(resGroup, filterFreeGroup);
taosArrayDestroyEx(leftGroup, filterFreeGroup);
taosArrayDestroyEx(rightGroup, filterFreeGroup);
return DEAL_RES_IGNORE_CHILD;
}
return TSDB_CODE_SUCCESS;
if (LOGIC_COND_TYPE_OR == node->condType) {
SListCell *cell = node->pParameterList->pHead;
for (int32_t i = 0; i < node->pParameterList->length; ++i) {
nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext);
FLT_ERR_JRET(ctx.code);
}
return DEAL_RES_IGNORE_CHILD;
}
if (tree->_node.optr == TSDB_RELATION_OR) {
ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group));
ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group));
fltError("invalid condition type, type:%d", node->condType);
return TSDB_CODE_SUCCESS;
return DEAL_RES_ERROR;
}
code = filterAddGroupUnitFromNode(info, tree, group);
if (QUERY_NODE_OPERATOR == nType) {
FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group));
return DEAL_RES_CONTINUE;
}
_return:
sclError("invalid node type for filter, type:%d", nodeType(pNode));
taosArrayDestroyEx(leftGroup, filterFreeGroup);
taosArrayDestroyEx(rightGroup, filterFreeGroup);
code = TSDB_CODE_QRY_INVALID_INPUT;
return code;
}
_return:
#if 0
int32_t filterInitUnitFunc(SFilterInfo *info) {
for (uint16_t i = 0; i < info->unitNum; ++i) {
SFilterUnit* unit = &info->units[i];
taosArrayDestroyEx(newGroup, filterFreeGroup);
taosArrayDestroyEx(preGroup, filterFreeGroup);
taosArrayDestroyEx(resGroup, filterFreeGroup);
info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr);
}
ctx->code = code;
return TSDB_CODE_SUCCESS;
return DEAL_RES_ERROR;
}
#endif
int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
int32_t n = 0;
switch (type) {
......@@ -1611,34 +1687,37 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le
void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) {
if (qDebugFlag & DEBUG_DEBUG) {
// CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg);
if (info == NULL) {
fltDebug("%s - FilterInfo: EMPTY", msg);
return;
}
if (options == 0) {
// //qDebug("%s - FilterInfo:", msg);
// //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
qDebug("%s - FilterInfo:", msg);
qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema *sch = field->desc;
// //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name);
qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name);
}
//qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num);
for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) {
qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num);
for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i];
if (field->desc) {
SVariant *var = field->desc;
if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
//qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1));
qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1));
} else {
//qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO
qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO
}
} else if (field->data) {
//qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO
qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO
}
}
//qDebug("UNIT Num:%u", info->unitNum);
for (uint16_t i = 0; i < info->unitNum; ++i) {
qDebug("UNIT Num:%u", info->unitNum);
for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i];
int32_t type = FILTER_UNIT_DATA_TYPE(unit);
int32_t len = 0;
......@@ -1647,7 +1726,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
SSchema *sch = left->desc;
if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_CONTAINS){
len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str);
}
if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit);
......@@ -1656,7 +1737,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
tlen = varDataLen(data);
data += VARSTR_HEADER_SIZE;
}
converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen);
if (data) fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen);
} else {
strcat(str, "NULL");
}
......@@ -1664,7 +1745,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
if (unit->compare.optr2) {
strcat(str, " && ");
if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_CONTAINS){
sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str);
}
if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit);
......@@ -1673,23 +1756,23 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
tlen = varDataLen(data);
data += VARSTR_HEADER_SIZE;
}
converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen);
fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen);
} else {
strcat(str, "NULL");
}
strcat(str, "]");
}
//qDebug("%s", str); //TODO
qDebug("%s", str); //TODO
}
//qDebug("GROUP Num:%u", info->groupNum);
for (uint16_t i = 0; i < info->groupNum; ++i) {
qDebug("GROUP Num:%u", info->groupNum);
for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup *group = &info->groups[i];
//qDebug("Group%d : unit num[%u]", i, group->unitNum);
qDebug("Group%d : unit num[%u]", i, group->unitNum);
for (uint16_t u = 0; u < group->unitNum; ++u) {
//qDebug("unit id:%u", group->unitIdxs[u]);
for (uint32_t u = 0; u < group->unitNum; ++u) {
qDebug("unit id:%u", group->unitIdxs[u]);
}
}
......@@ -1697,12 +1780,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
}
if (options == 1) {
//qDebug("%s - RANGE info:", msg);
qDebug("%s - RANGE info:", msg);
//qDebug("RANGE Num:%u", info->colRangeNum);
for (uint16_t i = 0; i < info->colRangeNum; ++i) {
qDebug("RANGE Num:%u", info->colRangeNum);
for (uint32_t i = 0; i < info->colRangeNum; ++i) {
SFilterRangeCtx *ctx = info->colRange[i];
//qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange);
qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange);
if (ctx->isrange) {
SFilterRangeNode *r = ctx->rs;
while (r) {
......@@ -1712,7 +1795,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
strcat(str,"(NULL)");
} else {
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"[");
converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen);
fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen);
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]");
}
strcat(str, " - ");
......@@ -1720,10 +1803,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
strcat(str, "(NULL)");
} else {
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"[");
converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen);
fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen);
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]");
}
//qDebug("range: %s", str);
qDebug("range: %s", str);
r = r->next;
}
......@@ -1733,25 +1816,25 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
return;
}
//qDebug("%s - Block Filter info:", msg);
qDebug("%s - Block Filter info:", msg);
if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) {
//qDebug("Flag:%s", "ALL");
qDebug("Flag:%s", "ALL");
return;
} else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) {
//qDebug("Flag:%s", "EMPTY");
qDebug("Flag:%s", "EMPTY");
return;
} else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){
//qDebug("Flag:%s", "ACTIVE");
qDebug("Flag:%s", "ACTIVE");
}
//qDebug("GroupNum:%d", info->blkGroupNum);
uint16_t *unitIdx = info->blkUnits;
for (uint16_t i = 0; i < info->blkGroupNum; ++i) {
//qDebug("Group[%d] UnitNum: %d:", i, *unitIdx);
uint16_t unitNum = *(unitIdx++);
for (uint16_t m = 0; m < unitNum; ++m) {
//qDebug("uidx[%d]", *(unitIdx++));
qDebug("GroupNum:%d", info->blkGroupNum);
uint32_t *unitIdx = info->blkUnits;
for (uint32_t i = 0; i < info->blkGroupNum; ++i) {
qDebug("Group[%d] UnitNum: %d:", i, *unitIdx);
uint32_t unitNum = *(unitIdx++);
for (uint32_t m = 0; m < unitNum; ++m) {
qDebug("uidx[%d]", *(unitIdx++));
}
}
}
......@@ -1837,21 +1920,23 @@ void filterFreePCtx(SFilterPCtx *pctx) {
}
void filterFreeInfo(SFilterInfo *info) {
CHK_RETV(info == NULL);
if (info == NULL) {
return;
}
tfree(info->cunits);
tfree(info->blkUnitRes);
tfree(info->blkUnits);
for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) {
for (uint16_t f = 0; f < info->fields[i].num; ++f) {
for (uint32_t f = 0; f < info->fields[i].num; ++f) {
filterFreeField(&info->fields[i].fields[f], i);
}
tfree(info->fields[i].fields);
}
for (int32_t i = 0; i < info->groupNum; ++i) {
for (uint32_t i = 0; i < info->groupNum; ++i) {
filterFreeGroup(&info->groups[i]);
}
......@@ -1863,7 +1948,7 @@ void filterFreeInfo(SFilterInfo *info) {
tfree(info->unitFlags);
for (uint16_t i = 0; i < info->colRangeNum; ++i) {
for (uint32_t i = 0; i < info->colRangeNum; ++i) {
filterFreeRangeCtx(info->colRange[i]);
}
......@@ -1901,7 +1986,7 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) {
int32_t filterInitValFieldData(SFilterInfo *info) {
for (uint16_t i = 0; i < info->unitNum; ++i) {
for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit* unit = &info->units[i];
if (unit->right.type != FLD_TYPE_VALUE) {
assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR);
......@@ -1923,8 +2008,11 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
}
if (unit->compare.optr == TSDB_RELATION_IN) {
filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type);
// CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param");
filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type, false);
if (fi->data == NULL) {
fltError("failed to convert in param");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH);
......@@ -1937,7 +2025,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
} else if (type == TSDB_DATA_TYPE_NCHAR) {
size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;
fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
} else {
} else if (type != TSDB_DATA_TYPE_JSON){
if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE
fi->data = calloc(var->nLen, tDataTypes[type].bytes);
for (int32_t a = 0; a < var->nLen; ++a) {
......@@ -1949,19 +2037,47 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
} else {
fi->data = calloc(1, sizeof(int64_t));
}
} else{ // type == TSDB_DATA_TYPE_JSON
// fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc)
}
if(type != TSDB_DATA_TYPE_JSON){
bool converted = false;
char extInfo = 0;
// if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) {
// if (converted) {
// filterHandleValueExtInfo(unit, extInfo);
//
// continue;
// }
// //qError("dump value to type[%d] failed", type);
// return TSDB_CODE_TSC_INVALID_OPERATION;
// }
if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) {
if (converted) {
filterHandleValueExtInfo(unit, extInfo);
continue;
}
qError("dump value to type[%d] failed", type);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
}
// match/nmatch for nchar type need convert from ucs4 to mbs
if(type == TSDB_DATA_TYPE_NCHAR &&
(unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
if (len < 0){
qError("filterInitValFieldData taosUcs4ToMbs error 1");
return TSDB_CODE_FAILED;
}
varDataSetLen(newValData, len);
varDataCopy(fi->data, newValData);
}else if(type == TSDB_DATA_TYPE_JSON &&
(unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
int32_t len = taosUcs4ToMbs(((tVariant*)(fi->desc))->pz, ((tVariant*)(fi->desc))->nLen, newValData);
if (len < 0){
qError("filterInitValFieldData taosUcs4ToMbs error 2");
return TSDB_CODE_FAILED;
}
memcpy(((tVariant*)(fi->desc))->pz, newValData, len);
((tVariant*)(fi->desc))->nLen = len;
}
}
return TSDB_CODE_SUCCESS;
......@@ -1971,6 +2087,8 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) {
int32_t ret = func(left, right);
if(ret == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
switch (optr) {
case TSDB_RELATION_EQUAL: {
return ret == 0;
......@@ -1996,6 +2114,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right)
case TSDB_RELATION_MATCH: {
return ret == 0;
}
case TSDB_RELATION_CONTAINS: {
return ret == 0;
}
case TSDB_RELATION_NMATCH: {
return ret == 0;
}
......@@ -2092,7 +2213,7 @@ _return:
}
int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) {
int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colIdx, bool *empty) {
SArray* colArray = (SArray *)gRes->colInfo[colIdx].info;
int32_t size = (int32_t)taosArrayGetSize(colArray);
int32_t type = gRes->colInfo[colIdx].dataType;
......@@ -2109,6 +2230,10 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colI
filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND);
CHK_JMP(MR_EMPTY_RES(ctx));
}
if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){
gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL;
SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u));
}
}
taosArrayDestroy(colArray);
......@@ -2129,11 +2254,11 @@ _return:
int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) {
bool empty = false;
uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t));
uint16_t colIdxi = 0;
uint16_t gResIdx = 0;
uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t));
uint32_t colIdxi = 0;
uint32_t gResIdx = 0;
for (uint16_t i = 0; i < info->groupNum; ++i) {
for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup* g = info->groups + i;
gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx));
......@@ -2141,9 +2266,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
colIdxi = 0;
empty = false;
for (uint16_t j = 0; j < g->unitNum; ++j) {
for (uint32_t j = 0; j < g->unitNum; ++j) {
SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j);
uint16_t cidx = FILTER_UNIT_COL_IDX(u);
uint32_t cidx = FILTER_UNIT_COL_IDX(u);
if (gRes[gResIdx]->colInfo[cidx].info == NULL) {
gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES);
......@@ -2159,10 +2284,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
}
if (colIdxi > 1) {
qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
}
for (uint16_t l = 0; l < colIdxi; ++l) {
for (uint32_t l = 0; l < colIdxi; ++l) {
int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType;
if (FILTER_NO_MERGE_DATA_TYPE(type)) {
......@@ -2201,7 +2326,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
}
void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) {
uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0;
uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
bool equal = false;
for (; m < gRes1->colNum; ++m) {
......@@ -2225,6 +2350,15 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool
return;
}
// for long in operation
if (gRes1->colInfo[idx1].optr == TSDB_RELATION_EQUAL && gRes2->colInfo[idx2].optr == TSDB_RELATION_EQUAL) {
SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info;
if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){
*conflict = true;
return;
}
}
++n;
equal = true;
break;
......@@ -2241,7 +2375,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool
}
int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) {
int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint32_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) {
SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx);
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
......@@ -2271,10 +2405,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter
FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE);
uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0;
uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum;
bool equal = false;
uint16_t equal1 = 0, equal2 = 0, merNum = 0;
uint32_t equal1 = 0, equal2 = 0, merNum = 0;
SFilterRangeCtx *ctx = NULL;
SFilterColCtx colCtx = {0};
SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx));
......@@ -2388,7 +2522,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx);
int32_t pEnd = 0, cStart = 0, cEnd = 0;
uint16_t pColNum = 0, cColNum = 0;
uint32_t pColNum = 0, cColNum = 0;
int32_t movedNum = 0;
bool all = false;
......@@ -2454,7 +2588,6 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
}
cStart = i;
cEnd = i;
cColNum = gRes[i]->colNum;
}
......@@ -2470,7 +2603,7 @@ _return:
int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) {
size_t groupSize = taosArrayGetSize(group);
info->groupNum = (uint16_t)groupSize;
info->groupNum = (uint32_t)groupSize;
if (info->groupNum > 0) {
info->groups = calloc(info->groupNum, sizeof(*info->groups));
......@@ -2487,7 +2620,7 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) {
int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) {
if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) {
//qDebug("no need rewrite");
qDebug("no need rewrite");
return TSDB_CODE_SUCCESS;
}
......@@ -2499,7 +2632,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
SFilterGroupCtx *res = NULL;
SFilterColInfo *colInfo = NULL;
int32_t optr = 0;
uint16_t uidx = 0;
uint32_t uidx = 0;
memset(info, 0, sizeof(*info));
......@@ -2519,7 +2652,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
SFilterGroup ng = {0};
for (uint16_t m = 0; m < res->colNum; ++m) {
for (uint32_t m = 0; m < res->colNum; ++m) {
colInfo = &res->colInfo[res->colIdx[m]];
if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) {
assert(colInfo->type == RANGE_TYPE_UNIT);
......@@ -2555,13 +2688,13 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
}
int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) {
uint16_t *idxs = NULL;
uint16_t colNum = 0;
uint32_t *idxs = NULL;
uint32_t colNum = 0;
SFilterGroupCtx *res = NULL;
uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum));
uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum));
for (int32_t i = 0; i < gResNum; ++i) {
for (uint16_t m = 0; m < gRes[i]->colNum; ++m) {
for (uint32_t m = 0; m < gRes[i]->colNum; ++m) {
SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]];
if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) {
continue;
......@@ -2571,7 +2704,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
}
}
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
if (idxNum[i] < gResNum) {
continue;
}
......@@ -2592,9 +2725,9 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
for (int32_t i = 0; i < gResNum; ++i) {
res = gRes[i];
uint16_t n = 0;
uint32_t n = 0;
for (uint16_t m = 0; m < info->colRangeNum; ++m) {
for (uint32_t m = 0; m < info->colRangeNum; ++m) {
for (; n < res->colNum; ++n) {
if (res->colIdx[n] < idxs[m]) {
continue;
......@@ -2642,7 +2775,7 @@ _return:
}
int32_t filterPostProcessRange(SFilterInfo *info) {
for (uint16_t i = 0; i < info->colRangeNum; ++i) {
for (uint32_t i = 0; i < info->colRangeNum; ++i) {
SFilterRangeCtx* ctx = info->colRange[i];
SFilterRangeNode *r = ctx->rs;
while (r) {
......@@ -2656,13 +2789,11 @@ int32_t filterPostProcessRange(SFilterInfo *info) {
int32_t filterGenerateComInfo(SFilterInfo *info) {
uint16_t n = 0;
info->cunits = malloc(info->unitNum * sizeof(*info->cunits));
info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum);
info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum);
for (uint16_t i = 0; i < info->unitNum; ++i) {
for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i];
info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr);
......@@ -2672,7 +2803,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
if (unit->right.type == FLD_TYPE_VALUE) {
if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant
info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit);
}else{
info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
}
} else {
info->cunits[i].valData = NULL;
}
......@@ -2686,29 +2821,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit);
}
uint16_t cgroupNum = info->groupNum + 1;
for (uint16_t i = 0; i < info->groupNum; ++i) {
cgroupNum += info->groups[i].unitNum;
}
info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups));
for (uint16_t i = 0; i < info->groupNum; ++i) {
info->cgroups[n++] = info->groups[i].unitNum;
for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) {
info->cgroups[n++] = info->groups[i].unitIdxs[m];
}
}
info->cgroups[n] = 0;
return TSDB_CODE_SUCCESS;
}
int32_t filterUpdateComUnits(SFilterInfo *info) {
for (uint16_t i = 0; i < info->unitNum; ++i) {
for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i];
info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
......@@ -2723,7 +2840,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum);
for (int32_t k = 0; k < info->unitNum; ++k) {
for (uint32_t k = 0; k < info->unitNum; ++k) {
int32_t index = -1;
SFilterComUnit *cunit = &info->cunits[k];
......@@ -2770,16 +2887,18 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL
|| cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH
|| cunit->optr == TSDB_RELATION_NOT_EQUAL) {
|| cunit->optr == TSDB_RELATION_NOT_EQUAL || cunit->optr == TSDB_RELATION_CONTAINS) {
continue;
}
SColumnDataAgg* pDataBlockst = &pDataStatis[index];
void *minVal, *maxVal;
float minv = 0;
float maxv = 0;
if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) {
float minv = (float)(*(double *)(&pDataBlockst->min));
float maxv = (float)(*(double *)(&pDataBlockst->max));
minv = (float)(*(double *)(&pDataBlockst->min));
maxv = (float)(*(double *)(&pDataBlockst->max));
minVal = &minv;
maxVal = &maxv;
......@@ -2834,12 +2953,15 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
}
// CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY");
if (rmUnit == 0) {
fltDebug("NO Block Filter APPLY");
FLT_RET(TSDB_CODE_SUCCESS);
}
info->blkGroupNum = info->groupNum;
uint16_t *unitNum = info->blkUnits;
uint16_t *unitIdx = unitNum + 1;
uint32_t *unitNum = info->blkUnits;
uint32_t *unitIdx = unitNum + 1;
int32_t all = 0, empty = 0;
for (uint32_t g = 0; g < info->groupNum; ++g) {
......@@ -2849,7 +2971,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
empty = 0;
for (uint32_t u = 0; u < group->unitNum; ++u) {
uint16_t uidx = group->unitIdxs[u];
uint32_t uidx = group->unitIdxs[u];
if (info->blkUnitRes[uidx] == 1) {
--(*unitNum);
all = 1;
......@@ -2896,9 +3018,11 @@ _return:
bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
SFilterInfo *info = (SFilterInfo *)pinfo;
bool all = true;
uint16_t *unitIdx = NULL;
uint32_t *unitIdx = NULL;
if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) {
//FILTER_UNIT_CLR_F(info);
......@@ -2906,7 +3030,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p,
unitIdx = info->blkUnits;
for (uint32_t g = 0; g < info->blkGroupNum; ++g) {
uint16_t unitNum = *(unitIdx++);
uint32_t unitNum = *(unitIdx++);
for (uint32_t u = 0; u < unitNum; ++u) {
SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)];
void *colData = (char *)cunit->colData + cunit->dataSize * i;
......@@ -3001,12 +3125,25 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
return all;
}
if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) {
uint16_t uidx = info->groups[0].unitIdxs[0];
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
if (!colData){ // for json->'key' is null
(*p)[i] = 1;
}else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null
colData = POINTER_SHIFT(colData, CHAR_BYTES);
(*p)[i] = isNull(colData, info->cunits[uidx].dataType);
}else{
(*p)[i] = 0;
}
}else{
(*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType));
}
if ((*p)[i] == 0) {
all = false;
}
......@@ -3022,12 +3159,27 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
return all;
}
if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) {
uint16_t uidx = info->groups[0].unitIdxs[0];
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
if (!colData) { // for json->'key' is not null
(*p)[i] = 0;
}else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null
colData = POINTER_SHIFT(colData, CHAR_BYTES);
(*p)[i] = !isNull(colData, info->cunits[uidx].dataType);
}else{ // for json->'key' is not null
(*p)[i] = 1;
}
}else {
(*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType));
}
if ((*p)[i] == 0) {
all = false;
}
......@@ -3036,7 +3188,48 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
return all;
}
bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
static void doJsonCompare(SFilterComUnit *cunit, int8_t *result, void* colData){
if(cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH){
uint8_t jsonType = *(char*)colData;
char* realData = POINTER_SHIFT(colData, CHAR_BYTES);
if (jsonType != TSDB_DATA_TYPE_NCHAR){
*result = false;
}else{
char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData));
if (len < 0){
qError("castConvert1 taosUcs4ToMbs error");
tfree(newColData);
return;
}
varDataSetLen(newColData, len);
tVariant* val = cunit->valData;
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE);
memcpy(varDataVal(newValData), val->pz, val->nLen);
varDataSetLen(newValData, val->nLen);
*result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, newValData);
tfree(newColData);
}
}else if(cunit->optr == TSDB_RELATION_LIKE){
uint8_t jsonType = *(char*)colData;
char* realData = POINTER_SHIFT(colData, CHAR_BYTES);
if (jsonType != TSDB_DATA_TYPE_NCHAR){
*result = false;
}else{
tVariant* val = cunit->valData;
char* newValData = calloc(val->nLen + VARSTR_HEADER_SIZE, 1);
memcpy(varDataVal(newValData), val->pz, val->nLen);
varDataSetLen(newValData, val->nLen);
*result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, realData, newValData);
tfree(newValData);
}
}else{
*result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
}
}
bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
SFilterInfo *info = (SFilterInfo *)pinfo;
bool all = true;
uint16_t dataSize = info->cunits[0].dataSize;
......@@ -3050,10 +3243,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD
return all;
}
if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) {
if (isNull(colData, info->cunits[0].dataType)) {
if (colData == NULL || isNull(colData, info->cunits[0].dataType)) {
all = false;
colData += dataSize;
continue;
......@@ -3079,17 +3274,35 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa
return all;
}
if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) {
uint16_t uidx = info->groups[0].unitIdxs[0];
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
if (isNull(colData, info->cunits[uidx].dataType)) {
if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) {
(*p)[i] = 0;
all = false;
continue;
}
// match/nmatch for nchar type need convert from ucs4 to mbs
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){
char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
if (len < 0){
qError("castConvert1 taosUcs4ToMbs error");
}else{
varDataSetLen(newColData, len);
(*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData);
}
tfree(newColData);
}else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
doJsonCompare(&(info->cunits[uidx]), &(*p)[i], colData);
}else{
(*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData);
}
if ((*p)[i] == 0) {
all = false;
......@@ -3108,7 +3321,9 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
return all;
}
if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) {
//FILTER_UNIT_CLR_F(info);
......@@ -3116,7 +3331,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
for (uint32_t g = 0; g < info->groupNum; ++g) {
SFilterGroup *group = &info->groups[g];
for (uint32_t u = 0; u < group->unitNum; ++u) {
uint16_t uidx = group->unitIdxs[u];
uint32_t uidx = group->unitIdxs[u];
SFilterComUnit *cunit = &info->cunits[uidx];
void *colData = (char *)cunit->colData + cunit->dataSize * i;
......@@ -3125,7 +3340,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
//} else {
uint8_t optr = cunit->optr;
if (isNull(colData, cunit->dataType)) {
if (colData == NULL || isNull(colData, cunit->dataType)) {
(*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false;
} else {
if (optr == TSDB_RELATION_NOTNULL) {
......@@ -3135,8 +3350,22 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
} else if (cunit->rfunc >= 0) {
(*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]);
} else {
if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){
char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
if (len < 0){
qError("castConvert1 taosUcs4ToMbs error");
}else{
varDataSetLen(newColData, len);
(*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData);
}
tfree(newColData);
}else if(cunit->dataType == TSDB_DATA_TYPE_JSON){
doJsonCompare(cunit, &(*p)[i], colData);
}else{
(*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
}
}
//FILTER_UNIT_SET_R(info, uidx, p[i]);
//FILTER_UNIT_SET_F(info, uidx);
......@@ -3211,13 +3440,13 @@ int32_t filterPreprocess(SFilterInfo *info) {
filterMergeGroups(info, gRes, &gResNum);
if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) {
// qInfo("Final - FilterInfo: [ALL]");
qInfo("Final - FilterInfo: [ALL]");
goto _return;
}
if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) {
// qInfo("Final - FilterInfo: [EMPTY]");
qInfo("Final - FilterInfo: [EMPTY]");
goto _return;
}
......@@ -3244,26 +3473,19 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) {
// CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
// CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
return TSDB_CODE_SUCCESS;
}
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema* sch = fi->desc;
for (int32_t j = 0; j < numOfCols; ++j) {
SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j);
if (sch->colId == pColInfo->info.colId) {
fi->data = pColInfo->pData;
break;
}
}
(*fp)(param, sch->colId, &fi->data);
}
filterUpdateComUnits(info);
......@@ -3271,46 +3493,52 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat
return TSDB_CODE_SUCCESS;
}
int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp) {
CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) {
int32_t code = TSDB_CODE_SUCCESS;
SFilterInfo *info = NULL;
if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
return TSDB_CODE_SUCCESS;
}
// CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param");
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema* sch = fi->desc;
if (*pinfo == NULL) {
*pinfo = calloc(1, sizeof(SFilterInfo));
(*fp)(param, sch->colId, sch->name, &fi->data);
}
info = *pinfo;
filterUpdateComUnits(info);
info->options = options;
return TSDB_CODE_SUCCESS;
}
int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup));
filterInitUnitsFields(info);
code = filterTreeToGroup(tree, info, group);
ERR_JRET(code);
SFltBuildGroupCtx tctx = {.info = info, .group = group};
nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx);
FLT_ERR_JRET(tctx.code);
filterConvertGroupFromArray(info, group);
taosArrayDestroy(group);
ERR_JRET(filterInitValFieldData(info));
FLT_ERR_JRET(filterInitValFieldData(info));
if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) {
filterDumpInfoToString(info, "Before preprocess", 0);
ERR_JRET(filterPreprocess(info));
FLT_ERR_JRET(filterPreprocess(info));
CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL));
FLT_CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL));
if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) {
taosArrayDestroy(group);
return code;
}
//ERR_JRET(filterInitUnitFunc(info));
}
info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes));
......@@ -3318,17 +3546,11 @@ int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t option
filterDumpInfoToString(info, "Final", 0);
taosArrayDestroy(group);
return code;
_return:
// qInfo("No filter, code:%d", code);
taosArrayDestroy(group);
filterFreeInfo(*pinfo);
*pinfo = NULL;
qInfo("No filter, code:%d", code);
return code;
}
......@@ -3348,7 +3570,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
bool ret = true;
void *minVal, *maxVal;
for (int32_t k = 0; k < info->colRangeNum; ++k) {
for (uint32_t k = 0; k < info->colRangeNum; ++k) {
int32_t index = -1;
SFilterRangeCtx *ctx = info->colRange[k];
for(int32_t i = 0; i < numOfCols; ++i) {
......@@ -3368,30 +3590,35 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
break;
}
if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) {
if (pDataStatis[index].numOfNull <= 0) {
if (ctx->isnull && !ctx->notnull && !ctx->isrange) {
ret = false;
break;
}
// all data in current column are NULL, no need to check its boundary value
} else if (pDataStatis[index].numOfNull > 0) {
if (pDataStatis[index].numOfNull == numOfRows) {
// if isNULL query exists, load the null data column
if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) {
ret = false;
break;
}
continue;
} else {
if (ctx->isnull) {
continue;
}
}
}
SColumnDataAgg* pDataBlockst = &pDataStatis[index];
SFilterRangeNode *r = ctx->rs;
float minv = 0;
float maxv = 0;
if (ctx->type == TSDB_DATA_TYPE_FLOAT) {
float minv = (float)(*(double *)(&pDataBlockst->min));
float maxv = (float)(*(double *)(&pDataBlockst->max));
minv = (float)(*(double *)(&pDataBlockst->min));
maxv = (float)(*(double *)(&pDataBlockst->max));
minVal = &minv;
maxVal = &maxv;
......@@ -3408,7 +3635,9 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
r = r->next;
}
CHK_RET(!ret, ret);
if (!ret) {
return ret;
}
}
return ret;
......@@ -3426,7 +3655,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
int32_t code = 0;
bool empty = false, all = false;
for (int32_t i = 0; i < info->groupNum; ++i) {
for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup *group = &info->groups[i];
if (group->unitNum > 1) {
cur = tmpc;
......@@ -3436,8 +3665,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
optr = TSDB_RELATION_OR;
}
for (int32_t u = 0; u < group->unitNum; ++u) {
uint16_t uidx = group->unitIdxs[u];
for (uint32_t u = 0; u < group->unitNum; ++u) {
uint32_t uidx = group->unitIdxs[u];
SFilterUnit *unit = &info->units[uidx];
uint8_t raOptr = FILTER_UNIT_OPTR(unit);
......@@ -3478,8 +3707,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
} else {
filterGetRangeNum(prev, &num);
if (num > 1) {
//qError("only one time range accepted, num:%d", num);
ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION);
qError("only one time range accepted, num:%d", num);
FLT_ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION);
}
CHK_JMP(num < 1);
......@@ -3493,7 +3722,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
filterFreeRangeCtx(prev);
filterFreeRangeCtx(tmpc);
//qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
return TSDB_CODE_SUCCESS;
_return:
......@@ -3503,14 +3732,18 @@ _return:
filterFreeRangeCtx(prev);
filterFreeRangeCtx(tmpc);
//qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
return code;
}
int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) {
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
if (FILTER_EMPTY_RES(info) || FILTER_ALL_RES(info)) {
return TSDB_CODE_SUCCESS;
}
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
if (type == TSDB_DATA_TYPE_NCHAR) {
......@@ -3523,7 +3756,20 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
char *src = FILTER_GET_COL_FIELD_DATA(fi, j);
char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j);
int32_t len = 0;
taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
char *varSrc = varDataVal(src);
size_t k = 0, varSrcLen = varDataLen(src);
while (k < varSrcLen && varSrc[k++] == -1) {}
if (k == varSrcLen) {
/* NULL */
varDataLen(dst) = (VarDataLenT) varSrcLen;
varDataCopy(dst, src);
continue;
}
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
if(!ret) {
qError("filterConverNcharColumns taosMbsToUcs4 error");
return TSDB_CODE_FAILED;
}
varDataLen(dst) = len;
}
......@@ -3541,7 +3787,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
}
int32_t filterFreeNcharColumns(SFilterInfo* info) {
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
if (type == TSDB_DATA_TYPE_NCHAR) {
......@@ -3552,7 +3798,74 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) {
return TSDB_CODE_SUCCESS;
}
##################################################################################
EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
SFltTreeStat *stat = (SFltTreeStat *)pContext;
}
int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat);
FLT_RET(pStat->code);
}
int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
}
int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) {
int32_t code = 0;
SFilterInfo *info = NULL;
if (pNode == NULL || pInfo == NULL) {
fltError("invalid param");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (*pInfo == NULL) {
*pInfo = calloc(1, sizeof(SFilterInfo));
if (NULL == *pInfo) {
fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo));
FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
}
info = *pInfo;
info->options = options;
SFltTreeStat stat = {0};
FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat));
if (!info->scalarMode) {
FLT_ERR_JRET(fltInitFromNode(pNode, info, options));
}
FLT_ERR_JRET(fltOptimizeNodes(info, &pNode, &stat));
return code;
_return:
filterFreeInfo(*pInfo);
*pInfo = NULL;
FLT_RET(code);
}
FORCE_INLINE bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
if (info->scalarMode) {
SScalarParam output = {0};
FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output));
*p = output.data;
return TSDB_CODE_SUCCESS;
}
return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols);
}
source/libs/scalar/src/scalar.c
浏览文件 @ 8d413f09
......@@ -45,6 +45,11 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
break;
}
case QUERY_NODE_NODE_LIST: {
SNodeListNode *nodeList = (SNodeListNode *)node;
//TODO BUILD HASH
break;
}
case QUERY_NODE_COLUMN_REF: {
if (NULL == ctx) {
sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
......
source/libs/scalar/src/sclvector.c
浏览文件 @ 8d413f09
......@@ -19,7 +19,7 @@
#include "sclvector.h"
#include "tcompare.h"
#include "querynodes.h"
#include "filter.h"
#include "filterInt.h"
#include "query.h"
//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i]));
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或