Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 8d413f09 编写于 作者: D dapan1121
浏览文件
操作

feature/qnode

上级 6d04b954
隐藏空白更改
内联 并排
Showing with 737 addition and 751 deletion
source/libs/scalar/inc/tsclfunc.h → include/libs/scalar/filter.h
浏览文件 @ 8d413f09
...@@ -12,34 +12,25 @@ ...@@ -12,34 +12,25 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef TDENGINE_TSCALARFUNCTION_H #ifndef TDENGINE_SCALAR_H
#define TDENGINE_TSCALARFUNCTION_H #define TDENGINE_SCALAR_H
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#include "function.h" #include "function.h"
#include "scalar.h" #include "nodes.h"
typedef struct SScalarFunctionSupport { typedef struct SFilterInfo SFilterInfo;
struct SExprInfo *pExprInfo;
int32_t numOfCols;
SColumnInfo *colList;
void *exprList; // client side used
int32_t offset;
char** data;
} SScalarFunctionSupport;
extern struct SScalarFunctionInfo scalarFunc[8];
int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput,
void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t));
int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif // TDENGINE_TSCALARFUNCTION_H #endif // TDENGINE_SCALAR_H
source/libs/scalar/inc/filter.h → source/libs/scalar/inc/filterInt.h
浏览文件 @ 8d413f09
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef TDENGINE_QFILTER_H #ifndef TDENGINE_FILTER_INT_H
#define TDENGINE_QFILTER_H #define TDENGINE_FILTER_INT_H
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
...@@ -85,6 +85,12 @@ enum { ...@@ -85,6 +85,12 @@ enum {
RANGE_TYPE_MR_CTX = 3, RANGE_TYPE_MR_CTX = 3,
}; };
enum {
FI_ACTION_NO_NEED = 1,
FI_ACTION_CONTINUE,
FI_ACTION_STOP,
};
typedef struct OptrStr { typedef struct OptrStr {
uint16_t optr; uint16_t optr;
char *str; char *str;
...@@ -97,17 +103,11 @@ typedef struct SFilterRange { ...@@ -97,17 +103,11 @@ typedef struct SFilterRange {
char eflag; char eflag;
} SFilterRange; } SFilterRange;
typedef struct SFilterColRange {
uint16_t idx; //column field idx
bool isNull;
bool notNull;
bool isRange;
SFilterRange ra;
} SFilterColRange;
typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t(*filter_desc_compare_func)(const void *, const void *); typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t);
typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **);
typedef struct SFilterRangeCompare { typedef struct SFilterRangeCompare {
int64_t s; int64_t s;
...@@ -155,37 +155,39 @@ typedef struct SFilterField { ...@@ -155,37 +155,39 @@ typedef struct SFilterField {
} SFilterField; } SFilterField;
typedef struct SFilterFields { typedef struct SFilterFields {
uint16_t size; uint32_t size;
uint16_t num; uint32_t num;
SFilterField *fields; SFilterField *fields;
} SFilterFields; } SFilterFields;
typedef struct SFilterFieldId { typedef struct SFilterFieldId {
uint16_t type; uint16_t type;
uint16_t idx; uint32_t idx;
} SFilterFieldId; } SFilterFieldId;
typedef struct SFilterGroup { typedef struct SFilterGroup {
uint16_t unitSize; uint32_t unitSize;
uint16_t unitNum; uint32_t unitNum;
uint16_t *unitIdxs; uint32_t *unitIdxs;
uint8_t *unitFlags; // !unit result uint8_t *unitFlags; // !unit result
} SFilterGroup; } SFilterGroup;
typedef struct SFilterColInfo { typedef struct SFilterColInfo {
uint8_t type; uint8_t type;
int32_t dataType; int32_t dataType;
uint8_t optr; // for equal operation in the relation of RELATION_IN
int64_t value; // for equal operation in the relation of RELATION_IN
void *info; void *info;
} SFilterColInfo; } SFilterColInfo;
typedef struct SFilterGroupCtx { typedef struct SFilterGroupCtx {
uint16_t colNum; uint32_t colNum;
uint16_t *colIdx; uint32_t *colIdx;
SFilterColInfo *colInfo; SFilterColInfo *colInfo;
} SFilterGroupCtx; } SFilterGroupCtx;
typedef struct SFilterColCtx { typedef struct SFilterColCtx {
uint16_t colIdx; uint32_t colIdx;
void* ctx; void* ctx;
} SFilterColCtx; } SFilterColCtx;
...@@ -219,16 +221,31 @@ typedef struct SFilterPCtx { ...@@ -219,16 +221,31 @@ typedef struct SFilterPCtx {
SHashObj *unitHash; SHashObj *unitHash;
} SFilterPCtx; } SFilterPCtx;
typedef struct SFltTreeStat {
int32_t code;
} SFltTreeStat;
typedef struct SFltScalarCtx {
SNode *node;
} SFltScalarCtx;
typedef struct SFltBuildGroupCtx {
SFilterInfo *info;
SArray *group;
int32_t code;
} SFltBuildGroupCtx;
typedef struct SFilterInfo { typedef struct SFilterInfo {
bool scalarMode;
SFltScalarCtx sclCtx;
uint32_t options; uint32_t options;
uint32_t status; uint32_t status;
uint16_t unitSize; uint32_t unitSize;
uint16_t unitNum; uint32_t unitNum;
uint16_t groupNum; uint32_t groupNum;
uint16_t colRangeNum; uint32_t colRangeNum;
SFilterFields fields[FLD_TYPE_MAX]; SFilterFields fields[FLD_TYPE_MAX];
SFilterGroup *groups; SFilterGroup *groups;
uint16_t *cgroups;
SFilterUnit *units; SFilterUnit *units;
SFilterComUnit *cunits; SFilterComUnit *cunits;
uint8_t *unitRes; // result uint8_t *unitRes; // result
...@@ -236,16 +253,15 @@ typedef struct SFilterInfo { ...@@ -236,16 +253,15 @@ typedef struct SFilterInfo {
SFilterRangeCtx **colRange; SFilterRangeCtx **colRange;
filter_exec_func func; filter_exec_func func;
uint8_t blkFlag; uint8_t blkFlag;
uint16_t blkGroupNum; uint32_t blkGroupNum;
uint16_t *blkUnits; uint32_t *blkUnits;
int8_t *blkUnitRes; int8_t *blkUnitRes;
void *pTable;
SFilterPCtx pctx; SFilterPCtx pctx;
} SFilterInfo; } SFilterInfo;
#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) #define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON)
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR)
#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define MR_EMPTY_RES(ctx) (ctx->rs == NULL) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
...@@ -261,7 +277,7 @@ typedef struct SFilterInfo { ...@@ -261,7 +277,7 @@ typedef struct SFilterInfo {
#define FILTER_CLR_FLAG(st, f) st &= (~f) #define FILTER_CLR_FLAG(st, f) st &= (~f)
#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) #define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src)
#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0) #define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint32_t *)(_t + 1) = idx1; *(uint32_t *)(_t + 3) = idx2; } while (0)
#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE)))) #define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE))))
#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0) #define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0)
...@@ -271,15 +287,20 @@ typedef struct SFilterInfo { ...@@ -271,15 +287,20 @@ typedef struct SFilterInfo {
#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
#define CHK_RETV(c) do { if (c) { return; } } while (0) #define fltFatal(...) qFatal(__VA_ARGS__)
#define CHK_RET(c, r) do { if (c) { return r; } } while (0) #define fltError(...) qError(__VA_ARGS__)
#define CHK_JMP(c) do { if (c) { goto _return; } } while (0) #define fltWarn(...) qWarn(__VA_ARGS__)
#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0) #define fltInfo(...) qInfo(__VA_ARGS__)
#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0) #define fltDebug(...) qDebug(__VA_ARGS__)
#define fltTrace(...) qTrace(__VA_ARGS__)
#define FLT_CHK_JMP(c) do { if (c) { goto _return; } } while (0)
#define FLT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define FLT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define FLT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
...@@ -290,6 +311,7 @@ typedef struct SFilterInfo { ...@@ -290,6 +311,7 @@ typedef struct SFilterInfo {
#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) #define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
...@@ -302,6 +324,7 @@ typedef struct SFilterInfo { ...@@ -302,6 +324,7 @@ typedef struct SFilterInfo {
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
...@@ -316,7 +339,7 @@ typedef struct SFilterInfo { ...@@ -316,7 +339,7 @@ typedef struct SFilterInfo {
#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) #define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0)
#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) #define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0)
#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0) #define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0)
#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0)
...@@ -325,9 +348,10 @@ typedef struct SFilterInfo { ...@@ -325,9 +348,10 @@ typedef struct SFilterInfo {
#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
#if 0 #if 0
extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp);
extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
...@@ -344,4 +368,4 @@ extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); ...@@ -344,4 +368,4 @@ extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
} }
#endif #endif
#endif // TDENGINE_QFILTER_H #endif // TDENGINE_FILTER_INT_H
\ No newline at end of file
source/libs/scalar/inc/tfilter.h 已删除 100644 → 0
浏览文件 @ 6d04b954
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QFILTER_H
#define TDENGINE_QFILTER_H
#ifdef __cplusplus
extern "C" {
#endif
#include "thash.h"
#include "tname.h"
#include "common.h"
#include "scalar.h"
#include "querynodes.h"
#define FILTER_DEFAULT_GROUP_SIZE 4
#define FILTER_DEFAULT_UNIT_SIZE 4
#define FILTER_DEFAULT_FIELD_SIZE 4
#define FILTER_DEFAULT_VALUE_SIZE 4
#define FILTER_DEFAULT_GROUP_UNIT_SIZE 2
#define FILTER_DUMMY_EMPTY_OPTR 127
#define MAX_NUM_STR_SIZE 40
#define FILTER_RM_UNIT_MIN_ROWS 100
enum {
FLD_TYPE_COLUMN = 1,
FLD_TYPE_VALUE = 2,
FLD_TYPE_MAX = 3,
FLD_DESC_NO_FREE = 4,
FLD_DATA_NO_FREE = 8,
FLD_DATA_IS_HASH = 16,
};
enum {
MR_ST_START = 1,
MR_ST_FIN = 2,
MR_ST_ALL = 4,
MR_ST_EMPTY = 8,
};
enum {
RANGE_FLG_EXCLUDE = 1,
RANGE_FLG_INCLUDE = 2,
RANGE_FLG_NULL = 4,
};
enum {
FI_OPTION_NO_REWRITE = 1,
FI_OPTION_TIMESTAMP = 2,
FI_OPTION_NEED_UNIQE = 4,
};
enum {
FI_STATUS_ALL = 1,
FI_STATUS_EMPTY = 2,
FI_STATUS_REWRITE = 4,
FI_STATUS_CLONED = 8,
};
enum {
FI_STATUS_BLK_ALL = 1,
FI_STATUS_BLK_EMPTY = 2,
FI_STATUS_BLK_ACTIVE = 4,
};
enum {
RANGE_TYPE_UNIT = 1,
RANGE_TYPE_VAR_HASH = 2,
RANGE_TYPE_MR_CTX = 3,
};
typedef struct OptrStr {
uint16_t optr;
char *str;
} OptrStr;
typedef struct SFilterRange {
int64_t s;
int64_t e;
char sflag;
char eflag;
} SFilterRange;
typedef struct SFilterColRange {
uint16_t idx; //column field idx
bool isNull;
bool notNull;
bool isRange;
SFilterRange ra;
} SFilterColRange;
typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t);
typedef struct SFilterRangeCompare {
int64_t s;
int64_t e;
rangeCompFunc func;
} SFilterRangeCompare;
typedef struct SFilterRangeNode {
struct SFilterRangeNode* prev;
struct SFilterRangeNode* next;
union {
SFilterRange ra;
SFilterRangeCompare rc;
};
} SFilterRangeNode;
typedef struct SFilterRangeCtx {
int32_t type;
int32_t options;
int8_t status;
bool isnull;
bool notnull;
bool isrange;
int16_t colId;
__compar_fn_t pCompareFunc;
SFilterRangeNode *rf; //freed
SFilterRangeNode *rs;
} SFilterRangeCtx ;
typedef struct SFilterVarCtx {
int32_t type;
int32_t options;
int8_t status;
bool isnull;
bool notnull;
bool isrange;
SHashObj *wild;
SHashObj *value;
} SFilterVarCtx;
typedef struct SFilterField {
uint16_t flag;
void* desc;
void* data;
} SFilterField;
typedef struct SFilterFields {
uint16_t size;
uint16_t num;
SFilterField *fields;
} SFilterFields;
typedef struct SFilterFieldId {
uint16_t type;
uint16_t idx;
} SFilterFieldId;
typedef struct SFilterGroup {
uint16_t unitSize;
uint16_t unitNum;
uint16_t *unitIdxs;
uint8_t *unitFlags; // !unit result
} SFilterGroup;
typedef struct SFilterColInfo {
uint8_t type;
int32_t dataType;
void *info;
} SFilterColInfo;
typedef struct SFilterGroupCtx {
uint16_t colNum;
uint16_t *colIdx;
SFilterColInfo *colInfo;
} SFilterGroupCtx;
typedef struct SFilterColCtx {
uint16_t colIdx;
void* ctx;
} SFilterColCtx;
typedef struct SFilterCompare {
uint8_t type;
uint8_t optr;
uint8_t optr2;
} SFilterCompare;
typedef struct SFilterUnit {
SFilterCompare compare;
SFilterFieldId left;
SFilterFieldId right;
SFilterFieldId right2;
} SFilterUnit;
typedef struct SFilterComUnit {
void *colData;
void *valData;
void *valData2;
uint16_t colId;
uint16_t dataSize;
uint8_t dataType;
uint8_t optr;
int8_t func;
int8_t rfunc;
} SFilterComUnit;
typedef struct SFilterPCtx {
SHashObj *valHash;
SHashObj *unitHash;
} SFilterPCtx;
typedef struct SFilterInfo {
uint32_t options;
uint32_t status;
uint16_t unitSize;
uint16_t unitNum;
uint16_t groupNum;
uint16_t colRangeNum;
SFilterFields fields[FLD_TYPE_MAX];
SFilterGroup *groups;
uint16_t *cgroups;
SFilterUnit *units;
SFilterComUnit *cunits;
uint8_t *unitRes; // result
uint8_t *unitFlags; // got result
SFilterRangeCtx **colRange;
filter_exec_func func;
uint8_t blkFlag;
uint16_t blkGroupNum;
uint16_t *blkUnits;
int8_t *blkUnitRes;
SFilterPCtx pctx;
} SFilterInfo;
#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t))
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR)
#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0)
#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0)
#define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true)
#define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true)))
#define FILTER_GET_FLAG(st, f) (st & f)
#define FILTER_SET_FLAG(st, f) st |= (f)
#define FILTER_CLR_FLAG(st, f) st &= (~f)
#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src)
#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0)
#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE))))
#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0)
#define RESET_RANGE(ctx, r) do { (r)->next = (ctx)->rf; (ctx)->rf = r; } while (0)
#define FREE_RANGE(ctx, r) do { if ((r)->prev) { (r)->prev->next = (r)->next; } else { (ctx)->rs = (r)->next;} if ((r)->next) { (r)->next->prev = (r)->prev; } RESET_RANGE(ctx, r); } while (0)
#define FREE_FROM_RANGE(ctx, r) do { SFilterRangeNode *_r = r; if ((_r)->prev) { (_r)->prev->next = NULL; } else { (ctx)->rs = NULL;} while (_r) {SFilterRangeNode *n = (_r)->next; RESET_RANGE(ctx, _r); _r = n; } } while (0)
#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
#define CHK_RETV(c) do { if (c) { return; } } while (0)
#define CHK_RET(c, r) do { if (c) { return r; } } while (0)
#define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0)
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes)
#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc))
#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
#define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left)
#define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right)
#define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2)
#define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type)
#define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri)
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
#define FILTER_UNIT_CLR_F(i) memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags))
#define FILTER_UNIT_SET_F(i, idx) (i)->unitFlags[idx] = 1
#define FILTER_UNIT_GET_F(i, idx) ((i)->unitFlags[idx])
#define FILTER_UNIT_GET_R(i, idx) ((i)->unitRes[idx])
#define FILTER_UNIT_SET_R(i, idx, v) (i)->unitRes[idx] = (v)
#define FILTER_PUSH_UNIT(colInfo, u) do { (colInfo).type = RANGE_TYPE_UNIT; (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u);taosArrayPush((SArray *)((colInfo).info), &u);} while (0)
#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0)
#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0)
#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0)
#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0)
#define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL)
#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
#if 0
extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
#else
//REMOVE THESE!!!!!!!!!!!!!!!!!!!!
#include "function.h"
#endif
extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QFILTER_H
source/libs/scalar/src/filter.c
浏览文件 @ 8d413f09
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
#include "thash.h" #include "thash.h"
//#include "queryLog.h" //#include "queryLog.h"
#include "tcompare.h" #include "tcompare.h"
#include "filter.h" #include "filterInt.h"
OptrStr gOptrStr[] = { OptrStr gOptrStr[] = {
{TSDB_RELATION_INVALID, "invalid"}, {TSDB_RELATION_INVALID, "invalid"},
...@@ -28,21 +28,22 @@ OptrStr gOptrStr[] = { ...@@ -28,21 +28,22 @@ OptrStr gOptrStr[] = {
{TSDB_RELATION_GREATER_EQUAL, ">="}, {TSDB_RELATION_GREATER_EQUAL, ">="},
{TSDB_RELATION_NOT_EQUAL, "!="}, {TSDB_RELATION_NOT_EQUAL, "!="},
{TSDB_RELATION_LIKE, "like"}, {TSDB_RELATION_LIKE, "like"},
{TSDB_RELATION_MATCH, "match"},
{TSDB_RELATION_MATCH, "nmatch"},
{TSDB_RELATION_ISNULL, "is null"}, {TSDB_RELATION_ISNULL, "is null"},
{TSDB_RELATION_NOTNULL, "not null"}, {TSDB_RELATION_NOTNULL, "not null"},
{TSDB_RELATION_IN, "in"}, {TSDB_RELATION_IN, "in"},
{TSDB_RELATION_AND, "and"}, {TSDB_RELATION_AND, "and"},
{TSDB_RELATION_OR, "or"}, {TSDB_RELATION_OR, "or"},
{TSDB_RELATION_NOT, "not"} {TSDB_RELATION_NOT, "not"},
{TSDB_RELATION_MATCH, "match"},
{TSDB_RELATION_NMATCH, "nmatch"},
{TSDB_RELATION_CONTAINS, "contains"},
}; };
static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) {
const SSchema *sch1 = desc1; const SSchema *sch1 = desc1;
const SSchema *sch2 = desc2; const SSchema *sch2 = desc2;
return sch1->colId != sch2->colId; return !(strcmp(sch1->name, sch2->name) == 0 && sch1->colId == sch2->colId);
} }
static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) {
...@@ -60,16 +61,24 @@ filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { ...@@ -60,16 +61,24 @@ filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = {
}; };
bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(maxv, minr) >= 0; int32_t result = cfunc(maxv, minr);
if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result >= 0;
} }
bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(maxv, minr) > 0; int32_t result = cfunc(maxv, minr);
if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result > 0;
} }
bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(minv, maxr) <= 0; int32_t result = cfunc(minv, maxr);
if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result <= 0;
} }
bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(minv, maxr) < 0; int32_t result = cfunc(minv, maxr);
if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
return result < 0;
} }
bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0;
...@@ -291,7 +300,7 @@ int32_t filterInitUnitsFields(SFilterInfo *info) { ...@@ -291,7 +300,7 @@ int32_t filterInitUnitsFields(SFilterInfo *info) {
info->fields[FLD_TYPE_COLUMN].num = 0; info->fields[FLD_TYPE_COLUMN].num = 0;
info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE;
info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField));
info->fields[FLD_TYPE_VALUE].num = 0; info->fields[FLD_TYPE_VALUE].num = 0;
info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE;
info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField));
...@@ -318,7 +327,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt ...@@ -318,7 +327,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt
void* filterInitRangeCtx(int32_t type, int32_t options) { void* filterInitRangeCtx(int32_t type, int32_t options) {
if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
//qError("not supported range type:%d", type); qError("not supported range type:%d", type);
return NULL; return NULL;
} }
...@@ -685,7 +694,7 @@ int32_t filterGetRangeRes(void* h, SFilterRange *ra) { ...@@ -685,7 +694,7 @@ int32_t filterGetRangeRes(void* h, SFilterRange *ra) {
} }
if (num == 0) { if (num == 0) {
//qError("no range result"); qError("no range result");
return TSDB_CODE_QRY_APP_ERROR; return TSDB_CODE_QRY_APP_ERROR;
} }
...@@ -775,8 +784,15 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { ...@@ -775,8 +784,15 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) {
int32_t leftSize = (int32_t)taosArrayGetSize(left); int32_t leftSize = (int32_t)taosArrayGetSize(left);
int32_t rightSize = (int32_t)taosArrayGetSize(right); int32_t rightSize = (int32_t)taosArrayGetSize(right);
// CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); if (taosArrayGetSize(left) <= 0) {
// CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); fltDebug("empty group");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (taosArrayGetSize(right) <= 0) {
fltDebug("empty group");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
for (int32_t l = 0; l < leftSize; ++l) { for (int32_t l = 0; l < leftSize; ++l) {
SFilterGroup *gp1 = taosArrayGet(left, l); SFilterGroup *gp1 = taosArrayGet(left, l);
...@@ -793,7 +809,7 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { ...@@ -793,7 +809,7 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) {
} }
int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) {
for (uint16_t i = 0; i < fields->num; ++i) { for (uint32_t i = 0; i < fields->num; ++i) {
if (0 == gDescCompare[type](fields->fields[i].desc, v)) { if (0 == gDescCompare[type](fields->fields[i].desc, v)) {
return i; return i;
} }
...@@ -806,7 +822,7 @@ int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { ...@@ -806,7 +822,7 @@ int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) {
int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) {
if (type == FLD_TYPE_VALUE) { if (type == FLD_TYPE_VALUE) {
if (info->pctx.valHash == false) { if (info->pctx.valHash == false) {
//qError("value hash is empty"); qError("value hash is empty");
return -1; return -1;
} }
...@@ -819,10 +835,11 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d ...@@ -819,10 +835,11 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d
return -1; return -1;
} }
// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0
// Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better.
int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) {
int32_t idx = -1; int32_t idx = -1;
uint16_t *num; uint32_t *num;
num = &info->fields[type].num; num = &info->fields[type].num;
...@@ -885,8 +902,14 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte ...@@ -885,8 +902,14 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte
int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) {
// CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); if (node == NULL) {
// CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR); fltError("empty node");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE) {
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
int32_t type; int32_t type;
void *v; void *v;
...@@ -906,7 +929,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI ...@@ -906,7 +929,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) {
if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) {
if (info->pctx.unitHash == NULL) { if (info->pctx.unitHash == NULL) {
info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false);
...@@ -915,14 +938,14 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi ...@@ -915,14 +938,14 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1);
void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v));
if (hu) { if (hu) {
*uidx = *(uint16_t *)hu; *uidx = *(uint32_t *)hu;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }
} }
if (info->unitNum >= info->unitSize) { if (info->unitNum >= info->unitSize) {
uint16_t psize = info->unitSize; uint32_t psize = info->unitSize;
info->unitSize += FILTER_DEFAULT_UNIT_SIZE; info->unitSize += FILTER_DEFAULT_UNIT_SIZE;
info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit));
memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE);
...@@ -940,7 +963,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi ...@@ -940,7 +963,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u);
assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE));
} else { } else {
assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); if(optr != TSDB_RELATION_ISNULL && optr != TSDB_RELATION_NOTNULL && optr != FILTER_DUMMY_EMPTY_OPTR){
return -1;
}
} }
SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u);
...@@ -963,7 +988,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi ...@@ -963,7 +988,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) {
if (group->unitNum >= group->unitSize) { if (group->unitNum >= group->unitSize) {
group->unitSize += FILTER_DEFAULT_UNIT_SIZE; group->unitSize += FILTER_DEFAULT_UNIT_SIZE;
group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs));
...@@ -974,12 +999,14 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { ...@@ -974,12 +999,14 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType, bool tolower) {
SBufferReader br = tbufInitReader(buf, len, false); SBufferReader br = tbufInitReader(buf, len, false);
uint32_t sType = tbufReadUint32(&br); uint32_t sType = tbufReadUint32(&br);
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false);
int32_t code = 0; int32_t code = 0;
taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType));
int dummy = -1; int dummy = -1;
SVariant tmpVar = {0}; SVariant tmpVar = {0};
size_t t = 0; size_t t = 0;
...@@ -1065,47 +1092,53 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3 ...@@ -1065,47 +1092,53 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
switch (tType) { switch (tType) {
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
if (sType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(sType)) {
goto _return;
}
if (tmpVar.i64 > 1 ||tmpVar.i64 < 0) {
goto _return;
}
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: { case TSDB_DATA_TYPE_TINYINT: {
// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
// if (converted) { if (converted) {
// taosVariantDestroy(&tmpVar); tVariantDestroy(&tmpVar);
// memset(&tmpVar, 0, sizeof(tmpVar)); memset(&tmpVar, 0, sizeof(tmpVar));
// continue; continue;
// } }
//
// goto _return; goto _return;
// } }
pvar = &val; pvar = &val;
t = sizeof(val); t = sizeof(val);
break; break;
} }
case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: { case TSDB_DATA_TYPE_SMALLINT: {
// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
// if (converted) { if (converted) {
// taosVariantDestroy(&tmpVar); tVariantDestroy(&tmpVar);
// memset(&tmpVar, 0, sizeof(tmpVar)); memset(&tmpVar, 0, sizeof(tmpVar));
// continue; continue;
// } }
//
// goto _return; goto _return;
// } }
pvar = &val; pvar = &val;
t = sizeof(val); t = sizeof(val);
break; break;
} }
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_INT: { case TSDB_DATA_TYPE_INT: {
// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
// if (converted) { if (converted) {
// taosVariantDestroy(&tmpVar); tVariantDestroy(&tmpVar);
// memset(&tmpVar, 0, sizeof(tmpVar)); memset(&tmpVar, 0, sizeof(tmpVar));
// continue; continue;
// } }
//
// goto _return; goto _return;
// } }
pvar = &val; pvar = &val;
t = sizeof(val); t = sizeof(val);
break; break;
...@@ -1129,15 +1162,15 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3 ...@@ -1129,15 +1162,15 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
break; break;
} }
case TSDB_DATA_TYPE_FLOAT: { case TSDB_DATA_TYPE_FLOAT: {
// if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
// if (converted) { if (converted) {
// taosVariantDestroy(&tmpVar); tVariantDestroy(&tmpVar);
// memset(&tmpVar, 0, sizeof(tmpVar)); memset(&tmpVar, 0, sizeof(tmpVar));
// continue; continue;
// } }
// goto _return; goto _return;
// } }
pvar = &val; pvar = &val;
t = sizeof(val); t = sizeof(val);
break; break;
...@@ -1148,6 +1181,10 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3 ...@@ -1148,6 +1181,10 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
} }
t = varDataLen(tmp); t = varDataLen(tmp);
pvar = varDataVal(tmp); pvar = varDataVal(tmp);
if (tolower) {
strntolower_s(pvar, pvar, (int32_t)t);
}
break; break;
} }
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
...@@ -1178,22 +1215,51 @@ _return: ...@@ -1178,22 +1215,51 @@ _return:
return code; return code;
} }
static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pLeft) {
if((*pLeft)->nodeType == TSQL_NODE_EXPR && (*pLeft)->_node.optr == TSDB_RELATION_ARROW){ // json tag -> operation
assert(info->pTable != NULL);
SSchema* schema = (*pLeft)->_node.pLeft->pSchema;
if((*pLeft)->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
jsonKeyMd5((*pLeft)->_node.pRight->pVal->pz, (*pLeft)->_node.pRight->pVal->nLen, keyMd5);
memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
(*pLeft) = (*pLeft)->_node.pLeft; // -> operation use left as input
}else if(((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) &&
(tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL)){
SSchema* schema = (*pLeft)->pSchema;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
uint32_t nullData = TSDB_DATA_JSON_NULL;
jsonKeyMd5(&nullData, INT_BYTES, keyMd5);
memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
}else if(tree->_node.optr == TSDB_RELATION_CONTAINS){
SSchema* schema = (*pLeft)->pSchema;
if(tree->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
jsonKeyMd5(tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen, keyMd5);
memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
}
return TSDB_CODE_SUCCESS;
}
int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
tExprNode* pLeft = tree->_node.pLeft;
int32_t ret = TSDB_CODE_SUCCESS;
if((ret = filterDealJson(info, tree, &pLeft)) != TSDB_CODE_SUCCESS) return ret;
SFilterFieldId left = {0}, right = {0}; SFilterFieldId left = {0}, right = {0};
filterAddFieldFromNode(info, pLeft, &left);
filterAddFieldFromNode(info, tree->_node.pLeft, &left); uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
int32_t len = 0;
SVariant* var = tree->_node.pRight->pVal; uint32_t uidx = 0;
int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
size_t len = 0;
uint16_t uidx = 0;
if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) {
void *data = NULL; void *data = NULL;
filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); SVariant* var = tree->_node.pRight->pVal;
// CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type, false);
if (data == NULL) {
fltError("failed to convert in param");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (taosHashGetSize((SHashObj *)data) <= 0) { if (taosHashGetSize((SHashObj *)data) <= 0) {
filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx);
...@@ -1209,23 +1275,13 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g ...@@ -1209,23 +1275,13 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
void *p = taosHashIterate((SHashObj *)data, NULL); void *p = taosHashIterate((SHashObj *)data, NULL);
while(p) { while(p) {
void* key = NULL; void *key = taosHashGetDataKey((SHashObj *)data, p);
len = 0;
taosHashGetKey(p, &key, &len);
void *fdata = NULL; void *fdata = NULL;
if (IS_VAR_DATA_TYPE(type)) { fdata = malloc(sizeof(int64_t));
fdata = malloc(len + VARSTR_HEADER_SIZE); SIMPLE_COPY_VALUES(fdata, key);
varDataLen(fdata) = len; len = tDataTypes[type].bytes;
memcpy(varDataVal(fdata), key, len);
len += VARSTR_HEADER_SIZE;
} else {
fdata = malloc(sizeof(int64_t));
SIMPLE_COPY_VALUES(fdata, key);
len = tDataTypes[type].bytes;
}
filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true);
filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx);
...@@ -1240,9 +1296,10 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g ...@@ -1240,9 +1296,10 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
taosHashCleanup(data); taosHashCleanup(data);
} else { } else {
filterAddFieldFromNode(info, tree->_node.pRight, &right); filterAddFieldFromNode(info, tree->_node.pRight, &right);
filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); ret = filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);
CHK_LRET(ret != TSDB_CODE_SUCCESS, TSDB_CODE_QRY_APP_ERROR, "invalid where condition");
SFilterGroup fgroup = {0}; SFilterGroup fgroup = {0};
filterAddUnitToGroup(&fgroup, uidx); filterAddUnitToGroup(&fgroup, uidx);
...@@ -1254,7 +1311,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g ...@@ -1254,7 +1311,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
} }
int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) {
SFilterFieldId left, right, *pright = &right; SFilterFieldId left, right, *pright = &right;
int32_t type = FILTER_UNIT_DATA_TYPE(u); int32_t type = FILTER_UNIT_DATA_TYPE(u);
uint16_t flag = FLD_DESC_NO_FREE; uint16_t flag = FLD_DESC_NO_FREE;
...@@ -1267,7 +1324,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u ...@@ -1267,7 +1324,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
void *data = FILTER_UNIT_VAL_DATA(src, u); void *data = FILTER_UNIT_VAL_DATA(src, u);
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) {
filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right.
t = FILTER_GET_FIELD(dst, right); t = FILTER_GET_FIELD(dst, right);
...@@ -1289,7 +1346,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u ...@@ -1289,7 +1346,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx);
} }
int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) {
SFilterUnit *u = &info->units[uidx]; SFilterUnit *u = &info->units[uidx];
u->compare.optr2 = optr; u->compare.optr2 = optr;
...@@ -1299,9 +1356,9 @@ int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *righ ...@@ -1299,9 +1356,9 @@ int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *righ
} }
int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) {
SFilterFieldId left, right, right2; SFilterFieldId left, right, right2;
uint16_t uidx = 0; uint32_t uidx = 0;
SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); SFilterField *col = FILTER_GET_COL_FIELD(src, cidx);
...@@ -1479,61 +1536,80 @@ static void filterFreeGroup(void *pItem) { ...@@ -1479,61 +1536,80 @@ static void filterFreeGroup(void *pItem) {
} }
int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SArray* leftGroup = NULL; SArray* preGroup = NULL;
SArray* rightGroup = NULL; SArray* newGroup = NULL;
SArray* resGroup = NULL;
if (tree->nodeType != TEXPR_BINARYEXPR_NODE) { ENodeType nType = nodeType(nType);
//qError("invalid nodeType:%d", tree->nodeType); SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext;
return TSDB_CODE_QRY_APP_ERROR;
} if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) {
SLogicConditionNode *node = (SLogicConditionNode *)pNode;
if (tree->_node.optr == TSDB_RELATION_AND) { if (LOGIC_COND_TYPE_AND == node->condType) {
leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); SListCell *cell = node->pParameterList->pHead;
rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); for (int32_t i = 0; i < node->pParameterList->length; ++i) {
ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); newGroup = taosArrayInit(4, sizeof(SFilterGroup));
ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); resGroup = taosArrayInit(4, sizeof(SFilterGroup));
SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup};
nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx);
FLT_ERR_JRET(tctx.code);
FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup));
taosArrayDestroyEx(newGroup, filterFreeGroup);
taosArrayDestroyEx(preGroup, filterFreeGroup);
preGroup = resGroup;
}
ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); taosArrayAddAll(ctx->group, resGroup);
taosArrayDestroyEx(leftGroup, filterFreeGroup); taosArrayDestroyEx(newGroup, filterFreeGroup);
taosArrayDestroyEx(rightGroup, filterFreeGroup); taosArrayDestroyEx(preGroup, filterFreeGroup);
taosArrayDestroyEx(resGroup, filterFreeGroup);
return TSDB_CODE_SUCCESS;
} return DEAL_RES_IGNORE_CHILD;
}
if (LOGIC_COND_TYPE_OR == node->condType) {
SListCell *cell = node->pParameterList->pHead;
for (int32_t i = 0; i < node->pParameterList->length; ++i) {
nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext);
FLT_ERR_JRET(ctx.code);
}
return DEAL_RES_IGNORE_CHILD;
}
if (tree->_node.optr == TSDB_RELATION_OR) { fltError("invalid condition type, type:%d", node->condType);
ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group));
ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group));
return TSDB_CODE_SUCCESS; return DEAL_RES_ERROR;
} }
code = filterAddGroupUnitFromNode(info, tree, group); if (QUERY_NODE_OPERATOR == nType) {
FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group));
return DEAL_RES_CONTINUE;
}
sclError("invalid node type for filter, type:%d", nodeType(pNode));
code = TSDB_CODE_QRY_INVALID_INPUT;
_return: _return:
taosArrayDestroyEx(leftGroup, filterFreeGroup); taosArrayDestroyEx(newGroup, filterFreeGroup);
taosArrayDestroyEx(rightGroup, filterFreeGroup); taosArrayDestroyEx(preGroup, filterFreeGroup);
taosArrayDestroyEx(resGroup, filterFreeGroup);
return code;
}
#if 0 ctx->code = code;
int32_t filterInitUnitFunc(SFilterInfo *info) {
for (uint16_t i = 0; i < info->unitNum; ++i) {
SFilterUnit* unit = &info->units[i];
info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr);
}
return TSDB_CODE_SUCCESS; return DEAL_RES_ERROR;
} }
#endif
int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
int32_t n = 0; int32_t n = 0;
switch (type) { switch (type) {
...@@ -1611,34 +1687,37 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le ...@@ -1611,34 +1687,37 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le
void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) {
if (qDebugFlag & DEBUG_DEBUG) { if (qDebugFlag & DEBUG_DEBUG) {
// CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); if (info == NULL) {
fltDebug("%s - FilterInfo: EMPTY", msg);
return;
}
if (options == 0) { if (options == 0) {
// //qDebug("%s - FilterInfo:", msg); qDebug("%s - FilterInfo:", msg);
// //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema *sch = field->desc; SSchema *sch = field->desc;
// //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name);
} }
//qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num);
for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i];
if (field->desc) { if (field->desc) {
SVariant *var = field->desc; SVariant *var = field->desc;
if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
//qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1));
} else { } else {
//qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO
} }
} else if (field->data) { } else if (field->data) {
//qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO
} }
} }
//qDebug("UNIT Num:%u", info->unitNum); qDebug("UNIT Num:%u", info->unitNum);
for (uint16_t i = 0; i < info->unitNum; ++i) { for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i]; SFilterUnit *unit = &info->units[i];
int32_t type = FILTER_UNIT_DATA_TYPE(unit); int32_t type = FILTER_UNIT_DATA_TYPE(unit);
int32_t len = 0; int32_t len = 0;
...@@ -1647,7 +1726,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ...@@ -1647,7 +1726,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
SSchema *sch = left->desc; SSchema *sch = left->desc;
len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_CONTAINS){
len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str);
}
if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit);
...@@ -1656,7 +1737,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ...@@ -1656,7 +1737,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
tlen = varDataLen(data); tlen = varDataLen(data);
data += VARSTR_HEADER_SIZE; data += VARSTR_HEADER_SIZE;
} }
converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); if (data) fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen);
} else { } else {
strcat(str, "NULL"); strcat(str, "NULL");
} }
...@@ -1664,7 +1745,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ...@@ -1664,7 +1745,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
if (unit->compare.optr2) { if (unit->compare.optr2) {
strcat(str, " && "); strcat(str, " && ");
sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_CONTAINS){
sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str);
}
if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit);
...@@ -1673,23 +1756,23 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ...@@ -1673,23 +1756,23 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
tlen = varDataLen(data); tlen = varDataLen(data);
data += VARSTR_HEADER_SIZE; data += VARSTR_HEADER_SIZE;
} }
converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen);
} else { } else {
strcat(str, "NULL"); strcat(str, "NULL");
} }
strcat(str, "]"); strcat(str, "]");
} }
//qDebug("%s", str); //TODO qDebug("%s", str); //TODO
} }
//qDebug("GROUP Num:%u", info->groupNum); qDebug("GROUP Num:%u", info->groupNum);
for (uint16_t i = 0; i < info->groupNum; ++i) { for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup *group = &info->groups[i]; SFilterGroup *group = &info->groups[i];
//qDebug("Group%d : unit num[%u]", i, group->unitNum); qDebug("Group%d : unit num[%u]", i, group->unitNum);
for (uint16_t u = 0; u < group->unitNum; ++u) { for (uint32_t u = 0; u < group->unitNum; ++u) {
//qDebug("unit id:%u", group->unitIdxs[u]); qDebug("unit id:%u", group->unitIdxs[u]);
} }
} }
...@@ -1697,12 +1780,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ...@@ -1697,12 +1780,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
} }
if (options == 1) { if (options == 1) {
//qDebug("%s - RANGE info:", msg); qDebug("%s - RANGE info:", msg);
//qDebug("RANGE Num:%u", info->colRangeNum); qDebug("RANGE Num:%u", info->colRangeNum);
for (uint16_t i = 0; i < info->colRangeNum; ++i) { for (uint32_t i = 0; i < info->colRangeNum; ++i) {
SFilterRangeCtx *ctx = info->colRange[i]; SFilterRangeCtx *ctx = info->colRange[i];
//qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange);
if (ctx->isrange) { if (ctx->isrange) {
SFilterRangeNode *r = ctx->rs; SFilterRangeNode *r = ctx->rs;
while (r) { while (r) {
...@@ -1712,7 +1795,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ...@@ -1712,7 +1795,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
strcat(str,"(NULL)"); strcat(str,"(NULL)");
} else { } else {
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"[");
converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen);
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]");
} }
strcat(str, " - "); strcat(str, " - ");
...@@ -1720,10 +1803,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ...@@ -1720,10 +1803,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
strcat(str, "(NULL)"); strcat(str, "(NULL)");
} else { } else {
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"[");
converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen);
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]");
} }
//qDebug("range: %s", str); qDebug("range: %s", str);
r = r->next; r = r->next;
} }
...@@ -1733,25 +1816,25 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ...@@ -1733,25 +1816,25 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
return; return;
} }
//qDebug("%s - Block Filter info:", msg); qDebug("%s - Block Filter info:", msg);
if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) {
//qDebug("Flag:%s", "ALL"); qDebug("Flag:%s", "ALL");
return; return;
} else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) {
//qDebug("Flag:%s", "EMPTY"); qDebug("Flag:%s", "EMPTY");
return; return;
} else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){
//qDebug("Flag:%s", "ACTIVE"); qDebug("Flag:%s", "ACTIVE");
} }
//qDebug("GroupNum:%d", info->blkGroupNum); qDebug("GroupNum:%d", info->blkGroupNum);
uint16_t *unitIdx = info->blkUnits; uint32_t *unitIdx = info->blkUnits;
for (uint16_t i = 0; i < info->blkGroupNum; ++i) { for (uint32_t i = 0; i < info->blkGroupNum; ++i) {
//qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); qDebug("Group[%d] UnitNum: %d:", i, *unitIdx);
uint16_t unitNum = *(unitIdx++); uint32_t unitNum = *(unitIdx++);
for (uint16_t m = 0; m < unitNum; ++m) { for (uint32_t m = 0; m < unitNum; ++m) {
//qDebug("uidx[%d]", *(unitIdx++)); qDebug("uidx[%d]", *(unitIdx++));
} }
} }
} }
...@@ -1837,21 +1920,23 @@ void filterFreePCtx(SFilterPCtx *pctx) { ...@@ -1837,21 +1920,23 @@ void filterFreePCtx(SFilterPCtx *pctx) {
} }
void filterFreeInfo(SFilterInfo *info) { void filterFreeInfo(SFilterInfo *info) {
CHK_RETV(info == NULL); if (info == NULL) {
return;
}
tfree(info->cunits); tfree(info->cunits);
tfree(info->blkUnitRes); tfree(info->blkUnitRes);
tfree(info->blkUnits); tfree(info->blkUnits);
for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) {
for (uint16_t f = 0; f < info->fields[i].num; ++f) { for (uint32_t f = 0; f < info->fields[i].num; ++f) {
filterFreeField(&info->fields[i].fields[f], i); filterFreeField(&info->fields[i].fields[f], i);
} }
tfree(info->fields[i].fields); tfree(info->fields[i].fields);
} }
for (int32_t i = 0; i < info->groupNum; ++i) { for (uint32_t i = 0; i < info->groupNum; ++i) {
filterFreeGroup(&info->groups[i]); filterFreeGroup(&info->groups[i]);
} }
...@@ -1863,7 +1948,7 @@ void filterFreeInfo(SFilterInfo *info) { ...@@ -1863,7 +1948,7 @@ void filterFreeInfo(SFilterInfo *info) {
tfree(info->unitFlags); tfree(info->unitFlags);
for (uint16_t i = 0; i < info->colRangeNum; ++i) { for (uint32_t i = 0; i < info->colRangeNum; ++i) {
filterFreeRangeCtx(info->colRange[i]); filterFreeRangeCtx(info->colRange[i]);
} }
...@@ -1901,7 +1986,7 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { ...@@ -1901,7 +1986,7 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) {
int32_t filterInitValFieldData(SFilterInfo *info) { int32_t filterInitValFieldData(SFilterInfo *info) {
for (uint16_t i = 0; i < info->unitNum; ++i) { for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit* unit = &info->units[i]; SFilterUnit* unit = &info->units[i];
if (unit->right.type != FLD_TYPE_VALUE) { if (unit->right.type != FLD_TYPE_VALUE) {
assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR);
...@@ -1923,8 +2008,11 @@ int32_t filterInitValFieldData(SFilterInfo *info) { ...@@ -1923,8 +2008,11 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
} }
if (unit->compare.optr == TSDB_RELATION_IN) { if (unit->compare.optr == TSDB_RELATION_IN) {
filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type, false);
// CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); if (fi->data == NULL) {
fltError("failed to convert in param");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH);
...@@ -1937,7 +2025,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) { ...@@ -1937,7 +2025,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
} else if (type == TSDB_DATA_TYPE_NCHAR) { } else if (type == TSDB_DATA_TYPE_NCHAR) {
size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;
fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
} else { } else if (type != TSDB_DATA_TYPE_JSON){
if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE
fi->data = calloc(var->nLen, tDataTypes[type].bytes); fi->data = calloc(var->nLen, tDataTypes[type].bytes);
for (int32_t a = 0; a < var->nLen; ++a) { for (int32_t a = 0; a < var->nLen; ++a) {
...@@ -1949,19 +2037,47 @@ int32_t filterInitValFieldData(SFilterInfo *info) { ...@@ -1949,19 +2037,47 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
} else { } else {
fi->data = calloc(1, sizeof(int64_t)); fi->data = calloc(1, sizeof(int64_t));
} }
} else{ // type == TSDB_DATA_TYPE_JSON
// fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc)
}
if(type != TSDB_DATA_TYPE_JSON){
bool converted = false;
char extInfo = 0;
if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) {
if (converted) {
filterHandleValueExtInfo(unit, extInfo);
continue;
}
qError("dump value to type[%d] failed", type);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
}
// match/nmatch for nchar type need convert from ucs4 to mbs
if(type == TSDB_DATA_TYPE_NCHAR &&
(unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
if (len < 0){
qError("filterInitValFieldData taosUcs4ToMbs error 1");
return TSDB_CODE_FAILED;
}
varDataSetLen(newValData, len);
varDataCopy(fi->data, newValData);
}else if(type == TSDB_DATA_TYPE_JSON &&
(unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
int32_t len = taosUcs4ToMbs(((tVariant*)(fi->desc))->pz, ((tVariant*)(fi->desc))->nLen, newValData);
if (len < 0){
qError("filterInitValFieldData taosUcs4ToMbs error 2");
return TSDB_CODE_FAILED;
}
memcpy(((tVariant*)(fi->desc))->pz, newValData, len);
((tVariant*)(fi->desc))->nLen = len;
} }
bool converted = false;
char extInfo = 0;
// if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) {
// if (converted) {
// filterHandleValueExtInfo(unit, extInfo);
//
// continue;
// }
// //qError("dump value to type[%d] failed", type);
// return TSDB_CODE_TSC_INVALID_OPERATION;
// }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
...@@ -1971,6 +2087,8 @@ int32_t filterInitValFieldData(SFilterInfo *info) { ...@@ -1971,6 +2087,8 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) { bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) {
int32_t ret = func(left, right); int32_t ret = func(left, right);
if(ret == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
switch (optr) { switch (optr) {
case TSDB_RELATION_EQUAL: { case TSDB_RELATION_EQUAL: {
return ret == 0; return ret == 0;
...@@ -1996,6 +2114,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) ...@@ -1996,6 +2114,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right)
case TSDB_RELATION_MATCH: { case TSDB_RELATION_MATCH: {
return ret == 0; return ret == 0;
} }
case TSDB_RELATION_CONTAINS: {
return ret == 0;
}
case TSDB_RELATION_NMATCH: { case TSDB_RELATION_NMATCH: {
return ret == 0; return ret == 0;
} }
...@@ -2092,7 +2213,7 @@ _return: ...@@ -2092,7 +2213,7 @@ _return:
} }
int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colIdx, bool *empty) {
SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; SArray* colArray = (SArray *)gRes->colInfo[colIdx].info;
int32_t size = (int32_t)taosArrayGetSize(colArray); int32_t size = (int32_t)taosArrayGetSize(colArray);
int32_t type = gRes->colInfo[colIdx].dataType; int32_t type = gRes->colInfo[colIdx].dataType;
...@@ -2109,6 +2230,10 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colI ...@@ -2109,6 +2230,10 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colI
filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND);
CHK_JMP(MR_EMPTY_RES(ctx)); CHK_JMP(MR_EMPTY_RES(ctx));
} }
if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){
gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL;
SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u));
}
} }
taosArrayDestroy(colArray); taosArrayDestroy(colArray);
...@@ -2129,11 +2254,11 @@ _return: ...@@ -2129,11 +2254,11 @@ _return:
int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) {
bool empty = false; bool empty = false;
uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t));
uint16_t colIdxi = 0; uint32_t colIdxi = 0;
uint16_t gResIdx = 0; uint32_t gResIdx = 0;
for (uint16_t i = 0; i < info->groupNum; ++i) { for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup* g = info->groups + i; SFilterGroup* g = info->groups + i;
gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx));
...@@ -2141,9 +2266,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t ...@@ -2141,9 +2266,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
colIdxi = 0; colIdxi = 0;
empty = false; empty = false;
for (uint16_t j = 0; j < g->unitNum; ++j) { for (uint32_t j = 0; j < g->unitNum; ++j) {
SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j);
uint16_t cidx = FILTER_UNIT_COL_IDX(u); uint32_t cidx = FILTER_UNIT_COL_IDX(u);
if (gRes[gResIdx]->colInfo[cidx].info == NULL) { if (gRes[gResIdx]->colInfo[cidx].info == NULL) {
gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES);
...@@ -2159,10 +2284,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t ...@@ -2159,10 +2284,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
} }
if (colIdxi > 1) { if (colIdxi > 1) {
qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
} }
for (uint16_t l = 0; l < colIdxi; ++l) { for (uint32_t l = 0; l < colIdxi; ++l) {
int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType;
if (FILTER_NO_MERGE_DATA_TYPE(type)) { if (FILTER_NO_MERGE_DATA_TYPE(type)) {
...@@ -2201,7 +2326,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t ...@@ -2201,7 +2326,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
} }
void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) {
uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
bool equal = false; bool equal = false;
for (; m < gRes1->colNum; ++m) { for (; m < gRes1->colNum; ++m) {
...@@ -2224,6 +2349,15 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool ...@@ -2224,6 +2349,15 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool
*conflict = true; *conflict = true;
return; return;
} }
// for long in operation
if (gRes1->colInfo[idx1].optr == TSDB_RELATION_EQUAL && gRes2->colInfo[idx2].optr == TSDB_RELATION_EQUAL) {
SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info;
if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){
*conflict = true;
return;
}
}
++n; ++n;
equal = true; equal = true;
...@@ -2241,7 +2375,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool ...@@ -2241,7 +2375,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool
} }
int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint32_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) {
SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx);
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
...@@ -2271,10 +2405,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ...@@ -2271,10 +2405,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter
FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE);
uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum;
bool equal = false; bool equal = false;
uint16_t equal1 = 0, equal2 = 0, merNum = 0; uint32_t equal1 = 0, equal2 = 0, merNum = 0;
SFilterRangeCtx *ctx = NULL; SFilterRangeCtx *ctx = NULL;
SFilterColCtx colCtx = {0}; SFilterColCtx colCtx = {0};
SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx));
...@@ -2388,7 +2522,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR ...@@ -2388,7 +2522,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx);
int32_t pEnd = 0, cStart = 0, cEnd = 0; int32_t pEnd = 0, cStart = 0, cEnd = 0;
uint16_t pColNum = 0, cColNum = 0; uint32_t pColNum = 0, cColNum = 0;
int32_t movedNum = 0; int32_t movedNum = 0;
bool all = false; bool all = false;
...@@ -2454,7 +2588,6 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR ...@@ -2454,7 +2588,6 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
} }
cStart = i; cStart = i;
cEnd = i;
cColNum = gRes[i]->colNum; cColNum = gRes[i]->colNum;
} }
...@@ -2470,7 +2603,7 @@ _return: ...@@ -2470,7 +2603,7 @@ _return:
int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) {
size_t groupSize = taosArrayGetSize(group); size_t groupSize = taosArrayGetSize(group);
info->groupNum = (uint16_t)groupSize; info->groupNum = (uint32_t)groupSize;
if (info->groupNum > 0) { if (info->groupNum > 0) {
info->groups = calloc(info->groupNum, sizeof(*info->groups)); info->groups = calloc(info->groupNum, sizeof(*info->groups));
...@@ -2487,7 +2620,7 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { ...@@ -2487,7 +2620,7 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) {
int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) {
if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) {
//qDebug("no need rewrite"); qDebug("no need rewrite");
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
...@@ -2499,7 +2632,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum ...@@ -2499,7 +2632,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
SFilterGroupCtx *res = NULL; SFilterGroupCtx *res = NULL;
SFilterColInfo *colInfo = NULL; SFilterColInfo *colInfo = NULL;
int32_t optr = 0; int32_t optr = 0;
uint16_t uidx = 0; uint32_t uidx = 0;
memset(info, 0, sizeof(*info)); memset(info, 0, sizeof(*info));
...@@ -2519,7 +2652,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum ...@@ -2519,7 +2652,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
SFilterGroup ng = {0}; SFilterGroup ng = {0};
for (uint16_t m = 0; m < res->colNum; ++m) { for (uint32_t m = 0; m < res->colNum; ++m) {
colInfo = &res->colInfo[res->colIdx[m]]; colInfo = &res->colInfo[res->colIdx[m]];
if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) {
assert(colInfo->type == RANGE_TYPE_UNIT); assert(colInfo->type == RANGE_TYPE_UNIT);
...@@ -2555,13 +2688,13 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum ...@@ -2555,13 +2688,13 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
} }
int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) {
uint16_t *idxs = NULL; uint32_t *idxs = NULL;
uint16_t colNum = 0; uint32_t colNum = 0;
SFilterGroupCtx *res = NULL; SFilterGroupCtx *res = NULL;
uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum));
for (int32_t i = 0; i < gResNum; ++i) { for (int32_t i = 0; i < gResNum; ++i) {
for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { for (uint32_t m = 0; m < gRes[i]->colNum; ++m) {
SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]];
if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) {
continue; continue;
...@@ -2571,7 +2704,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ ...@@ -2571,7 +2704,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
} }
} }
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
if (idxNum[i] < gResNum) { if (idxNum[i] < gResNum) {
continue; continue;
} }
...@@ -2592,9 +2725,9 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ ...@@ -2592,9 +2725,9 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
for (int32_t i = 0; i < gResNum; ++i) { for (int32_t i = 0; i < gResNum; ++i) {
res = gRes[i]; res = gRes[i];
uint16_t n = 0; uint32_t n = 0;
for (uint16_t m = 0; m < info->colRangeNum; ++m) { for (uint32_t m = 0; m < info->colRangeNum; ++m) {
for (; n < res->colNum; ++n) { for (; n < res->colNum; ++n) {
if (res->colIdx[n] < idxs[m]) { if (res->colIdx[n] < idxs[m]) {
continue; continue;
...@@ -2642,7 +2775,7 @@ _return: ...@@ -2642,7 +2775,7 @@ _return:
} }
int32_t filterPostProcessRange(SFilterInfo *info) { int32_t filterPostProcessRange(SFilterInfo *info) {
for (uint16_t i = 0; i < info->colRangeNum; ++i) { for (uint32_t i = 0; i < info->colRangeNum; ++i) {
SFilterRangeCtx* ctx = info->colRange[i]; SFilterRangeCtx* ctx = info->colRange[i];
SFilterRangeNode *r = ctx->rs; SFilterRangeNode *r = ctx->rs;
while (r) { while (r) {
...@@ -2656,13 +2789,11 @@ int32_t filterPostProcessRange(SFilterInfo *info) { ...@@ -2656,13 +2789,11 @@ int32_t filterPostProcessRange(SFilterInfo *info) {
int32_t filterGenerateComInfo(SFilterInfo *info) { int32_t filterGenerateComInfo(SFilterInfo *info) {
uint16_t n = 0;
info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); info->cunits = malloc(info->unitNum * sizeof(*info->cunits));
info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum);
info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum);
for (uint16_t i = 0; i < info->unitNum; ++i) { for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i]; SFilterUnit *unit = &info->units[i];
info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr);
...@@ -2672,7 +2803,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { ...@@ -2672,7 +2803,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
if (unit->right.type == FLD_TYPE_VALUE) { if (unit->right.type == FLD_TYPE_VALUE) {
info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant
info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit);
}else{
info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
}
} else { } else {
info->cunits[i].valData = NULL; info->cunits[i].valData = NULL;
} }
...@@ -2685,30 +2820,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { ...@@ -2685,30 +2820,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit);
info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit);
} }
uint16_t cgroupNum = info->groupNum + 1;
for (uint16_t i = 0; i < info->groupNum; ++i) {
cgroupNum += info->groups[i].unitNum;
}
info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups));
for (uint16_t i = 0; i < info->groupNum; ++i) {
info->cgroups[n++] = info->groups[i].unitNum;
for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) {
info->cgroups[n++] = info->groups[i].unitIdxs[m];
}
}
info->cgroups[n] = 0;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t filterUpdateComUnits(SFilterInfo *info) { int32_t filterUpdateComUnits(SFilterInfo *info) {
for (uint16_t i = 0; i < info->unitNum; ++i) { for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i]; SFilterUnit *unit = &info->units[i];
info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
...@@ -2723,7 +2840,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 ...@@ -2723,7 +2840,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum);
for (int32_t k = 0; k < info->unitNum; ++k) { for (uint32_t k = 0; k < info->unitNum; ++k) {
int32_t index = -1; int32_t index = -1;
SFilterComUnit *cunit = &info->cunits[k]; SFilterComUnit *cunit = &info->cunits[k];
...@@ -2768,18 +2885,20 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 ...@@ -2768,18 +2885,20 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
} }
} }
if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL
|| cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH
|| cunit->optr == TSDB_RELATION_NOT_EQUAL) { || cunit->optr == TSDB_RELATION_NOT_EQUAL || cunit->optr == TSDB_RELATION_CONTAINS) {
continue; continue;
} }
SColumnDataAgg* pDataBlockst = &pDataStatis[index]; SColumnDataAgg* pDataBlockst = &pDataStatis[index];
void *minVal, *maxVal; void *minVal, *maxVal;
float minv = 0;
float maxv = 0;
if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) {
float minv = (float)(*(double *)(&pDataBlockst->min)); minv = (float)(*(double *)(&pDataBlockst->min));
float maxv = (float)(*(double *)(&pDataBlockst->max)); maxv = (float)(*(double *)(&pDataBlockst->max));
minVal = &minv; minVal = &minv;
maxVal = &maxv; maxVal = &maxv;
...@@ -2834,12 +2953,15 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 ...@@ -2834,12 +2953,15 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
} }
// CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); if (rmUnit == 0) {
fltDebug("NO Block Filter APPLY");
FLT_RET(TSDB_CODE_SUCCESS);
}
info->blkGroupNum = info->groupNum; info->blkGroupNum = info->groupNum;
uint16_t *unitNum = info->blkUnits; uint32_t *unitNum = info->blkUnits;
uint16_t *unitIdx = unitNum + 1; uint32_t *unitIdx = unitNum + 1;
int32_t all = 0, empty = 0; int32_t all = 0, empty = 0;
for (uint32_t g = 0; g < info->groupNum; ++g) { for (uint32_t g = 0; g < info->groupNum; ++g) {
...@@ -2849,7 +2971,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 ...@@ -2849,7 +2971,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
empty = 0; empty = 0;
for (uint32_t u = 0; u < group->unitNum; ++u) { for (uint32_t u = 0; u < group->unitNum; ++u) {
uint16_t uidx = group->unitIdxs[u]; uint32_t uidx = group->unitIdxs[u];
if (info->blkUnitRes[uidx] == 1) { if (info->blkUnitRes[uidx] == 1) {
--(*unitNum); --(*unitNum);
all = 1; all = 1;
...@@ -2896,17 +3018,19 @@ _return: ...@@ -2896,17 +3018,19 @@ _return:
bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
SFilterInfo *info = (SFilterInfo *)pinfo; SFilterInfo *info = (SFilterInfo *)pinfo;
bool all = true; bool all = true;
uint16_t *unitIdx = NULL; uint32_t *unitIdx = NULL;
*p = calloc(numOfRows, sizeof(int8_t));
if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
//FILTER_UNIT_CLR_F(info); //FILTER_UNIT_CLR_F(info);
unitIdx = info->blkUnits; unitIdx = info->blkUnits;
for (uint32_t g = 0; g < info->blkGroupNum; ++g) { for (uint32_t g = 0; g < info->blkGroupNum; ++g) {
uint16_t unitNum = *(unitIdx++); uint32_t unitNum = *(unitIdx++);
for (uint32_t u = 0; u < unitNum; ++u) { for (uint32_t u = 0; u < unitNum; ++u) {
SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)];
void *colData = (char *)cunit->colData + cunit->dataSize * i; void *colData = (char *)cunit->colData + cunit->dataSize * i;
...@@ -3001,12 +3125,25 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, ...@@ -3001,12 +3125,25 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
return all; return all;
} }
*p = calloc(numOfRows, sizeof(int8_t)); if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
uint16_t uidx = info->groups[0].unitIdxs[0]; uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
(*p)[i] = isNull(colData, info->cunits[uidx].dataType); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
if (!colData){ // for json->'key' is null
(*p)[i] = 1;
}else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null
colData = POINTER_SHIFT(colData, CHAR_BYTES);
(*p)[i] = isNull(colData, info->cunits[uidx].dataType);
}else{
(*p)[i] = 0;
}
}else{
(*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType));
}
if ((*p)[i] == 0) { if ((*p)[i] == 0) {
all = false; all = false;
} }
...@@ -3022,12 +3159,27 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows ...@@ -3022,12 +3159,27 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
return all; return all;
} }
*p = calloc(numOfRows, sizeof(int8_t)); if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
uint16_t uidx = info->groups[0].unitIdxs[0]; uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
(*p)[i] = !isNull(colData, info->cunits[uidx].dataType);
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
if (!colData) { // for json->'key' is not null
(*p)[i] = 0;
}else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null
colData = POINTER_SHIFT(colData, CHAR_BYTES);
(*p)[i] = !isNull(colData, info->cunits[uidx].dataType);
}else{ // for json->'key' is not null
(*p)[i] = 1;
}
}else {
(*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType));
}
if ((*p)[i] == 0) { if ((*p)[i] == 0) {
all = false; all = false;
} }
...@@ -3036,7 +3188,48 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows ...@@ -3036,7 +3188,48 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
return all; return all;
} }
bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { static void doJsonCompare(SFilterComUnit *cunit, int8_t *result, void* colData){
if(cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH){
uint8_t jsonType = *(char*)colData;
char* realData = POINTER_SHIFT(colData, CHAR_BYTES);
if (jsonType != TSDB_DATA_TYPE_NCHAR){
*result = false;
}else{
char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData));
if (len < 0){
qError("castConvert1 taosUcs4ToMbs error");
tfree(newColData);
return;
}
varDataSetLen(newColData, len);
tVariant* val = cunit->valData;
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE);
memcpy(varDataVal(newValData), val->pz, val->nLen);
varDataSetLen(newValData, val->nLen);
*result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, newValData);
tfree(newColData);
}
}else if(cunit->optr == TSDB_RELATION_LIKE){
uint8_t jsonType = *(char*)colData;
char* realData = POINTER_SHIFT(colData, CHAR_BYTES);
if (jsonType != TSDB_DATA_TYPE_NCHAR){
*result = false;
}else{
tVariant* val = cunit->valData;
char* newValData = calloc(val->nLen + VARSTR_HEADER_SIZE, 1);
memcpy(varDataVal(newValData), val->pz, val->nLen);
varDataSetLen(newValData, val->nLen);
*result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, realData, newValData);
tfree(newValData);
}
}else{
*result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
}
}
bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
SFilterInfo *info = (SFilterInfo *)pinfo; SFilterInfo *info = (SFilterInfo *)pinfo;
bool all = true; bool all = true;
uint16_t dataSize = info->cunits[0].dataSize; uint16_t dataSize = info->cunits[0].dataSize;
...@@ -3050,10 +3243,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD ...@@ -3050,10 +3243,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD
return all; return all;
} }
*p = calloc(numOfRows, sizeof(int8_t)); if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
if (isNull(colData, info->cunits[0].dataType)) { if (colData == NULL || isNull(colData, info->cunits[0].dataType)) {
all = false; all = false;
colData += dataSize; colData += dataSize;
continue; continue;
...@@ -3078,19 +3273,37 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa ...@@ -3078,19 +3273,37 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa
if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
return all; return all;
} }
*p = calloc(numOfRows, sizeof(int8_t)); if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
uint16_t uidx = info->groups[0].unitIdxs[0]; uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
if (isNull(colData, info->cunits[uidx].dataType)) { if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) {
(*p)[i] = 0;
all = false; all = false;
continue; continue;
} }
// match/nmatch for nchar type need convert from ucs4 to mbs
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){
char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
if (len < 0){
qError("castConvert1 taosUcs4ToMbs error");
}else{
varDataSetLen(newColData, len);
(*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData);
}
tfree(newColData);
}else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
doJsonCompare(&(info->cunits[uidx]), &(*p)[i], colData);
}else{
(*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData);
}
(*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData);
if ((*p)[i] == 0) { if ((*p)[i] == 0) {
all = false; all = false;
} }
...@@ -3108,15 +3321,17 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg ...@@ -3108,15 +3321,17 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
return all; return all;
} }
*p = calloc(numOfRows, sizeof(int8_t)); if (*p == NULL) {
*p = calloc(numOfRows, sizeof(int8_t));
}
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
//FILTER_UNIT_CLR_F(info); //FILTER_UNIT_CLR_F(info);
for (uint32_t g = 0; g < info->groupNum; ++g) { for (uint32_t g = 0; g < info->groupNum; ++g) {
SFilterGroup *group = &info->groups[g]; SFilterGroup *group = &info->groups[g];
for (uint32_t u = 0; u < group->unitNum; ++u) { for (uint32_t u = 0; u < group->unitNum; ++u) {
uint16_t uidx = group->unitIdxs[u]; uint32_t uidx = group->unitIdxs[u];
SFilterComUnit *cunit = &info->cunits[uidx]; SFilterComUnit *cunit = &info->cunits[uidx];
void *colData = (char *)cunit->colData + cunit->dataSize * i; void *colData = (char *)cunit->colData + cunit->dataSize * i;
...@@ -3125,7 +3340,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg ...@@ -3125,7 +3340,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
//} else { //} else {
uint8_t optr = cunit->optr; uint8_t optr = cunit->optr;
if (isNull(colData, cunit->dataType)) { if (colData == NULL || isNull(colData, cunit->dataType)) {
(*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false;
} else { } else {
if (optr == TSDB_RELATION_NOTNULL) { if (optr == TSDB_RELATION_NOTNULL) {
...@@ -3135,7 +3350,21 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg ...@@ -3135,7 +3350,21 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
} else if (cunit->rfunc >= 0) { } else if (cunit->rfunc >= 0) {
(*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]);
} else { } else {
(*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){
char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
if (len < 0){
qError("castConvert1 taosUcs4ToMbs error");
}else{
varDataSetLen(newColData, len);
(*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData);
}
tfree(newColData);
}else if(cunit->dataType == TSDB_DATA_TYPE_JSON){
doJsonCompare(cunit, &(*p)[i], colData);
}else{
(*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
}
} }
//FILTER_UNIT_SET_R(info, uidx, p[i]); //FILTER_UNIT_SET_R(info, uidx, p[i]);
...@@ -3211,13 +3440,13 @@ int32_t filterPreprocess(SFilterInfo *info) { ...@@ -3211,13 +3440,13 @@ int32_t filterPreprocess(SFilterInfo *info) {
filterMergeGroups(info, gRes, &gResNum); filterMergeGroups(info, gRes, &gResNum);
if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) {
// qInfo("Final - FilterInfo: [ALL]"); qInfo("Final - FilterInfo: [ALL]");
goto _return; goto _return;
} }
if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) {
// qInfo("Final - FilterInfo: [EMPTY]"); qInfo("Final - FilterInfo: [EMPTY]");
goto _return; goto _return;
} }
...@@ -3244,26 +3473,19 @@ _return: ...@@ -3244,26 +3473,19 @@ _return:
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
// CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
// CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema* sch = fi->desc; SSchema* sch = fi->desc;
for (int32_t j = 0; j < numOfCols; ++j) { (*fp)(param, sch->colId, &fi->data);
SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j);
if (sch->colId == pColInfo->info.colId) {
fi->data = pColInfo->pData;
break;
}
}
} }
filterUpdateComUnits(info); filterUpdateComUnits(info);
...@@ -3271,46 +3493,52 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat ...@@ -3271,46 +3493,52 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp) {
CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
int32_t code = TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
SFilterInfo *info = NULL; }
// CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param");
if (*pinfo == NULL) { for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
*pinfo = calloc(1, sizeof(SFilterInfo)); SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema* sch = fi->desc;
(*fp)(param, sch->colId, sch->name, &fi->data);
} }
info = *pinfo; filterUpdateComUnits(info);
info->options = options; return TSDB_CODE_SUCCESS;
}
int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup));
filterInitUnitsFields(info); filterInitUnitsFields(info);
code = filterTreeToGroup(tree, info, group); SFltBuildGroupCtx tctx = {.info = info, .group = group};
nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx);
ERR_JRET(code); FLT_ERR_JRET(tctx.code);
filterConvertGroupFromArray(info, group); filterConvertGroupFromArray(info, group);
taosArrayDestroy(group);
ERR_JRET(filterInitValFieldData(info)); FLT_ERR_JRET(filterInitValFieldData(info));
if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) {
filterDumpInfoToString(info, "Before preprocess", 0); filterDumpInfoToString(info, "Before preprocess", 0);
ERR_JRET(filterPreprocess(info)); FLT_ERR_JRET(filterPreprocess(info));
CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); FLT_CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL));
if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) {
taosArrayDestroy(group);
return code; return code;
} }
//ERR_JRET(filterInitUnitFunc(info));
} }
info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes));
...@@ -3318,18 +3546,12 @@ int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t option ...@@ -3318,18 +3546,12 @@ int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t option
filterDumpInfoToString(info, "Final", 0); filterDumpInfoToString(info, "Final", 0);
taosArrayDestroy(group);
return code; return code;
_return: _return:
// qInfo("No filter, code:%d", code);
taosArrayDestroy(group);
filterFreeInfo(*pinfo);
*pinfo = NULL;
qInfo("No filter, code:%d", code);
return code; return code;
} }
...@@ -3348,7 +3570,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t ...@@ -3348,7 +3570,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
bool ret = true; bool ret = true;
void *minVal, *maxVal; void *minVal, *maxVal;
for (int32_t k = 0; k < info->colRangeNum; ++k) { for (uint32_t k = 0; k < info->colRangeNum; ++k) {
int32_t index = -1; int32_t index = -1;
SFilterRangeCtx *ctx = info->colRange[k]; SFilterRangeCtx *ctx = info->colRange[k];
for(int32_t i = 0; i < numOfCols; ++i) { for(int32_t i = 0; i < numOfCols; ++i) {
...@@ -3368,30 +3590,35 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t ...@@ -3368,30 +3590,35 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
break; break;
} }
if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { if (pDataStatis[index].numOfNull <= 0) {
ret = false; if (ctx->isnull && !ctx->notnull && !ctx->isrange) {
break;
}
// all data in current column are NULL, no need to check its boundary value
if (pDataStatis[index].numOfNull == numOfRows) {
// if isNULL query exists, load the null data column
if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) {
ret = false; ret = false;
break; break;
} }
} else if (pDataStatis[index].numOfNull > 0) {
if (pDataStatis[index].numOfNull == numOfRows) {
if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) {
ret = false;
break;
}
continue; continue;
} else {
if (ctx->isnull) {
continue;
}
}
} }
SColumnDataAgg* pDataBlockst = &pDataStatis[index]; SColumnDataAgg* pDataBlockst = &pDataStatis[index];
SFilterRangeNode *r = ctx->rs; SFilterRangeNode *r = ctx->rs;
float minv = 0;
float maxv = 0;
if (ctx->type == TSDB_DATA_TYPE_FLOAT) { if (ctx->type == TSDB_DATA_TYPE_FLOAT) {
float minv = (float)(*(double *)(&pDataBlockst->min)); minv = (float)(*(double *)(&pDataBlockst->min));
float maxv = (float)(*(double *)(&pDataBlockst->max)); maxv = (float)(*(double *)(&pDataBlockst->max));
minVal = &minv; minVal = &minv;
maxVal = &maxv; maxVal = &maxv;
...@@ -3408,7 +3635,9 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t ...@@ -3408,7 +3635,9 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
r = r->next; r = r->next;
} }
CHK_RET(!ret, ret); if (!ret) {
return ret;
}
} }
return ret; return ret;
...@@ -3426,7 +3655,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { ...@@ -3426,7 +3655,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
int32_t code = 0; int32_t code = 0;
bool empty = false, all = false; bool empty = false, all = false;
for (int32_t i = 0; i < info->groupNum; ++i) { for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup *group = &info->groups[i]; SFilterGroup *group = &info->groups[i];
if (group->unitNum > 1) { if (group->unitNum > 1) {
cur = tmpc; cur = tmpc;
...@@ -3436,8 +3665,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { ...@@ -3436,8 +3665,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
optr = TSDB_RELATION_OR; optr = TSDB_RELATION_OR;
} }
for (int32_t u = 0; u < group->unitNum; ++u) { for (uint32_t u = 0; u < group->unitNum; ++u) {
uint16_t uidx = group->unitIdxs[u]; uint32_t uidx = group->unitIdxs[u];
SFilterUnit *unit = &info->units[uidx]; SFilterUnit *unit = &info->units[uidx];
uint8_t raOptr = FILTER_UNIT_OPTR(unit); uint8_t raOptr = FILTER_UNIT_OPTR(unit);
...@@ -3478,8 +3707,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { ...@@ -3478,8 +3707,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
} else { } else {
filterGetRangeNum(prev, &num); filterGetRangeNum(prev, &num);
if (num > 1) { if (num > 1) {
//qError("only one time range accepted, num:%d", num); qError("only one time range accepted, num:%d", num);
ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); FLT_ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION);
} }
CHK_JMP(num < 1); CHK_JMP(num < 1);
...@@ -3493,7 +3722,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { ...@@ -3493,7 +3722,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
filterFreeRangeCtx(prev); filterFreeRangeCtx(prev);
filterFreeRangeCtx(tmpc); filterFreeRangeCtx(tmpc);
//qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
...@@ -3503,14 +3732,18 @@ _return: ...@@ -3503,14 +3732,18 @@ _return:
filterFreeRangeCtx(prev); filterFreeRangeCtx(prev);
filterFreeRangeCtx(tmpc); filterFreeRangeCtx(tmpc);
//qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
return code; return code;
} }
int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) {
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { if (FILTER_EMPTY_RES(info) || FILTER_ALL_RES(info)) {
return TSDB_CODE_SUCCESS;
}
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
if (type == TSDB_DATA_TYPE_NCHAR) { if (type == TSDB_DATA_TYPE_NCHAR) {
...@@ -3523,7 +3756,20 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar ...@@ -3523,7 +3756,20 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
char *src = FILTER_GET_COL_FIELD_DATA(fi, j); char *src = FILTER_GET_COL_FIELD_DATA(fi, j);
char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j);
int32_t len = 0; int32_t len = 0;
taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); char *varSrc = varDataVal(src);
size_t k = 0, varSrcLen = varDataLen(src);
while (k < varSrcLen && varSrc[k++] == -1) {}
if (k == varSrcLen) {
/* NULL */
varDataLen(dst) = (VarDataLenT) varSrcLen;
varDataCopy(dst, src);
continue;
}
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
if(!ret) {
qError("filterConverNcharColumns taosMbsToUcs4 error");
return TSDB_CODE_FAILED;
}
varDataLen(dst) = len; varDataLen(dst) = len;
} }
...@@ -3541,7 +3787,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar ...@@ -3541,7 +3787,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
} }
int32_t filterFreeNcharColumns(SFilterInfo* info) { int32_t filterFreeNcharColumns(SFilterInfo* info) {
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
if (type == TSDB_DATA_TYPE_NCHAR) { if (type == TSDB_DATA_TYPE_NCHAR) {
...@@ -3552,7 +3798,74 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { ...@@ -3552,7 +3798,74 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
##################################################################################
EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
SFltTreeStat *stat = (SFltTreeStat *)pContext;
}
int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat);
FLT_RET(pStat->code);
}
int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
}
int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) {
int32_t code = 0;
SFilterInfo *info = NULL;
if (pNode == NULL || pInfo == NULL) {
fltError("invalid param");
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (*pInfo == NULL) {
*pInfo = calloc(1, sizeof(SFilterInfo));
if (NULL == *pInfo) {
fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo));
FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
}
info = *pInfo;
info->options = options;
SFltTreeStat stat = {0};
FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat));
if (!info->scalarMode) {
FLT_ERR_JRET(fltInitFromNode(pNode, info, options));
}
FLT_ERR_JRET(fltOptimizeNodes(info, &pNode, &stat));
return code;
_return:
filterFreeInfo(*pInfo);
*pInfo = NULL;
FLT_RET(code);
}
FORCE_INLINE bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
if (info->scalarMode) {
SScalarParam output = {0};
FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output));
*p = output.data;
return TSDB_CODE_SUCCESS;
}
return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols);
}
source/libs/scalar/src/scalar.c
浏览文件 @ 8d413f09
...@@ -45,6 +45,11 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t ...@@ -45,6 +45,11 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
break; break;
} }
case QUERY_NODE_NODE_LIST: {
SNodeListNode *nodeList = (SNodeListNode *)node;
//TODO BUILD HASH
break;
}
case QUERY_NODE_COLUMN_REF: { case QUERY_NODE_COLUMN_REF: {
if (NULL == ctx) { if (NULL == ctx) {
sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
......
source/libs/scalar/src/sclvector.c
浏览文件 @ 8d413f09
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
#include "sclvector.h" #include "sclvector.h"
#include "tcompare.h" #include "tcompare.h"
#include "querynodes.h" #include "querynodes.h"
#include "filter.h" #include "filterInt.h"
#include "query.h" #include "query.h"
//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i]));
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或