From b4e2a79c1724c23ac1332bf80c3abc17a27b3f1c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Dec 2021 19:47:38 +0800 Subject: [PATCH] fix bugs and add test case --- src/client/src/tscSQLParser.c | 10 ++- src/query/src/qFilter.c | 113 +++++++++++--------------- src/tsdb/src/tsdbRead.c | 6 +- tests/pytest/stable/json_tag.py | 135 +++++++++++++++++++++----------- 4 files changed, 145 insertions(+), 119 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2863ff4c9b..1a54342fcb 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4979,8 +4979,9 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql tSqlExpr *rexpr = NULL; if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR - && pSchema->type != TSDB_DATA_TYPE_BOOL)) { - handleNeOptr(&rexpr, *pExpr); //todo json check + && pSchema->type != TSDB_DATA_TYPE_BOOL + && pSchema->type != TSDB_DATA_TYPE_JSON)) { + handleNeOptr(&rexpr, *pExpr); *pExpr = rexpr; } @@ -9907,7 +9908,10 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS // NOTE: binary|nchar data allows the >|< type filter if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) { + if (pLeft->_node.optr == TSDB_RELATION_ARROW){ + pLeft = pLeft->_node.pLeft; + } + if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && (pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL || pLeft->pSchema->type == TSDB_DATA_TYPE_JSON)) { return TSDB_CODE_TSC_INVALID_OPERATION; } } diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index f7a64ee50b..fbef608a35 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -62,15 +62,23 @@ filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { }; bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + int32_t result = cfunc(maxv, minr); + if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return cfunc(maxv, minr) >= 0; } bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + int32_t result = cfunc(maxv, minr); + if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return cfunc(maxv, minr) > 0; } bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + int32_t result = cfunc(minv, maxr); + if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return cfunc(minv, maxr) <= 0; } bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + int32_t result = cfunc(minv, maxr); + if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return cfunc(minv, maxr) < 0; } bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -2990,6 +2998,42 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } +static void doJsonCompare(SFilterComUnit *cunit, int8_t *result, void* colData){ + if(cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH){ + uint8_t jsonType = *(char*)colData; + char* realData = colData + CHAR_BYTES; + if (jsonType != TSDB_DATA_TYPE_NCHAR){ + *result = false; + }else{ + char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData)); + varDataSetLen(newColData, len); + tVariant* val = cunit->valData; + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE); + memcpy(varDataVal(newValData), val->pz, val->nLen); + varDataSetLen(newValData, val->nLen); + *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, newValData); + tfree(newColData); + } + }else if(cunit->optr == TSDB_RELATION_LIKE){ + uint8_t jsonType = *(char*)colData; + char* realData = colData + CHAR_BYTES; + if (jsonType != TSDB_DATA_TYPE_NCHAR){ + *result = false; + }else{ + tVariant* val = cunit->valData; + char* newValData = calloc(val->nLen + VARSTR_HEADER_SIZE, 1); + memcpy(varDataVal(newValData), val->pz, val->nLen); + varDataSetLen(newValData, val->nLen); + *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, realData, newValData); + tfree(newValData); + } + }else{ + *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } +} + bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; @@ -3056,39 +3100,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStat (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); tfree(newColData); }else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ - if(info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH){ - uint8_t jsonType = *(char*)colData; - char* realData = colData + CHAR_BYTES; - if (jsonType != TSDB_DATA_TYPE_NCHAR){ - (*p)[i] = false; - }else{ - char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); - int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData)); - varDataSetLen(newColData, len); - tVariant* val = info->cunits[uidx].valData; - char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; - assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE); - memcpy(varDataVal(newValData), val->pz, val->nLen); - varDataSetLen(newValData, val->nLen); - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, newValData); - tfree(newColData); - } - }else if(info->cunits[uidx].optr == TSDB_RELATION_LIKE){ - uint8_t jsonType = *(char*)colData; - char* realData = colData + CHAR_BYTES; - if (jsonType != TSDB_DATA_TYPE_NCHAR){ - (*p)[i] = false; - }else{ - tVariant* val = info->cunits[uidx].valData; - char* newValData = calloc(val->nLen + VARSTR_HEADER_SIZE, 1); - memcpy(varDataVal(newValData), val->pz, val->nLen); - varDataSetLen(newValData, val->nLen); - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, realData, newValData); - tfree(newValData); - } - }else{ - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); - } + doJsonCompare(&(info->cunits[uidx]), &(*p)[i], colData); }else{ (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); } @@ -3101,7 +3113,6 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStat return all; } - bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; @@ -3146,39 +3157,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis * (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); tfree(newColData); }else if(cunit->dataType == TSDB_DATA_TYPE_JSON){ - if(cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH){ - uint8_t jsonType = *(char*)colData; - char* realData = colData + CHAR_BYTES; - if (jsonType != TSDB_DATA_TYPE_NCHAR){ - (*p)[i] = false; - }else{ - char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); - int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData)); - varDataSetLen(newColData, len); - tVariant* val = cunit->valData; - char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; - assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE); - memcpy(varDataVal(newValData), val->pz, val->nLen); - varDataSetLen(newValData, val->nLen); - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, newValData); - tfree(newColData); - } - }else if(cunit->optr == TSDB_RELATION_LIKE){ - uint8_t jsonType = *(char*)colData; - char* realData = colData + CHAR_BYTES; - if (jsonType != TSDB_DATA_TYPE_NCHAR){ - (*p)[i] = false; - }else{ - tVariant* val = cunit->valData; - char* newValData = calloc(val->nLen + VARSTR_HEADER_SIZE, 1); - memcpy(varDataVal(newValData), val->pz, val->nLen); - varDataSetLen(newValData, val->nLen); - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, realData, newValData); - tfree(newValData); - } - }else{ - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); - } + doJsonCompare(cunit, &(*p)[i], colData); }else{ (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6c5ddbfc5b..e63c6a0c8f 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -4106,11 +4106,9 @@ static int32_t queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){ } } } - if(tabList == NULL || taosArrayGetSize(tabList) == 0){ + if(tabList == NULL){ tsdbError("json key not exist, no candidate table"); - terrno = TSDB_CODE_TDB_NO_JSON_TAG_KEY; - taosArrayDestroy(tabList); - return TSDB_CODE_TDB_NO_JSON_TAG_KEY; + return TSDB_CODE_SUCCESS; } int32_t size = taosArrayGetSize(tabList); int8_t *addToResult = NULL; diff --git a/tests/pytest/stable/json_tag.py b/tests/pytest/stable/json_tag.py index cd86191a81..c1edee24a4 100644 --- a/tests/pytest/stable/json_tag.py +++ b/tests/pytest/stable/json_tag.py @@ -146,6 +146,9 @@ class TDTestCase: # test where with json tag tdSql.error("select * from jsons1_1 where jtag is not null") + tdSql.error("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") + tdSql.error("select * from jsons1 where jtag->'tag1'={}") + # where json value is string tdSql.query("select * from jsons1 where jtag->'tag2'='beijing'") tdSql.checkRows(2) @@ -161,6 +164,19 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query("select * from jsons1 where jtag->'tag1'='收到货'") tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag2'>'beijing'") + tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag2'>='beijing'") + tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag2'<'beijing'") + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag2'<='beijing'") + tdSql.checkRows(4) + tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'") + tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag2'=''") + tdSql.checkRows(2) + # where json value is int tdSql.query("select * from jsons1 where jtag->'tag1'=5") tdSql.checkRows(1) @@ -169,12 +185,58 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query("select * from jsons1 where jtag->'tag1'<54") tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1'<=11") + tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1'>4") + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1'>=5") + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1'!=5") + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1'!=55") + tdSql.checkRows(3) + # where json value is double tdSql.query("select * from jsons1 where jtag->'tag1'=1.232") tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") + tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") + tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'>1.23") + tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232") + tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232") + tdSql.checkRows(3) + tdSql.error("select * from jsons1 where jtag->'tag1'/0=3") + tdSql.error("select * from jsons1 where jtag->'tag1'/5=1") + + # where json value is bool + tdSql.query("select * from jsons1 where jtag->'tag1'=true") + tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'=false") + tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'!=false") + tdSql.checkRows(0) + tdSql.error("select * from jsons1 where jtag->'tag1'>false") + # where json value is null - tdSql.query("select * from jsons1 where jtag->'tag1'=null") + tdSql.query("select * from jsons1 where jtag->'tag1'=null") # only json suport =null. This synatx will change later. tdSql.checkRows(1) + + # where json is null + tdSql.query("select * from jsons1 where jtag is null") + tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag is not null") + tdSql.checkRows(8) + + # where json key is null + tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") + tdSql.checkRows(0) + # where json value is not exist tdSql.query("select * from jsons1 where jtag->'tag1' is null") tdSql.checkData(0, 0, 'jsons1_9') @@ -184,55 +246,42 @@ class TDTestCase: tdSql.query("select * from jsons1 where jtag->'tag3' is not null") tdSql.checkRows(4) - - # test json tag in where condition with and/or/? - tdSql.query("select * from jsons1 where jtag->'location'!='beijing'") - tdSql.checkRows(1) - - tdSql.query("select jtag->'num' from jsons1 where jtag->'level'='l1'") - tdSql.checkData(0, 0, 34) - - # test json number value - tdSql.query("select *,tbname from jsons1 where jtag->'class'>5 and jtag->'class'<9") + # test ? + tdSql.query("select * from jsons1 where jtag?'tag1'") + tdSql.checkRows(8) + tdSql.query("select * from jsons1 where jtag?'tag3'") + tdSql.checkRows(4) + tdSql.query("select * from jsons1 where jtag?'tag_no_exist'") tdSql.checkRows(0) - tdSql.query("select *,tbname from jsons1 where jtag->'class'>5 and jtag->'class'<92") - tdSql.checkRows(1) - - # test where condition - tdSql.query("select * from jsons1 where jtag?'sex' or jtag?'num'") - tdSql.checkRows(3) - - tdSql.query("select * from jsons1 where jtag?'sex' or jtag?'numww'") + # test json tag in where condition with and/or + tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='beijing'") tdSql.checkRows(1) - - tdSql.query("select * from jsons1 where jtag?'sex' and jtag?'num'") - tdSql.checkRows(0) - - tdSql.query("select jtag->'sex' from jsons1 where jtag?'sex' or jtag?'num'") - tdSql.checkRows(3) - - tdSql.query("select *,tbname from jsons1 where jtag->'location'='beijing'") + tdSql.query("select * from jsons1 where jtag->'tag1'=false or jtag->'tag2'='beijing'") tdSql.checkRows(2) - - tdSql.query("select *,tbname from jsons1 where jtag->'num'=5 or jtag?'sex'") + tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='shanghai'") + tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='shanghai'") + tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'=13 or jtag->'tag2'>35") + tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'=13 or jtag->'tag2'>35") + tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1' is not null and jtag?'tag3'") + tdSql.checkRows(4) + tdSql.query("select * from jsons1 where jtag->'tag1'='femail' and jtag?'tag3'") tdSql.checkRows(2) - tdSql.query("select *,tbname from jsons1 where jtag->'num'=5") - tdSql.checkRows(1) - - # test with tbname + # test with tbname/normal cloumn tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") - tdSql.checkRows(1) - - tdSql.query("select * from jsons1 where tbname = 'jsons1_1' or jtag?'num'") - tdSql.checkRows(3) - - tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag?'num'") + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag?'tag3'") + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag?'tag3' and dataint=3") tdSql.checkRows(0) + tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag?'tag3' and dataint=23") + tdSql.checkRows(1) - tdSql.query("select * from jsons1 where tbname = 'jsons1_1' or jtag->'num'=5") - tdSql.checkRows(2) # test where condition like tdSql.query("select *,tbname from jsons1 where jtag->'location' like 'bei%'") @@ -298,11 +347,7 @@ class TDTestCase: tdSql.query("select jtag from jsons1 where jtag->'k2'=true") tdSql.checkRows(1) - tdSql.query("select jtag from jsons1 where jtag is null") - tdSql.checkRows(4) - tdSql.query("select jtag from jsons1 where jtag is not null") - tdSql.checkRows(6) tdSql.query("select * from jsons1 where jtag->'location' is not null") tdSql.checkRows(3) -- GitLab