From 79e5d6832b153009dd091b5e6c55bd913c40a4f0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 Nov 2021 21:33:22 +0800 Subject: [PATCH] enable different value type of the same key in json --- src/common/src/tvariant.c | 4 +- src/inc/taoserror.h | 4 +- src/query/inc/qFilter.h | 2 + src/query/src/qFilter.c | 150 +++++++++++++++++++++++--------------- src/tsdb/src/tsdbMeta.c | 31 ++------ src/tsdb/src/tsdbRead.c | 2 +- src/util/inc/tcompare.h | 3 +- src/util/src/tcompare.c | 53 +++++++++++--- src/util/src/terror.c | 2 - 9 files changed, 146 insertions(+), 105 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 72659368e5..8af44cdf12 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -925,9 +925,7 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc break; } case TSDB_DATA_TYPE_JSON:{ - if (pVariant->nType == TSDB_DATA_TYPE_NULL) { - //*(int8_t *)payload = TSDB_DATA_TINYINT_NULL; - } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY){ + if (pVariant->nType == TSDB_DATA_TYPE_BINARY){ *((int8_t *)payload) = TSDB_DATA_JSON_PLACEHOLDER; } else if (pVariant->nType == TSDB_DATA_TYPE_JSON){ // select * from stable, set tag type to json，from setTagValue/tag_project_function memcpy(payload, pVariant->pz, pVariant->nLen); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index db786c530e..47fd9e6aaf 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -272,8 +272,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) //"TSDB invalid tag value") #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) //"TSDB no cache last row data") #define TSDB_CODE_TDB_INCOMPLETE_DFILESET TAOS_DEF_ERROR_CODE(0, 0x0617) //"TSDB incomplete DFileSet") -#define TSDB_CODE_TDB_IVLD_SAME_JSON_VALUE TAOS_DEF_ERROR_CODE(0, 0x0618) //"TSDB invalid same json value") -#define TSDB_CODE_TDB_NO_JSON_TAG_KEY TAOS_DEF_ERROR_CODE(0, 0x0619) //"TSDB no tag json key") +#define TSDB_CODE_TDB_NO_JSON_TAG_KEY TAOS_DEF_ERROR_CODE(0, 0x0618) //"TSDB no tag json key") // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") @@ -292,7 +291,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070D) //"System error") #define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070E) //"invalid time condition") #define TSDB_CODE_QRY_JSON_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x070F) //"only support is [not] null") -#define TSDB_CODE_QRY_JSON_INVALID_EXP TAOS_DEF_ERROR_CODE(0, 0x0710) //"invalid regular expression") // grant #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") diff --git a/src/query/inc/qFilter.h b/src/query/inc/qFilter.h index d290930ce4..fe9ef0f47f 100644 --- a/src/query/inc/qFilter.h +++ b/src/query/inc/qFilter.h @@ -288,6 +288,7 @@ typedef struct SFilterInfo { #define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) +#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) @@ -300,6 +301,7 @@ typedef struct SFilterInfo { #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) +#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index ee4b30cb57..ad1788a5c6 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -161,7 +161,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, - compareStrRegexCompNMatch, comparreStrContainJson + compareStrRegexCompNMatch, compareStrContainJson, compareJsonVal }; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { @@ -204,8 +204,6 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 19; } else if (optr == TSDB_RELATION_NMATCH) { comparFn = 20; - } else if (optr == TSDB_RELATION_QUESTION) { - comparFn = 21; } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ comparFn = 7; } else if (optr == TSDB_RELATION_IN) { @@ -217,14 +215,11 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { break; } - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON:{ + case TSDB_DATA_TYPE_NCHAR:{ if (optr == TSDB_RELATION_MATCH) { comparFn = 19; } else if (optr == TSDB_RELATION_NMATCH) { comparFn = 20; - } else if (optr == TSDB_RELATION_QUESTION) { - comparFn = 21; } else if (optr == TSDB_RELATION_LIKE) { comparFn = 9; } else if (optr == TSDB_RELATION_IN) { @@ -234,6 +229,18 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } break; } + case TSDB_DATA_TYPE_JSON:{ + if (optr == TSDB_RELATION_MATCH) { + comparFn = 19; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = 20; + } else if (optr == TSDB_RELATION_QUESTION) { + comparFn = 21; + } else { + comparFn = 22; + } + break; + } case TSDB_DATA_TYPE_UTINYINT: comparFn = 11; break; case TSDB_DATA_TYPE_USMALLINT: comparFn = 12;break; @@ -1168,30 +1175,15 @@ static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pL char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; jsonKeyMd5((*pLeft)->_node.pRight->pVal->pz, (*pLeft)->_node.pRight->pVal->nLen, keyMd5); memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); - - void* data = getJsonTagValue(info->pTable, schema->name, TSDB_MAX_JSON_KEY_MD5_LEN, &schema->colId); - if(data != NULL){ - schema->type = *(char*)data; // if exist json tag-> operation get type so that can set data if (tree->_node.optr == TSDB_RELATION_IN_IN) the next and set value in filterInitValFieldData - assert(schema->type > TSDB_DATA_TYPE_NULL && schema->type < TSDB_DATA_TYPE_JSON); - } (*pLeft) = (*pLeft)->_node.pLeft; // -> operation use left as input - - if (tree->_node.pRight->nodeType == TSQL_NODE_VALUE && IS_VAR_DATA_TYPE(tree->_node.pRight->pVal->nType)) { - schema = (*pLeft)->pSchema; - if (!IS_VAR_DATA_TYPE(schema->type)) { - char *v = strndup(tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen); - uint32_t type = 0; - tGetToken(v, &type); - if (type == TK_NULL) { - free(v); - return TSDB_CODE_QRY_JSON_SUPPORT_ERROR; - } - free(v); - } - } - } - - if(tree->_node.optr == TSDB_RELATION_QUESTION){ + }else if(((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) && + (tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL)){ + SSchema* schema = (*pLeft)->pSchema; + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + uint32_t nullData = TSDB_DATA_JSON_NULL; + jsonKeyMd5(&nullData, INT_BYTES, keyMd5); + memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); + }else if(tree->_node.optr == TSDB_RELATION_QUESTION){ SSchema* schema = (*pLeft)->pSchema; if(tree->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; @@ -1199,19 +1191,6 @@ static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pL memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); } - SSchema* schema = (*pLeft)->pSchema; - if(tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL){ - if((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) { - char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; - uint32_t nullData = TSDB_DATA_JSON_NULL; - jsonKeyMd5(&nullData, INT_BYTES, keyMd5); - memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); - } - }else if(tree->_node.optr == TSDB_RELATION_MATCH || tree->_node.optr == TSDB_RELATION_NMATCH || tree->_node.optr == TSDB_RELATION_LIKE){ - if(!IS_VAR_DATA_TYPE(schema->type)){ - return TSDB_CODE_QRY_JSON_INVALID_EXP; - } - } return TSDB_CODE_SUCCESS; } @@ -1226,7 +1205,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g int32_t len = 0; uint32_t uidx = 0; - if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { + if (tree->_node.optr == TSDB_RELATION_IN && !IS_VAR_DATA_TYPE(type) && type != TSDB_DATA_TYPE_JSON) { void *data = NULL; filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type, false); CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); @@ -1872,7 +1851,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) { if (type == TSDB_DATA_TYPE_BINARY) { size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); - } else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON) { + } else if (type == TSDB_DATA_TYPE_NCHAR) { size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); } else { @@ -1889,16 +1868,18 @@ int32_t filterInitValFieldData(SFilterInfo *info) { } } - bool converted = false; - char extInfo = 0; - if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { - if (converted) { - filterHandleValueExtInfo(unit, extInfo); - - continue; + if(type != TSDB_DATA_TYPE_JSON){ + bool converted = false; + char extInfo = 0; + if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { + if (converted) { + filterHandleValueExtInfo(unit, extInfo); + + continue; + } + qError("dump value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; } - qError("dump value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; } // match/nmatch for nchar type need convert from ucs4 to mbs @@ -1908,7 +1889,14 @@ int32_t filterInitValFieldData(SFilterInfo *info) { int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); + }else if(type == TSDB_DATA_TYPE_JSON && + (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + int32_t len = taosUcs4ToMbs(((tVariant*)(fi->desc))->pz, ((tVariant*)(fi->desc))->nLen, newValData); + memcpy(((tVariant*)(fi->desc))->pz, newValData, len); + ((tVariant*)(fi->desc))->nLen = len; } + } return TSDB_CODE_SUCCESS; @@ -2632,7 +2620,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); if (unit->right.type == FLD_TYPE_VALUE) { - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant + info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit); + }else{ + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + } } else { info->cunits[i].valData = NULL; } @@ -3051,6 +3043,27 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStat varDataSetLen(newColData, len); (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); tfree(newColData); + }else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ + if(info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH){ + uint8_t jsonType = *(char*)colData; + char* realData = colData + CHAR_BYTES; + if (jsonType != TSDB_DATA_TYPE_NCHAR){ + (*p)[i] = false; + }else{ + char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData)); + varDataSetLen(newColData, len); + tVariant* val = info->cunits[uidx].valData; + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE); + memcpy(varDataVal(newValData), val->pz, val->nLen); + varDataSetLen(newValData, val->nLen); + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, newValData); + tfree(newColData); + } + }else{ + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + } }else{ (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); } @@ -3107,6 +3120,27 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis * varDataSetLen(newColData, len); (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); tfree(newColData); + }else if(cunit->dataType == TSDB_DATA_TYPE_JSON){ + if(cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH){ + uint8_t jsonType = *(char*)colData; + char* realData = colData + CHAR_BYTES; + if (jsonType != TSDB_DATA_TYPE_NCHAR){ + (*p)[i] = false; + }else{ + char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData)); + varDataSetLen(newColData, len); + tVariant* val = cunit->valData; + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE); + memcpy(varDataVal(newValData), val->pz, val->nLen); + varDataSetLen(newValData, val->nLen); + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, newValData); + tfree(newColData); + } + }else{ + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } }else{ (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } @@ -3280,6 +3314,7 @@ int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) { ERR_JRET(code); filterConvertGroupFromArray(info, group); + taosArrayDestroy(group); ERR_JRET(filterInitValFieldData(info)); @@ -3291,7 +3326,6 @@ int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) { CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - taosArrayDestroy(group); return code; } } @@ -3301,15 +3335,11 @@ int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) { filterDumpInfoToString(info, "Final", 0); - taosArrayDestroy(group); - return code; _return: qInfo("No filter, code:%d", code); - taosArrayDestroy(group); - filterFreeInfo(*pinfo); *pinfo = NULL; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 58f2ab6572..95654538f2 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1106,6 +1106,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper ASSERT(jsonNULL == TSDB_DATA_JSON_NULL); } + // then insert char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; jsonKeyMd5(varDataVal(val), varDataLen(val), keyMd5); SArray *tablistNew = NULL; @@ -1126,36 +1127,14 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper tablistNew = *tablist; } - if (taosArrayGetSize(tablistNew) > 0) { - // validate type - JsonMapValue *tmp = taosArrayGet(tablistNew, 0); - void *data1 = tdGetKVRowValOfCol(((STable *)(tmp->table))->tagVal, tmp->colId + 1); - SColIdx *pInsertColIdx = kvRowColIdxAt(pTable->tagVal, j + 1); - void *data2 = (kvRowColVal(pTable->tagVal, pInsertColIdx)); - if (*(uint8_t *)data1 != *(uint8_t *)data2) { - terrno = TSDB_CODE_TDB_IVLD_SAME_JSON_VALUE; - tsdbError("invalidate same json tag value"); - return -1; - } - } - } - - // then insert - for (int j = 1; j < nCols; j = j + 2) { - SColIdx *pColIdx = kvRowColIdxAt(pTable->tagVal, j); - void *val = (kvRowColVal(pTable->tagVal, pColIdx)); - char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; - jsonKeyMd5(varDataVal(val), varDataLen(val), keyMd5); - SArray **tablist = (SArray **)taosHashGet(pSTable->jsonKeyMap, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); - JsonMapValue jmvalue = {pTable, pColIdx->colId}; - void* p = taosArraySearch(*tablist, &jmvalue, tsdbCompareJsonMapValue, TD_EQ); + void* p = taosArraySearch(tablistNew, &jmvalue, tsdbCompareJsonMapValue, TD_EQ); if (p == NULL) { - p = taosArraySearch(*tablist, &jmvalue, tsdbCompareJsonMapValue, TD_GE); + p = taosArraySearch(tablistNew, &jmvalue, tsdbCompareJsonMapValue, TD_GE); if(p == NULL){ - taosArrayPush(*tablist, &jmvalue); + taosArrayPush(tablistNew, &jmvalue); }else{ - taosArrayInsert(*tablist, TARRAY_ELEM_IDX(*tablist, p), &jmvalue); + taosArrayInsert(tablistNew, TARRAY_ELEM_IDX(tablistNew, p), &jmvalue); } }else{ tsdbError("insert dumplicate"); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index aaaf87545b..6c5ddbfc5b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -4049,7 +4049,7 @@ static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, ch } else { void* jsonData = getJsonTagValue(pTable, name, TSDB_MAX_JSON_KEY_MD5_LEN, NULL); // jsonData == NULL for ? operation - if(jsonData != NULL) jsonData += CHAR_BYTES; // jump type + // if(jsonData != NULL) jsonData += CHAR_BYTES; // jump type *data = jsonData; } diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h index a4e34a6d9c..fe0e504e88 100644 --- a/src/util/inc/tcompare.h +++ b/src/util/inc/tcompare.h @@ -88,7 +88,8 @@ int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight); int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight); int32_t compareFindItemInSet(const void *pLeft, const void* pRight); int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); -int32_t comparreStrContainJson(const void* pLeft, const void* pRight); +int32_t compareStrContainJson(const void* pLeft, const void* pRight); +int32_t compareJsonVal(const void* pLeft, const void* pRight); #ifdef __cplusplus } diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 0ebf15a5a5..07bd8061e0 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -18,11 +18,12 @@ #define _DEFAULT_SOURCE #include "tcompare.h" -#include "tulog.h" +#include "tvariant.h" #include "hash.h" -#include "regex.h" #include "os.h" +#include "regex.h" #include "ttype.h" +#include "tulog.h" int32_t setCompareBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -218,6 +219,33 @@ int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) { return compareLenPrefixedWStr(pRight, pLeft); } +int32_t compareJsonVal(const void *pLeft, const void *pRight) { + const tVariant* right = pRight; + if(right->nType != *(char*)pLeft) return -1; + + uint8_t type = *(char*)pLeft; + char* realData = POINTER_SHIFT(pLeft, CHAR_BYTES); + if(type == TSDB_DATA_TYPE_BOOL) { + DEFAULT_COMP(GET_INT8_VAL(realData), right->i64); + }else if(type == TSDB_DATA_TYPE_BIGINT){ + DEFAULT_COMP(GET_INT64_VAL(realData), right->i64); + }else if(type == TSDB_DATA_TYPE_DOUBLE){ + DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(realData), right->dKey); + }else if(type == TSDB_DATA_TYPE_NCHAR){ + if (varDataLen(realData) != right->nLen) { + return varDataLen(realData) > right->nLen ? 1 : -1; + } + int32_t ret = memcmp(varDataVal(realData), right->pz, right->nLen); + if (ret == 0) { + return ret; + } + return (ret < 0) ? -1 : 1; + }else{ + assert(0); + } + return 0; +} + /* * Compare two strings * TSDB_MATCH: Match @@ -405,7 +433,7 @@ int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { return result; } -int32_t comparreStrContainJson(const void* pLeft, const void* pRight) { +int32_t compareStrContainJson(const void* pLeft, const void* pRight) { if(pLeft) return 0; return 1; } @@ -484,8 +512,6 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { comparFn = compareStrRegexCompNMatch; } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ comparFn = compareStrPatternComp; - } else if (optr == TSDB_RELATION_QUESTION) { - comparFn = comparreStrContainJson; } else if (optr == TSDB_RELATION_IN) { comparFn = compareFindItemInSet; } else { /* normal relational comparFn */ @@ -495,14 +521,11 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { break; } - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON:{ + case TSDB_DATA_TYPE_NCHAR:{ if (optr == TSDB_RELATION_MATCH) { comparFn = compareStrRegexCompMatch; } else if (optr == TSDB_RELATION_NMATCH) { comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_QUESTION) { - comparFn = comparreStrContainJson; } else if (optr == TSDB_RELATION_LIKE) { comparFn = compareWStrPatternComp; } else if (optr == TSDB_RELATION_IN) { @@ -512,6 +535,18 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } break; } + case TSDB_DATA_TYPE_JSON:{ + if (optr == TSDB_RELATION_MATCH) { + comparFn = compareStrRegexCompMatch; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = compareStrRegexCompNMatch; + } else if (optr == TSDB_RELATION_QUESTION) { + comparFn = compareStrContainJson; + } else { + comparFn = compareJsonVal; + } + break; + } case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 6024d8d9aa..8963a39ca9 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -280,7 +280,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INCOMPLETE_DFILESET, "Incomplete DFileSet") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_SAME_JSON_VALUE, "TSDB invalid same json value") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_JSON_TAG_KEY, "TSDB no tag json key") // query @@ -300,7 +299,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION, "One valid time range condition expected") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_SUPPORT_ERROR, "only support is [not] null") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_INVALID_EXP, "invalid regular expression") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired") -- GitLab