From ad299634232e4c07dec38eab35ed6461f250b2bc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Apr 2022 21:00:35 +0800 Subject: [PATCH] refactor: modify json compare in where condition --- source/client/src/clientImpl.c | 72 ++++++++++++++++++++++++++- source/libs/scalar/inc/filterInt.h | 2 - source/libs/scalar/src/filter.c | 78 +++++++++--------------------- 3 files changed, 93 insertions(+), 59 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 74b8e711dc..c1351d0fc8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -619,7 +619,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { int32_t bytes = pResultInfo->fields[i].bytes; if (IS_VAR_DATA_TYPE(type)) { - if (pCol->offset[pResultInfo->current] != -1) { + if (!colDataIsNull_var(pCol,pResultInfo->current)) { char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData; pResultInfo->length[i] = varDataLen(pStart); @@ -725,6 +725,76 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; pResultInfo->row[i] = pResultInfo->pCol[i].pData; } + + if (type == TSDB_DATA_TYPE_JSON) { + char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pResultInfo->convertBuf[i] = p; + int32_t len = 0; + SResultColumn* pCol = &pResultInfo->pCol[i]; + for (int32_t j = 0; j < numOfRows; ++j) { + if (pCol->offset[j] != -1) { + char* pStart = pCol->offset[j] + pCol->pData; + + int32_t jsonInnerType = *pStart; + char *jsonInnerData = pStart + CHAR_BYTES; + char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; + if(jsonInnerType == TSDB_DATA_TYPE_NULL){ + sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else if(jsonInnerType == TSDB_DATA_TYPE_JSON){ + int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); + + if (length <= 0) { + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData)); + length = 0; + } + varDataSetLen(dst, length); + }else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" + *(char*)varDataVal(dst) = '\"'; + int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst) + CHAR_BYTES); + if (length <= 0) { + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, varDataVal(jsonInnerData)); + length = 0; + } + varDataSetLen(dst, length + CHAR_BYTES*2); + *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; + }else if(jsonInnerType == TSDB_DATA_TYPE_DOUBLE){ + double jsonVd = *(double*)(jsonInnerData); + sprintf(varDataVal(dst), "%.9lf", jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else if(jsonInnerType == TSDB_DATA_TYPE_BIGINT){ + int64_t jsonVd = *(int64_t*)(jsonInnerData); + sprintf(varDataVal(dst), "%" PRId64, jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else if(jsonInnerType == TSDB_DATA_TYPE_BOOL){ + sprintf(varDataVal(dst), "%s", (*((char *)jsonInnerData) == 1) ? "true" : "false"); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else { + ASSERT(0); + } + + if(len + varDataTLen(dst) > colLength[i]){ + p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pResultInfo->convertBuf[i] = p; + } + p = pResultInfo->convertBuf[i] + len; + memcpy(p, dst, varDataTLen(dst)); + pCol->offset[j] = len; + len += varDataTLen(dst); + } + } + + pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; + pResultInfo->row[i] = pResultInfo->pCol[i].pData; + } } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 977fab2e85..2b9e3c7f39 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -308,7 +308,6 @@ struct SFilterInfo { #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) -#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) @@ -321,7 +320,6 @@ struct SFilterInfo { #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) -#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ea249005b4..ee85b99ad5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -63,22 +63,18 @@ OptrStr gOptrStr[] = { bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result >= 0; } bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result > 0; } bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(minv, maxr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result <= 0; } bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(minv, maxr); - //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result < 0; } bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -1783,9 +1779,9 @@ int32_t fltInitValFieldData(SFilterInfo *info) { bytes = (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; fi->data = taosMemoryCalloc(1, bytes); - } else if (type != TSDB_DATA_TYPE_JSON){ + } else{ if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE -/* +/* fi->data = taosMemoryCalloc(dType->bytes, tDataTypes[type].bytes); for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); @@ -1796,31 +1792,27 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } else { fi->data = taosMemoryCalloc(1, sizeof(int64_t)); } - } else{ // type == TSDB_DATA_TYPE_JSON - // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) } - if(type != TSDB_DATA_TYPE_JSON) { - if (dType->type == type) { - assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + if (dType->type == type) { + assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + } else { + SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; + out.columnData->info.type = type; + if (IS_VAR_DATA_TYPE(type)) { + out.columnData->info.bytes = bytes; } else { - SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; - out.columnData->info.type = type; - if (IS_VAR_DATA_TYPE(type)) { - out.columnData->info.bytes = bytes; - } else { - out.columnData->info.bytes = tDataTypes[type].bytes; - } - - // todo refactor the convert - int32_t code = doConvertDataType(var, &out); - if (code != TSDB_CODE_SUCCESS) { - qError("convert value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; - } + out.columnData->info.bytes = tDataTypes[type].bytes; + } - memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); + // todo refactor the convert + int32_t code = doConvertDataType(var, &out); + if (code != TSDB_CODE_SUCCESS) { + qError("convert value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; } + + memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); } // match/nmatch for nchar type need convert from ucs4 to mbs @@ -2561,11 +2553,7 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); if (unit->right.type == FLD_TYPE_VALUE) { - if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant - info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit); - }else{ - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); - } + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); } else { info->cunits[i].valData = NULL; } @@ -2891,18 +2879,8 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); - if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ - if (!colData){ // for json->'key' is null - (*p)[i] = 1; - }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null - colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); - }else{ - (*p)[i] = 0; - } - }else{ - (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); - } + (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); + if ((*p)[i] == 0) { all = false; } @@ -2926,19 +2904,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); - if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ - if (!colData) { // for json->'key' is not null - (*p)[i] = 0; - }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null - colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); - }else{ // for json->'key' is not null - (*p)[i] = 1; - } - }else { - (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); - } - + (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); if ((*p)[i] == 0) { all = false; } -- GitLab