From d648694d6e2fd0d93f88851235c0e90dbc52bc4e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 21 Dec 2021 18:17:20 +0800 Subject: [PATCH] [TD-12261] fix json describe length error & judge charset convert return value to avoid core --- src/client/src/tscLocal.c | 11 ++++++----- src/client/src/tscUtil.c | 26 +++++++++++++++---------- src/common/src/texpr.c | 29 +++++++++++++++++++++++++--- src/common/src/tvariant.c | 19 +++++++++++++++---- src/query/src/qExtbuffer.c | 10 ++++++++-- src/query/src/qFilter.c | 39 +++++++++++++++++++++++++++++++------- 6 files changed, 103 insertions(+), 31 deletions(-) diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 90379e6f7e..85c2215a2e 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -128,12 +128,13 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { // type length int32_t bytes = pSchema[i].bytes; pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY){ bytes -= VARSTR_HEADER_SIZE; - - if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes / TSDB_NCHAR_SIZE; - } + } + else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { + bytes -= VARSTR_HEADER_SIZE; + bytes = bytes / TSDB_NCHAR_SIZE; } *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b1af413138..2d759daeac 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -774,11 +774,12 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo memcpy(dst, p, varDataTLen(p)); } else if (varDataLen(p) > 0) { int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst)); - varDataSetLen(dst, length); - - if (length == 0) { + if (length <= 0) { tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); } + if (length >= 0){ + varDataSetLen(dst, length); + } } else { varDataSetLen(dst, 0); } @@ -809,18 +810,23 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo varDataSetLen(dst, strlen(varDataVal(dst))); }else if (type == TSDB_DATA_TYPE_JSON) { int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(dst)); - varDataSetLen(dst, length); - if (length == 0) { + + if (length <= 0) { tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); } + if (length >= 0){ + varDataSetLen(dst, length); + } }else if (type == TSDB_DATA_TYPE_NCHAR) { // value -> "value" *(char*)varDataVal(dst) = '\"'; int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), POINTER_SHIFT(varDataVal(dst), CHAR_BYTES)); - *(char*)(POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES)) = '\"'; - varDataSetLen(dst, length + CHAR_BYTES*2); - if (length == 0) { + if (length <= 0) { tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p); } + if (length >= 0){ + varDataSetLen(dst, length + CHAR_BYTES*2); + *(char*)(POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES)) = '\"'; + } }else if (type == TSDB_DATA_TYPE_DOUBLE) { double jsonVd = *(double*)(realData); sprintf(varDataVal(dst), "%.9lf", jsonVd); @@ -5516,8 +5522,8 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES); if (strlen(jsonValue) > 0 && !taosMbsToUcs4(jsonValue, strlen(jsonValue), varDataVal(tagData), (int32_t)(strlen(jsonValue) * TSDB_NCHAR_SIZE), &outLen)) { - tscError("json string error:%s|%s", strerror(errno), jsonValue); - retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL); + tscError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); + retCode = tscSQLSyntaxErrMsg(errMsg, "charset convert json error", NULL); free(tagVal); goto end; } diff --git a/src/common/src/texpr.c b/src/common/src/texpr.c index 63a19c5c1b..db3bec2ddf 100644 --- a/src/common/src/texpr.c +++ b/src/common/src/texpr.c @@ -27,6 +27,7 @@ #include "tskiplist.h" #include "texpr.h" #include "tarithoperator.h" +#include "tulog.h" static int32_t exprValidateMathNode(tExprNode *pExpr); static int32_t exprValidateStringConcatNode(tExprNode *pExpr); @@ -1274,6 +1275,11 @@ void castConvert(int16_t inputType, int16_t inputBytes, char *input, int16_t Out } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newColData = calloc(1, outputBytes * TSDB_NCHAR_SIZE + 1); int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), newColData); + if (len < 0){ + uError("castConvert taosUcs4ToMbs error 1"); + tfree(newColData); + return; + } newColData[len] = 0; *(int64_t *)output = strtoll(newColData, NULL, 10); tfree(newColData); @@ -1291,6 +1297,11 @@ void castConvert(int16_t inputType, int16_t inputBytes, char *input, int16_t Out } else if (inputType == TSDB_DATA_TYPE_NCHAR) { char *newColData = calloc(1, outputBytes * TSDB_NCHAR_SIZE + 1); int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), newColData); + if (len < 0){ + uError("castConvert taosUcs4ToMbs error 2"); + tfree(newColData); + return; + } newColData[len] = 0; *(int64_t *)output = strtoull(newColData, NULL, 10); tfree(newColData); @@ -1332,11 +1343,19 @@ void castConvert(int16_t inputType, int16_t inputBytes, char *input, int16_t Out if (inputType == TSDB_DATA_TYPE_BOOL) { char tmp[8] = {0}; int32_t len = sprintf(tmp, "%.*s", ncharSize, *(int8_t*)input ? "true" : "false"); - taosMbsToUcs4(tmp, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len); + bool ret = taosMbsToUcs4(tmp, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len); + if(!ret) { + uError("castConvert1 taosMbsToUcs4 error"); + return; + } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { int32_t len = ncharSize > varDataLen(input) ? varDataLen(input) : ncharSize; - taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len); + bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len); + if(!ret) { + uError("castConvert2 taosMbsToUcs4 error"); + return; + } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_TIMESTAMP) { assert(0); @@ -1348,7 +1367,11 @@ void castConvert(int16_t inputType, int16_t inputBytes, char *input, int16_t Out char tmp[400] = {0}; NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); int32_t len = (int32_t)(ncharSize > strlen(tmp) ? strlen(tmp) : ncharSize); - taosMbsToUcs4(tmp, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len); + bool ret = taosMbsToUcs4(tmp, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len); + if(!ret) { + uError("castConvert3 taosMbsToUcs4 error"); + return; + } varDataSetLen(output, len); } break; diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 33b93cfde4..f1453ea334 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -23,6 +23,7 @@ #include "ttype.h" #include "tutil.h" #include "tvariant.h" +#include "tulog.h" #define SET_EXT_INFO(converted, res, minv, maxv, exti) do { \ if (converted == NULL || exti == NULL || *converted == false) { break; } \ @@ -361,8 +362,12 @@ int32_t tVariantToString(tVariant *pVar, char *dst) { case TSDB_DATA_TYPE_NCHAR: { dst[0] = '\''; - taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); - int32_t len = (int32_t)strlen(dst); + int32_t len = taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); + if (len < 0){ + uError("castConvert1 taosUcs4ToMbs error"); + return 0 ; + } + len = (int32_t)strlen(dst); dst[len] = '\''; dst[len + 1] = 0; return len + 1; @@ -430,11 +435,17 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) { pBuf = realloc(pBuf, newSize + 1); } - taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf); + int32_t len = taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf); + if (len < 0){ + uError("castConvert1 taosUcs4ToMbs error"); + } free(pVariant->wpz); pBuf[newSize] = 0; } else { - taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest); + int32_t len = taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest); + if (len < 0){ + uError("castConvert1 taosUcs4ToMbs error"); + } } } else { diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index cc214b9533..9d174b0389 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -1040,7 +1040,10 @@ void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int3 break; case TSDB_DATA_TYPE_NCHAR: { char buf[4096] = {0}; - taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); + int32_t len = taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + } printf("%s\t", buf); break; } @@ -1092,7 +1095,10 @@ void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, in break; case TSDB_DATA_TYPE_NCHAR: { char buf[128] = {0}; - taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); + int32_t len = taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + } printf("%s\t", buf); break; } diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index d5009c85ae..c81faa9282 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -1899,12 +1899,20 @@ int32_t filterInitValFieldData(SFilterInfo *info) { (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); + if (len < 0){ + qError("filterInitValFieldData taosUcs4ToMbs error 1"); + return TSDB_CODE_FAILED; + } varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); }else if(type == TSDB_DATA_TYPE_JSON && (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; int32_t len = taosUcs4ToMbs(((tVariant*)(fi->desc))->pz, ((tVariant*)(fi->desc))->nLen, newValData); + if (len < 0){ + qError("filterInitValFieldData taosUcs4ToMbs error 2"); + return TSDB_CODE_FAILED; + } memcpy(((tVariant*)(fi->desc))->pz, newValData, len); ((tVariant*)(fi->desc))->nLen = len; } @@ -3025,6 +3033,11 @@ static void doJsonCompare(SFilterComUnit *cunit, int8_t *result, void* colData){ }else{ char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + tfree(newColData); + return; + } varDataSetLen(newColData, len); tVariant* val = cunit->valData; char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; @@ -3113,9 +3126,13 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStat if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){ char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); - int len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); - varDataSetLen(newColData, len); - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); + int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + }else{ + varDataSetLen(newColData, len); + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); + } tfree(newColData); }else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ doJsonCompare(&(info->cunits[uidx]), &(*p)[i], colData); @@ -3170,9 +3187,13 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis * } else { if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){ char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); - int len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); - varDataSetLen(newColData, len); - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); + int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + }else{ + varDataSetLen(newColData, len); + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); + } tfree(newColData); }else if(cunit->dataType == TSDB_DATA_TYPE_JSON){ doJsonCompare(cunit, &(*p)[i], colData); @@ -3577,7 +3598,11 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar char *src = FILTER_GET_COL_FIELD_DATA(fi, j); char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); int32_t len = 0; - taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + if(!ret) { + qError("filterConverNcharColumns taosMbsToUcs4 error"); + return TSDB_CODE_FAILED; + } varDataLen(dst) = len; } -- GitLab