<fix>[TD-12261] fix json describe length error & judge charset convert return value to avoid core

src/client/src/tscLocal.c

@@ -128,12 +128,13 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     // type length
     int32_t bytes = pSchema[i].bytes;
     pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2);
-    if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+
+    if (pSchema[i].type == TSDB_DATA_TYPE_BINARY){
       bytes -= VARSTR_HEADER_SIZE;
-      
-      if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
-        bytes = bytes / TSDB_NCHAR_SIZE;
-      }
+    }
+    else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
+      bytes -= VARSTR_HEADER_SIZE;
+      bytes = bytes / TSDB_NCHAR_SIZE;
     }
 
     *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes;

src/client/src/tscUtil.c

@@ -774,11 +774,12 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
         memcpy(dst, p, varDataTLen(p));
       } else if (varDataLen(p) > 0) {
         int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst));
-        varDataSetLen(dst, length);
-
-        if (length == 0) {
+        if (length <= 0) {
           tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p);
         }
+        if (length >= 0){
+          varDataSetLen(dst, length);
+        }
       } else {
         varDataSetLen(dst, 0);
       }
@@ -809,18 +810,23 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
             varDataSetLen(dst, strlen(varDataVal(dst)));
           }else if (type == TSDB_DATA_TYPE_JSON) {
             int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(dst));
-            varDataSetLen(dst, length);
-            if (length == 0) {
+
+            if (length <= 0) {
               tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p);
             }
+            if (length >= 0){
+              varDataSetLen(dst, length);
+            }
           }else if (type == TSDB_DATA_TYPE_NCHAR) {   // value -> "value"
             *(char*)varDataVal(dst) = '\"';
             int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), POINTER_SHIFT(varDataVal(dst), CHAR_BYTES));
-            *(char*)(POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES)) = '\"';
-            varDataSetLen(dst, length + CHAR_BYTES*2);
-            if (length == 0) {
+            if (length <= 0) {
               tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)p);
             }
+            if (length >= 0){
+              varDataSetLen(dst, length + CHAR_BYTES*2);
+              *(char*)(POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES)) = '\"';
+            }
           }else if (type == TSDB_DATA_TYPE_DOUBLE) {
             double jsonVd = *(double*)(realData);
             sprintf(varDataVal(dst), "%.9lf", jsonVd);
@@ -5516,8 +5522,8 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
       char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
       if (strlen(jsonValue) > 0 && !taosMbsToUcs4(jsonValue, strlen(jsonValue), varDataVal(tagData),
                          (int32_t)(strlen(jsonValue) * TSDB_NCHAR_SIZE), &outLen)) {
-        tscError("json string error:%s|%s", strerror(errno), jsonValue);
-        retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
+        tscError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
+        retCode = tscSQLSyntaxErrMsg(errMsg, "charset convert json error", NULL);
         free(tagVal);
         goto end;
       }

src/common/src/texpr.c

@@ -27,6 +27,7 @@
 #include "tskiplist.h"
 #include "texpr.h"
 #include "tarithoperator.h"
+#include "tulog.h"
 
 static int32_t exprValidateMathNode(tExprNode *pExpr);
 static int32_t exprValidateStringConcatNode(tExprNode *pExpr);
@@ -1274,6 +1275,11 @@ void castConvert(int16_t inputType, int16_t inputBytes, char *input, int16_t Out
       } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
         char *newColData = calloc(1, outputBytes * TSDB_NCHAR_SIZE + 1);
         int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), newColData);
+        if (len < 0){
+          uError("castConvert taosUcs4ToMbs error 1");
+          tfree(newColData);
+          return;
+        }
         newColData[len] = 0;
         *(int64_t *)output = strtoll(newColData, NULL, 10);
         tfree(newColData);
@@ -1291,6 +1297,11 @@ void castConvert(int16_t inputType, int16_t inputBytes, char *input, int16_t Out
       } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
         char *newColData = calloc(1, outputBytes * TSDB_NCHAR_SIZE + 1);
         int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), newColData);
+        if (len < 0){
+          uError("castConvert taosUcs4ToMbs error 2");
+          tfree(newColData);
+          return;
+        }
         newColData[len] = 0;
         *(int64_t *)output = strtoull(newColData, NULL, 10);
         tfree(newColData);
@@ -1332,11 +1343,19 @@ void castConvert(int16_t inputType, int16_t inputBytes, char *input, int16_t Out
         if (inputType == TSDB_DATA_TYPE_BOOL) {
           char tmp[8] = {0};
           int32_t len = sprintf(tmp, "%.*s", ncharSize, *(int8_t*)input ? "true" : "false");
-          taosMbsToUcs4(tmp, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len);
+          bool ret = taosMbsToUcs4(tmp, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len);
+          if(!ret) {
+            uError("castConvert1 taosMbsToUcs4 error");
+            return;
+          }
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_BINARY) {
           int32_t len = ncharSize > varDataLen(input) ? varDataLen(input) : ncharSize;
-          taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len);
+          bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len);
+          if(!ret) {
+            uError("castConvert2 taosMbsToUcs4 error");
+            return;
+          }
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_TIMESTAMP) {
           assert(0);
@@ -1348,7 +1367,11 @@ void castConvert(int16_t inputType, int16_t inputBytes, char *input, int16_t Out
           char tmp[400] = {0};
           NUM_TO_STRING(inputType, input, sizeof(tmp), tmp);
           int32_t len = (int32_t)(ncharSize > strlen(tmp) ? strlen(tmp) : ncharSize);
-          taosMbsToUcs4(tmp, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len);
+          bool ret = taosMbsToUcs4(tmp, len, varDataVal(output), outputBytes - VARSTR_HEADER_SIZE, &len);
+          if(!ret) {
+            uError("castConvert3 taosMbsToUcs4 error");
+            return;
+          }
           varDataSetLen(output, len);
         }
         break;

src/common/src/tvariant.c

@@ -23,6 +23,7 @@
 #include "ttype.h"
 #include "tutil.h"
 #include "tvariant.h"
+#include "tulog.h"
 
 #define SET_EXT_INFO(converted, res, minv, maxv, exti) do {                                                       \
                                                         if (converted == NULL || exti == NULL || *converted == false) { break; }  \
@@ -361,8 +362,12 @@ int32_t tVariantToString(tVariant *pVar, char *dst) {
 
     case TSDB_DATA_TYPE_NCHAR: {
       dst[0] = '\'';
-      taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
-      int32_t len = (int32_t)strlen(dst);
+      int32_t len = taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
+      if (len < 0){
+        uError("castConvert1 taosUcs4ToMbs error");
+        return 0 ;
+      }
+      len = (int32_t)strlen(dst);
       dst[len] = '\'';
       dst[len + 1] = 0;
       return len + 1;
@@ -430,11 +435,17 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
         pBuf = realloc(pBuf, newSize + 1);
       }
 
-      taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf);
+      int32_t len = taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, pBuf);
+      if (len < 0){
+        uError("castConvert1 taosUcs4ToMbs error");
+      }
       free(pVariant->wpz);
       pBuf[newSize] = 0;
     } else {
-      taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest);
+      int32_t len = taosUcs4ToMbs(pVariant->wpz, (int32_t)newSize, *pDest);
+      if (len < 0){
+        uError("castConvert1 taosUcs4ToMbs error");
+      }
     }
 
   } else {

src/query/src/qExtbuffer.c

@@ -1040,7 +1040,10 @@ void tColModelDisplay(SColumnModel *pModel, void *pData, int32_t numOfRows, int3
           break;
         case TSDB_DATA_TYPE_NCHAR: {
           char buf[4096] = {0};
-          taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf);
+          int32_t len = taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf);
+          if (len < 0){
+            qError("castConvert1 taosUcs4ToMbs error");
+          }
           printf("%s\t", buf);
           break;
         }
@@ -1092,7 +1095,10 @@ void tColModelDisplayEx(SColumnModel *pModel, void *pData, int32_t numOfRows, in
           break;
         case TSDB_DATA_TYPE_NCHAR: {
           char buf[128] = {0};
-          taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf);
+          int32_t len = taosUcs4ToMbs(val, pModel->pFields[j].field.bytes, buf);
+          if (len < 0){
+            qError("castConvert1 taosUcs4ToMbs error");
+          }
           printf("%s\t", buf);
           break;
         }

src/query/src/qFilter.c

@@ -1899,12 +1899,20 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
         (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
       char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
       int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
+      if (len < 0){
+        qError("filterInitValFieldData taosUcs4ToMbs error 1");
+        return TSDB_CODE_FAILED;
+      }
       varDataSetLen(newValData, len);
       varDataCopy(fi->data, newValData);
     }else if(type == TSDB_DATA_TYPE_JSON &&
         (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
       char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
       int32_t len = taosUcs4ToMbs(((tVariant*)(fi->desc))->pz, ((tVariant*)(fi->desc))->nLen, newValData);
+      if (len < 0){
+        qError("filterInitValFieldData taosUcs4ToMbs error 2");
+        return TSDB_CODE_FAILED;
+      }
       memcpy(((tVariant*)(fi->desc))->pz, newValData, len);
       ((tVariant*)(fi->desc))->nLen = len;
     }
@@ -3025,6 +3033,11 @@ static void doJsonCompare(SFilterComUnit *cunit, int8_t *result, void* colData){
     }else{
       char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
       int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData));
+      if (len < 0){
+        qError("castConvert1 taosUcs4ToMbs error");
+        tfree(newColData);
+        return;
+      }
       varDataSetLen(newColData, len);
       tVariant* val = cunit->valData;
       char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
@@ -3113,9 +3126,13 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStat
 
     if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){
       char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
-      int len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
-      varDataSetLen(newColData, len);
-      (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData);
+      int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
+      if (len < 0){
+        qError("castConvert1 taosUcs4ToMbs error");
+      }else{
+        varDataSetLen(newColData, len);
+        (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData);
+      }
       tfree(newColData);
     }else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
       doJsonCompare(&(info->cunits[uidx]), &(*p)[i], colData);
@@ -3170,9 +3187,13 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *
             } else {
               if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){
                 char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
-                int len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
-                varDataSetLen(newColData, len);
-                (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData);
+                int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
+                if (len < 0){
+                  qError("castConvert1 taosUcs4ToMbs error");
+                }else{
+                  varDataSetLen(newColData, len);
+                  (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData);
+                }
                 tfree(newColData);
               }else if(cunit->dataType == TSDB_DATA_TYPE_JSON){
                 doJsonCompare(cunit, &(*p)[i], colData);
@@ -3577,7 +3598,11 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
         char *src = FILTER_GET_COL_FIELD_DATA(fi, j);
         char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j);
         int32_t len = 0;
-        taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
+        bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
+        if(!ret) {
+          qError("filterConverNcharColumns taosMbsToUcs4 error");
+          return TSDB_CODE_FAILED;
+        }
         varDataLen(dst) = len;
       }