[TD-225]handle the error in converting mbs to nchar

src/client/src/tscParseInsert.c

@@ -332,7 +332,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload,
           return tscInvalidSQLErrMsg(msg, "illegal double data", pToken->z);
         }
 
-        if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv)) {
+        if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
           return tscInvalidSQLErrMsg(msg, "illegal double data", pToken->z);
         }
 
@@ -359,7 +359,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload,
         setVardataNull(payload, TSDB_DATA_TYPE_NCHAR);
       } else {
         // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
-        size_t output = 0;
+        int32_t output = 0;
         if (!taosMbsToUcs4(pToken->z, pToken->n, varDataVal(payload), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) {
           char buf[512] = {0};
           snprintf(buf, tListLen(buf), "%s", strerror(errno));

src/client/src/tscPrepare.c

@@ -616,7 +616,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
       case TSDB_DATA_TYPE_NCHAR: {
         switch (bind->buffer_type) {
           case TSDB_DATA_TYPE_NCHAR: {
-            size_t output = 0;
+            int32_t output = 0;
             if (!taosMbsToUcs4(bind->buffer, *bind->length, varDataVal(data + param->offset), param->bytes - VARSTR_HEADER_SIZE, &output)) {
               return TSDB_CODE_TSC_INVALID_VALUE;
             }

src/common/src/tvariant.c

@@ -341,7 +341,6 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   return 0;
 }
 
-// todo handle the error
 static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   char tmpBuf[40] = {0};
   
@@ -364,7 +363,10 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   
   if (*pDest == pVariant->pz) {
     wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE);
-    taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
+    bool ret = taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
+    if (!ret) {
+      return -1;
+    }
 
     // free the binary buffer in the first place
     if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
@@ -380,11 +382,14 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
     
     pVariant->wpz = (wchar_t *)tmp;
   } else {
-    size_t output = -1;
-    taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output);
+    int32_t output = 0;
+    bool ret = taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output);
+    if (!ret) {
+      return -1;
+    }
 
     if (pDestSize != NULL) {
-      *pDestSize = (int32_t)output;
+      *pDestSize = output;
     }
   }
   
@@ -416,11 +421,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
     *result = (int64_t) pVariant->dKey;
   } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
     SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen};
-    int32_t n = tSQLGetToken(pVariant->pz, &token.type);
-    if (token.type == TK_MINUS || token.type == TK_PLUS) {
-      // decide if pVariant->pz is NULL or not
-      tSQLGetToken(pVariant->pz + n, &token.type);
-    }
+    /*int32_t n = */tSQLGetToken(pVariant->pz, &token.type);
 
     if (token.type == TK_NULL) {
       if (releaseVariantPtr) {
@@ -549,8 +550,6 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) {
 /*
  * transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType
  * to column type defined in schema
- *
- * todo handle the return value
  */
 int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) {
   if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) {
@@ -709,7 +708,9 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
           *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL;
         } else {
           if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
-            toNchar(pVariant, &payload, &newlen);
+            if (toNchar(pVariant, &payload, &newlen) != 0) {
+              return -1;
+            }
           } else {
             wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen);
           }
@@ -721,7 +722,9 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
           char *p = varDataVal(payload);
 
           if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
-            toNchar(pVariant, &p, &newlen);
+            if (toNchar(pVariant, &p, &newlen) != 0) {
+              return -1;
+            }
           } else {
             wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen);
             newlen = pVariant->nLen;
@@ -805,7 +808,9 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) {
     }
     case TSDB_DATA_TYPE_NCHAR: {
       if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
-        toNchar(pVariant, &pVariant->pz, &pVariant->nLen);
+        if (toNchar(pVariant, &pVariant->pz, &pVariant->nLen) != 0) {
+          return -1;
+        }
       }
       pVariant->nType = type;
       break;

src/cq/src/cqMain.c

@@ -341,7 +341,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
       val = ((char*)val) - sizeof(VarDataLenT);
     } else if (c->type == TSDB_DATA_TYPE_NCHAR) {
       char buf[TSDB_MAX_NCHAR_LEN];
-      size_t len = taos_fetch_lengths(tres)[i];
+      int32_t len = taos_fetch_lengths(tres)[i];
       taosMbsToUcs4(val, len, buf, sizeof(buf), &len);
       memcpy(val + sizeof(VarDataLenT), buf, len);
       varDataLen(val) = len;

src/os/inc/osString.h

@@ -47,7 +47,7 @@ extern "C" {
 
 // USE_LIBICONV
 int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
-bool    taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len);
+bool    taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int *len);
 int     tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes);
 bool    taosValidateEncodec(const char *encodec);
 char *  taosCharsetReplace(char *charsetstr);

src/os/src/detail/osString.c

@@ -46,7 +46,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
   return (int32_t)(ucs4_max_len - outLen);
 }
 
-bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) {
+bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
   memset(ucs4, 0, ucs4_max_len);
   iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
   size_t  ucs4_input_len = mbsLength;
@@ -92,7 +92,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
   return len;
 }
 
-bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) {
+bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
   memset(ucs4, 0, ucs4_max_len);
   mbstate_t state = {0};
   int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);