[td-168] fix bug in handling variable nchar string

src/client/src/tscParseInsert.c

@@ -305,32 +305,37 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload,
     case TSDB_DATA_TYPE_BINARY:
       // binary data cannot be null-terminated char string, otherwise the last char of the string is lost
       if (pToken->type == TK_NULL) {
+        *(int16_t*) payload = sizeof(int8_t);
+        payload += VARSTR_HEADER_SIZE;
+        
         *payload = TSDB_DATA_BINARY_NULL;
       } else { // too long values will return invalid sql, not be truncated automatically
         if (pToken->n > pSchema->bytes) {
           return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z);
         }
         
-        strncpy(payload, pToken->z, pToken->n);
-        
-        if (pToken->n < pSchema->bytes) {
-          payload[pToken->n] = 0;   // add the null-terminated char if the length of the string is shorter than the available space
-        }
+        STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n);
       }
 
       break;
 
     case TSDB_DATA_TYPE_NCHAR:
       if (pToken->type == TK_NULL) {
-        *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL;
+        *(int16_t*) payload = sizeof(int32_t);
+        payload += VARSTR_HEADER_SIZE;
+  
+        *(uint32_t*) payload = TSDB_DATA_NCHAR_NULL;
       } else {
         // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
-        if (!taosMbsToUcs4(pToken->z, pToken->n, payload, pSchema->bytes)) {
+        int32_t resLen = -1;
+        if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes, &resLen)) {
           char buf[512] = {0};
           snprintf(buf, 512, "%s", strerror(errno));
           
           return tscInvalidSQLErrMsg(msg, buf, pToken->z);
         }
+        
+        *(uint16_t*)payload = (uint16_t) (resLen * TSDB_NCHAR_SIZE);
       }
       break;
 

src/client/src/tscPrepare.c

@@ -300,7 +300,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) {
       break;
     
     case TSDB_DATA_TYPE_NCHAR:
-      if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes)) {
+      if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes, NULL)) {
         return TSDB_CODE_INVALID_VALUE;
       }
       return TSDB_CODE_SUCCESS;

src/client/src/tscSubquery.c

@@ -1934,7 +1934,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF
     /* string terminated char for binary data*/
     memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE);
     
-    if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes, pRes->buffer[columnIndex])) {
+    if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes - VARSTR_HEADER_SIZE, pRes->buffer[columnIndex])) {
       pRes->tsrow[columnIndex] = pRes->buffer[columnIndex];
     } else {
       tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow);

src/client/src/tscUtil.c

@@ -2132,12 +2132,12 @@ char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t colum
   char* pData = ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row;
   
   if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
-    int32_t realLen = *(int16_t*) pData;
-    if (realLen < pInfo->pSqlExpr->resBytes - sizeof(int16_t)) { // todo refactor
+    int32_t realLen = varDataLen(pData);
+    if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor
       *(char*) (pData + realLen + sizeof(int16_t)) = 0;
     }
     
-    return pData + sizeof(int16_t); // head is the length of binary/nchar data
+    return pData + VARSTR_HEADER_SIZE; // head is the length of binary/nchar data
   } else {
     return pData;
   }

src/common/src/tdataformat.c

@@ -172,28 +172,9 @@ int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_
   switch (type) {
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
-      // set offset
       *(VarDataOffsetT *)POINTER_DRIFT(row, toffset) = dataRowLen(row);
-
-      // set length
-      VarDataLenT slen = 0;
-      if (isNull(value, type)) {
-        slen = (type == TSDB_DATA_TYPE_BINARY) ? sizeof(int8_t) : sizeof(int32_t);
-      } else {
-        if (type == TSDB_DATA_TYPE_BINARY) {
-          slen = strnlen((char *)value, bytes);
-        } else {
-          slen = wcsnlen((wchar_t *)value, (bytes) / TSDB_NCHAR_SIZE) * TSDB_NCHAR_SIZE;
-        }
-      }
-
-      ASSERT(slen <= bytes);
-      *(VarDataLenT *)ptr = slen;
-      ptr = POINTER_DRIFT(ptr, sizeof(VarDataLenT));
-
-      memcpy((void *)ptr, value, slen);
-      dataRowLen(row) += (sizeof(int16_t) + slen);
-
+      memcpy(ptr, value, varDataTLen(value));
+      dataRowLen(row) += varDataTLen(value);
       break;
     default:
       memcpy(POINTER_DRIFT(row, toffset), value, TYPE_BYTES[type]);

src/query/src/tvariant.c

@@ -408,7 +408,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   
   if (*pDest == pVariant->pz) {
     wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE);
-    taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE);
+    taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
     
     // free the binary buffer in the first place
     if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
@@ -424,7 +424,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
     
     pVariant->wpz = (wchar_t *)tmp;
   } else {
-    taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE);
+    taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
   }
   
   return 0;

src/util/inc/tutil.h

@@ -144,7 +144,7 @@ void getTmpfilePath(const char *fileNamePattern, char *dstPath);
 int32_t taosInitTimer(void (*callback)(int), int32_t ms);
 void taosUninitTimer();
 
-bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len);
+bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t* len);
 
 int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes);
 

src/util/src/tutil.c

@@ -490,7 +490,7 @@ bool taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
 #endif
 }
 
-bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len) {
+bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t* len) {
   memset(ucs4, 0, ucs4_max_len);
 #ifdef USE_LIBICONV
   iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
@@ -501,6 +501,10 @@ bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len)
     return false;
   }
   iconv_close(cd);
+  if (len != NULL) {
+    *len = outLen;
+  }
+  
   return true;
 #else
   mbstate_t state = {0};