diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ab1869d233e496bc513c560757498eb5b628f936..8f40cf6fb9fbbfeb191e720f946fcc34890df5ee 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -305,32 +305,37 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, case TSDB_DATA_TYPE_BINARY: // binary data cannot be null-terminated char string, otherwise the last char of the string is lost if (pToken->type == TK_NULL) { + *(int16_t*) payload = sizeof(int8_t); + payload += VARSTR_HEADER_SIZE; + *payload = TSDB_DATA_BINARY_NULL; } else { // too long values will return invalid sql, not be truncated automatically if (pToken->n > pSchema->bytes) { return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z); } - strncpy(payload, pToken->z, pToken->n); - - if (pToken->n < pSchema->bytes) { - payload[pToken->n] = 0; // add the null-terminated char if the length of the string is shorter than the available space - } + STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n); } break; case TSDB_DATA_TYPE_NCHAR: if (pToken->type == TK_NULL) { - *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL; + *(int16_t*) payload = sizeof(int32_t); + payload += VARSTR_HEADER_SIZE; + + *(uint32_t*) payload = TSDB_DATA_NCHAR_NULL; } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - if (!taosMbsToUcs4(pToken->z, pToken->n, payload, pSchema->bytes)) { + int32_t resLen = -1; + if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes, &resLen)) { char buf[512] = {0}; snprintf(buf, 512, "%s", strerror(errno)); return tscInvalidSQLErrMsg(msg, buf, pToken->z); } + + *(uint16_t*)payload = (uint16_t) (resLen * TSDB_NCHAR_SIZE); } break; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index fdd032d373fd16f124acaf6b79a7edc12526118d..260649050be5448a6c975abdf1b82582a9a17575 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -300,7 +300,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { break; case TSDB_DATA_TYPE_NCHAR: - if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes)) { + if (!taosMbsToUcs4(bind->buffer, *bind->length, data + param->offset, param->bytes, NULL)) { return TSDB_CODE_INVALID_VALUE; } return TSDB_CODE_SUCCESS; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 796887e628406118f317e7524a0430436290418e..9aceecd377cd682442a63f6a6f51826dca723e74 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1934,7 +1934,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF /* string terminated char for binary data*/ memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE); - if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes, pRes->buffer[columnIndex])) { + if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes - VARSTR_HEADER_SIZE, pRes->buffer[columnIndex])) { pRes->tsrow[columnIndex] = pRes->buffer[columnIndex]; } else { tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 8324a7470a0f0325e352e6417b00a429f0e36101..d07185b0ba798060818e86bd0160ad279ff6bf36 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2132,12 +2132,12 @@ char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t colum char* pData = ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { - int32_t realLen = *(int16_t*) pData; - if (realLen < pInfo->pSqlExpr->resBytes - sizeof(int16_t)) { // todo refactor + int32_t realLen = varDataLen(pData); + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor *(char*) (pData + realLen + sizeof(int16_t)) = 0; } - return pData + sizeof(int16_t); // head is the length of binary/nchar data + return pData + VARSTR_HEADER_SIZE; // head is the length of binary/nchar data } else { return pData; } diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 7321e1c921bab082f3074b087cab12b30da00c4c..02351f7d3aba514628305370b23734b0039ff5e8 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -172,28 +172,9 @@ int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_ switch (type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - // set offset *(VarDataOffsetT *)POINTER_DRIFT(row, toffset) = dataRowLen(row); - - // set length - VarDataLenT slen = 0; - if (isNull(value, type)) { - slen = (type == TSDB_DATA_TYPE_BINARY) ? sizeof(int8_t) : sizeof(int32_t); - } else { - if (type == TSDB_DATA_TYPE_BINARY) { - slen = strnlen((char *)value, bytes); - } else { - slen = wcsnlen((wchar_t *)value, (bytes) / TSDB_NCHAR_SIZE) * TSDB_NCHAR_SIZE; - } - } - - ASSERT(slen <= bytes); - *(VarDataLenT *)ptr = slen; - ptr = POINTER_DRIFT(ptr, sizeof(VarDataLenT)); - - memcpy((void *)ptr, value, slen); - dataRowLen(row) += (sizeof(int16_t) + slen); - + memcpy(ptr, value, varDataTLen(value)); + dataRowLen(row) += varDataTLen(value); break; default: memcpy(POINTER_DRIFT(row, toffset), value, TYPE_BYTES[type]); diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index ab5e30f6c40609068496098b4846e7783c09faf3..0c4a2b67578c8f59643941d5f5ac3caaceebce54 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -408,7 +408,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { if (*pDest == pVariant->pz) { wchar_t *pWStr = calloc(1, (nLen + 1) * TSDB_NCHAR_SIZE); - taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE); + taosMbsToUcs4(pDst, nLen, (char *)pWStr, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); // free the binary buffer in the first place if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { @@ -424,7 +424,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { pVariant->wpz = (wchar_t *)tmp; } else { - taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE); + taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); } return 0; diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index d272910abf7f8115a99101aa05e8c7b116c6f18a..b03e0a1c5b5cb72a7741cae4b233c16db6c6bebf 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -144,7 +144,7 @@ void getTmpfilePath(const char *fileNamePattern, char *dstPath); int32_t taosInitTimer(void (*callback)(int), int32_t ms); void taosUninitTimer(); -bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len); +bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t* len); int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 707d6741f78ac40e500d6168e7925b8fbc836bce..396f6f1ef889f390d0449c5d44add78b1a30b6ba 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -490,7 +490,7 @@ bool taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { #endif } -bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len) { +bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t* len) { memset(ucs4, 0, ucs4_max_len); #ifdef USE_LIBICONV iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); @@ -501,6 +501,10 @@ bool taosMbsToUcs4(char *mbs, int32_t mbs_len, char *ucs4, int32_t ucs4_max_len) return false; } iconv_close(cd); + if (len != NULL) { + *len = outLen; + } + return true; #else mbstate_t state = {0};