Merge pull request #12124 from taosdata/feature/3.0_glzhao

fix(query): add hex encoding/decoding for transmitting nchar constant from client to server in JSON

include/os/osString.h

@@ -59,6 +59,8 @@ bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs
 int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
 TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
 bool    taosValidateEncodec(const char *encodec);
+int32_t taosHexEncode(const char *src, char *dst, int32_t len);
+int32_t taosHexDecode(const char *src, char *dst, int32_t len);
 
 int32_t taosWcharWidth(TdWchar wchar);
 int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size);

source/libs/nodes/src/nodesCodeFuncs.c

@@ -1971,7 +1971,18 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) {
     case TSDB_DATA_TYPE_DOUBLE:
       code = tjsonAddDoubleToObject(pJson, jkValueDatum, pNode->datum.d);
       break;
-    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_NCHAR: {
+      //cJSON only support utf-8 encoding. Convert memory content to hex string.
+      char *buf = taosMemoryCalloc(varDataLen(pNode->datum.p) * 2 + 1, sizeof(char));
+      code = taosHexEncode(varDataVal(pNode->datum.p), buf, varDataLen(pNode->datum.p));
+      if(code != TSDB_CODE_SUCCESS) {
+        taosMemoryFree(buf);
+        return TSDB_CODE_TSC_INVALID_VALUE;
+      }
+      code = tjsonAddStringToObject(pJson, jkValueDatum, buf);
+      taosMemoryFree(buf);
+      break;
+    }
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY:
       code = tjsonAddStringToObject(pJson, jkValueDatum, varDataVal(pNode->datum.p));
@@ -2074,7 +2085,26 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) {
         break;
       }
       varDataSetLen(pNode->datum.p, pNode->node.resType.bytes);
+      if (TSDB_DATA_TYPE_NCHAR == pNode->node.resType.type) {
+        char *buf = taosMemoryCalloc(1, pNode->node.resType.bytes * 2 + VARSTR_HEADER_SIZE + 1);
+        if (NULL == buf) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          break;
+        }
+        code = tjsonGetStringValue(pJson, jkValueDatum, buf);
+        if (code != TSDB_CODE_SUCCESS) {
+          taosMemoryFree(buf);
+          break;
+        }
+        code = taosHexDecode(buf, varDataVal(pNode->datum.p), pNode->node.resType.bytes - VARSTR_HEADER_SIZE);
+        if (code != TSDB_CODE_SUCCESS) {
+          taosMemoryFree(buf);
+          break;
+        }
+        taosMemoryFree(buf);
+      } else {
         code = tjsonGetStringValue(pJson, jkValueDatum, varDataVal(pNode->datum.p));
+      }
       break;
     }
     case TSDB_DATA_TYPE_JSON:

source/libs/parser/src/parAstCreater.c

@@ -363,10 +363,8 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d
   CHECK_OUT_OF_MEM(func);
   strcpy(func->functionName, "cast");
   func->node.resType = dt;
-  if (TSDB_DATA_TYPE_BINARY == dt.type) {
-    func->node.resType.bytes += 2;
-  } else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
-    func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + 2;
+  if (TSDB_DATA_TYPE_NCHAR == dt.type) {
+    func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE;
   }
   nodesListMakeAppend(&func->pParameterList, pExpr);
   return (SNode*)func;

source/libs/scalar/src/sclfunc.c

@@ -709,6 +709,10 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
   int16_t outputType = GET_PARAM_TYPE(&pOutput[0]);
   int64_t outputLen  = GET_PARAM_BYTES(&pOutput[0]);
 
+  if (IS_VAR_DATA_TYPE(outputType)) {
+    outputLen += VARSTR_HEADER_SIZE;
+  }
+
   char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1);
   char *output = outputBuf;
 
@@ -790,29 +794,30 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
       }
       case TSDB_DATA_TYPE_NCHAR: {
         int32_t outputCharLen = (outputLen - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+        int32_t len;
         if (inputType == TSDB_DATA_TYPE_BOOL) {
           char tmp[8] = {0};
-          int32_t len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" );
+          len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" );
           bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
           if (!ret) {
             return TSDB_CODE_FAILED;
           }
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_BINARY) {
-          int32_t len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen;
+          len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen;
           bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
           if (!ret) {
             return TSDB_CODE_FAILED;
           }
           varDataSetLen(output, len);
         } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
-          int32_t len = TMIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE);
-          memcpy(output, input, len);
-          varDataSetLen(output, len - VARSTR_HEADER_SIZE);
+          len = TMIN(outputLen - VARSTR_HEADER_SIZE, varDataLen(input));
+          memcpy(output, input, len + VARSTR_HEADER_SIZE);
+          varDataSetLen(output, len);
         } else {
           char tmp[400] = {0};
           NUM_TO_STRING(inputType, input, sizeof(tmp), tmp);
-          int32_t len = (int32_t)strlen(tmp);
+          len = (int32_t)strlen(tmp);
           len = outputCharLen > len ? len : outputCharLen;
           bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
           if (!ret) {
@@ -820,6 +825,10 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
           }
           varDataSetLen(output, len);
         }
+        //for constant conversion, need to set proper length of pOutput description
+        if (len < outputLen - VARSTR_HEADER_SIZE) {
+          pOutput->columnData->info.bytes = len;
+        }
         break;
       }
       default: {

source/os/src/osString.c

@@ -195,6 +195,36 @@ int32_t taosUcs4len(TdUcs4 *ucs4) {
   return n;
 }
 
+//dst buffer size should be at least 2*len + 1
+int32_t taosHexEncode(const char *src, char *dst, int32_t len) {
+  if (!dst) {
+    return -1;
+  }
+
+  for (int32_t i = 0; i < len; ++i) {
+    sprintf(dst + i * 2, "%02x", src[i] & 0xff);
+  }
+
+  return 0;
+}
+
+int32_t taosHexDecode(const char *src, char *dst, int32_t len) {
+  if (!dst) {
+    return -1;
+  }
+
+  uint16_t hn, ln, out;
+  for (int i = 0, j = 0; i < len * 2; i += 2, ++j ) {
+    hn = src[i] > '9' ? src[i] - 'A' + 10 : src[i] - '0';
+    ln = src[i + 1] > '9' ? src[i + 1] - 'A' + 10 : src[i + 1] - '0';
+
+    out = (hn << 4) | ln;
+    memcpy(dst + j, &out, 1);
+  }
+
+  return 0;
+}
+
 int32_t taosWcharWidth(TdWchar wchar) { return wcwidth(wchar); }
 
 int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { return wcswidth(pWchar, size); }