fix:to_json() function

include/common/tdataformat.h

@@ -78,6 +78,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
 int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
 int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
 void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);  // TODO: remove
+int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf);
 
 // STRUCT =================
 struct STColumn {

source/common/src/tvariant.c

@@ -155,7 +155,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin
 void taosVariantDestroy(SVariant *pVar) {
   if (pVar == NULL) return;
 
-  if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) {
+  if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR
+     || pVar->nType == TSDB_DATA_TYPE_JSON) {
     taosMemoryFreeClear(pVar->pz);
     pVar->nLen = 0;
   }
@@ -184,7 +185,8 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (pSrc == NULL || pDst == NULL) return;
 
   pDst->nType = pSrc->nType;
-  if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) {
+  if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR
+      || pSrc->nType == TSDB_DATA_TYPE_JSON) {
     int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
     char   *p = taosMemoryRealloc(pDst->pz, len);
     assert(p);
@@ -976,6 +978,7 @@ char *taosVariantGet(SVariant *pVar, int32_t type) {
     case TSDB_DATA_TYPE_FLOAT:
       return (char *)&pVar->d;
     case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_JSON:
       return (char *)pVar->pz;
     case TSDB_DATA_TYPE_NCHAR:
       return (char *)pVar->ucs4;

source/libs/command/src/command.c

@@ -297,7 +297,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
   SArray* pTagVals = NULL;
   STag*   pTag = (STag*)pCfg->pTags;
 
-  if (pCfg->pTags && pTag->flags & TD_TAG_JSON) {
+  if (pCfg->pTags && tTagIsJson(pTag)) {
     char* pJson = parseTagDatatoJson(pTag);
     if (pJson) {
       *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson);

source/libs/function/src/builtins.c

@@ -1543,7 +1543,7 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BINARY].bytes, .type = TSDB_DATA_TYPE_BINARY};
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes, .type = TSDB_DATA_TYPE_JSON};
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/nodes/src/nodesCloneFuncs.c

@@ -18,6 +18,7 @@
 #include "querynodes.h"
 #include "taos.h"
 #include "taoserror.h"
+#include "tdatablock.h"
 
 #define COPY_SCALAR_FIELD(fldname)     \
   do {                                 \
@@ -164,7 +165,15 @@ static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
       memcpy(pDst->datum.p, pSrc->datum.p, len);
       break;
     }
-    case TSDB_DATA_TYPE_JSON:
+    case TSDB_DATA_TYPE_JSON:{
+      int32_t len = getJsonValueLen(pSrc->datum.p);
+      pDst->datum.p = taosMemoryCalloc(1, len);
+      if (NULL == pDst->datum.p) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+      memcpy(pDst->datum.p, pSrc->datum.p, len);
+      break;
+    }
     case TSDB_DATA_TYPE_DECIMAL:
     case TSDB_DATA_TYPE_BLOB:
     case TSDB_DATA_TYPE_MEDIUMBLOB:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -20,6 +20,7 @@
 #include "querynodes.h"
 #include "taoserror.h"
 #include "tjson.h"
+#include "tdatablock.h"
 
 static int32_t nodeToJson(const void* pObj, SJson* pJson);
 static int32_t jsonToNode(const SJson* pJson, void* pObj);
@@ -2638,7 +2639,18 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) {
     case TSDB_DATA_TYPE_VARBINARY:
       code = tjsonAddStringToObject(pJson, jkValueDatum, varDataVal(pNode->datum.p));
       break;
-    case TSDB_DATA_TYPE_JSON:
+    case TSDB_DATA_TYPE_JSON:{
+      int32_t len = getJsonValueLen(pNode->datum.p);
+      char* buf = taosMemoryCalloc( len * 2 + 1, sizeof(char));
+      code = taosHexEncode(pNode->datum.p, buf, len);
+      if (code != TSDB_CODE_SUCCESS) {
+        taosMemoryFree(buf);
+        return TSDB_CODE_TSC_INVALID_VALUE;
+      }
+      code = tjsonAddStringToObject(pJson, jkValueDatum, buf);
+      taosMemoryFree(buf);
+      break;
+    }
     case TSDB_DATA_TYPE_DECIMAL:
     case TSDB_DATA_TYPE_BLOB:
       // todo
@@ -2761,7 +2773,30 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) {
       }
       break;
     }
-    case TSDB_DATA_TYPE_JSON:
+    case TSDB_DATA_TYPE_JSON:{
+      pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes);
+      if (NULL == pNode->datum.p) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        break;
+      }
+      char* buf = taosMemoryCalloc(1, pNode->node.resType.bytes * 2 + 1);
+      if (NULL == buf) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        break;
+      }
+      code = tjsonGetStringValue(pJson, jkValueDatum, buf);
+      if (code != TSDB_CODE_SUCCESS) {
+        taosMemoryFree(buf);
+        break;
+      }
+      code = taosHexDecode(buf, pNode->datum.p, pNode->node.resType.bytes);
+      if (code != TSDB_CODE_SUCCESS) {
+        taosMemoryFree(buf);
+        break;
+      }
+      taosMemoryFree(buf);
+      break;
+    }
     case TSDB_DATA_TYPE_DECIMAL:
     case TSDB_DATA_TYPE_BLOB:
       // todo

source/libs/nodes/src/nodesUtilFuncs.c

@@ -20,6 +20,7 @@
 #include "taos.h"
 #include "taoserror.h"
 #include "thash.h"
+#include "tdatablock.h"
 
 static SNode* makeNode(ENodeType type, size_t size) {
   SNode* p = taosMemoryCalloc(1, size);
@@ -1675,6 +1676,10 @@ void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal) {
       pVal->pz[pVal->nLen + VARSTR_HEADER_SIZE] = 0;
       break;
     case TSDB_DATA_TYPE_JSON:
+      pVal->nLen = getJsonValueLen(pNode->datum.p);
+      pVal->pz = taosMemoryMalloc(pVal->nLen);
+      memcpy(pVal->pz, pNode->datum.p, pVal->nLen);
+      break;
     case TSDB_DATA_TYPE_DECIMAL:
     case TSDB_DATA_TYPE_BLOB:
       // todo

source/libs/parser/inc/parUtil.h

@@ -62,7 +62,6 @@ int32_t       getNumOfColumns(const STableMeta* pTableMeta);
 int32_t       getNumOfTags(const STableMeta* pTableMeta);
 STableComInfo getTableInfo(const STableMeta* pTableMeta);
 STableMeta*   tableMetaDup(const STableMeta* pTableMeta);
-int32_t       parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMsgBuf* pMsgBuf);
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
 

source/libs/parser/src/parUtil.c

@@ -219,6 +219,7 @@ int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) {
 }
 
 int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) {
+  if(pBuf == NULL) return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
   const char* msgFormat1 = "syntax error near \'%s\'";
   const char* msgFormat2 = "syntax error near \'%s\' (%s)";
   const char* msgFormat3 = "%s";
@@ -346,7 +347,7 @@ static bool isValidateTag(char* input) {
   return true;
 }
 
-int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMsgBuf* pMsgBuf) {
+int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf) {
   int32_t   retCode = TSDB_CODE_SUCCESS;
   cJSON*    root = NULL;
   SHashObj* keyHash = NULL;

source/libs/qcom/src/queryUtil.c

@@ -304,18 +304,21 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t
 
 char* parseTagDatatoJson(void* p) {
   char*  string = NULL;
-  cJSON* json = cJSON_CreateObject();
-  if (json == NULL) {
-    goto end;
-  }
-
   SArray* pTagVals = NULL;
+  cJSON* json = NULL;
   if (tTagToValArray((const STag*)p, &pTagVals) != 0) {
     goto end;
   }
 
   int16_t nCols = taosArrayGetSize(pTagVals);
+  if (nCols == 0) {
+    goto end;
+  }
   char    tagJsonKey[256] = {0};
+  json = cJSON_CreateObject();
+  if (json == NULL) {
+    goto end;
+  }
   for (int j = 0; j < nCols; ++j) {
     STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j);
     // json key  encode by binary
@@ -373,6 +376,10 @@ char* parseTagDatatoJson(void* p) {
   string = cJSON_PrintUnformatted(json);
 end:
   cJSON_Delete(json);
+  taosArrayDestroy(pTagVals);
+  if(string == NULL){
+    string = strdup(TSDB_DATA_NULL_STR_L);
+  }
   return string;
 }
 

source/libs/scalar/src/scalar.c

@@ -695,7 +695,11 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
     res->node.resType.scale = output.columnData->info.scale;
     res->node.resType.precision = output.columnData->info.precision;
     int32_t type = output.columnData->info.type;
-    if (IS_VAR_DATA_TYPE(type)) {
+    if (type == TSDB_DATA_TYPE_JSON){
+      int32_t len = getJsonValueLen(output.columnData->pData);
+      res->datum.p = taosMemoryCalloc(len, 1);
+      memcpy(res->datum.p, output.columnData->pData, len);
+    } else if (IS_VAR_DATA_TYPE(type)) {
       res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1);
       memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData));
     } else {

source/libs/scalar/src/sclfunc.c

@@ -1152,42 +1152,30 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 
   char tmp[TSDB_MAX_JSON_TAG_LEN] = {0};
   for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
-    if (colDataIsNull_s(pInput[0].columnData, i)) {
-      colDataAppendNULL(pOutput->columnData, i);
-      continue;
-    }
-    char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[i];
+    SArray* pTagVals = taosArrayInit(8, sizeof(STagVal));
+    STag*   pTag = NULL;
 
-    if(type == TSDB_DATA_TYPE_NCHAR){
-      if (varDataTLen(input) > TSDB_MAX_JSON_TAG_LEN){
-        colDataAppendNULL(pOutput->columnData, i);
-        continue;
-      }
-      int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), tmp);
-      if (len < 0) {
-        colDataAppendNULL(pOutput->columnData, i);
-        continue;
-      }
-      tmp[len] = 0;
+    if (colDataIsNull_s(pInput[0].columnData, i)) {
+      tTagNew(pTagVals, 1, true, &pTag);
     }else{
+      char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[i];
       if (varDataLen(input) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
-        colDataAppendNULL(pOutput->columnData, i);
-        continue;
+        taosArrayDestroy(pTagVals);
+        return TSDB_CODE_FAILED;
       }
       memcpy(tmp, varDataVal(input), varDataLen(input));
       tmp[varDataLen(input)] = 0;
+      if(parseJsontoTagData(tmp, pTagVals, &pTag, NULL)){
+        tTagNew(pTagVals, 1, true, &pTag);
+      }
     }
 
-    if(!tjsonValidateJson(tmp)){
-      colDataAppendNULL(pOutput->columnData, i);
-      continue;
-    }
-
-    colDataAppend(pOutput->columnData, i, input, false);
+    colDataAppend(pOutput->columnData, i, (const char*)pTag, false);
+    tTagFree(pTag);
+    taosArrayDestroy(pTagVals);
   }
 
   pOutput->numOfRows = pInput->numOfRows;
-
   return TSDB_CODE_SUCCESS;
 }
 

source/os/src/osString.c

@@ -202,7 +202,7 @@ int32_t taosHexEncode(const char *src, char *dst, int32_t len) {
   }
 
   for (int32_t i = 0; i < len; ++i) {
-    sprintf(dst + i * 2, "%02x", src[i] & 0xff);
+    sprintf(dst + i * 2, "%02x", src[i]);
   }
 
   return 0;
@@ -213,10 +213,10 @@ int32_t taosHexDecode(const char *src, char *dst, int32_t len) {
     return -1;
   }
 
-  uint16_t hn, ln, out;
+  uint8_t hn, ln, out;
   for (int i = 0, j = 0; i < len * 2; i += 2, ++j ) {
-    hn = src[i] > '9' ? src[i] - 'A' + 10 : src[i] - '0';
-    ln = src[i + 1] > '9' ? src[i + 1] - 'A' + 10 : src[i + 1] - '0';
+    hn = src[i] > '9' ? src[i] - 'a' + 10 : src[i] - '0';
+    ln = src[i + 1] > '9' ? src[i + 1] - 'a' + 10 : src[i + 1] - '0';
 
     out = (hn << 4) | ln;
     memcpy(dst + j, &out, 1);

tests/system-test/2-query/json_tag.py

@@ -684,6 +684,19 @@ class TDTestCase:
         tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;")
         tdSql.checkRows(1)
 
+        # to_json()
+        tdSql.query("select to_json('{\"abc\":123}') from jsons1_1")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, '{"abc":123}')
+        tdSql.checkData(1, 0, '{"abc":123}')
+        tdSql.query("select to_json('null') from jsons1_1")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, 'null')
+        tdSql.checkData(1, 0, 'null')
+        tdSql.query("select to_json('{\"key\"}') from jsons1_1")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, 'null')
+        tdSql.checkData(1, 0, 'null')
 
         #test TD-12077
         tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')")