TD-6129<feature> fix json encode error

src/client/inc/tscUtil.h

@@ -375,7 +375,7 @@ void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id);
 char* cloneCurrentDBName(SSqlObj* pSql);
 
 int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId);
-
+char* parseTagDatatoJson(void *p);
 #ifdef __cplusplus
 }
 #endif

src/client/src/tscSubquery.c

@@ -3552,7 +3552,7 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql) {
     int32_t type  = pInfo->field.type;
     int32_t bytes = pInfo->field.bytes;
 
-    if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) {
+    if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_JSON) {
       pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : pRes->urow[i];
     } else {
       pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : varDataVal(pRes->urow[i]);

src/client/src/tscUtil.c

@@ -712,6 +712,35 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
       p += pInfo->field.bytes;
     }
 
+    memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
+  }else if (pInfo->field.type == TSDB_DATA_TYPE_JSON) {
+    // convert unicode to native code in a temporary buffer extra one byte for terminated symbol
+    char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
+    if(buffer == NULL)
+      return ;
+    pRes->buffer[i] = buffer;
+    // string terminated char for binary data
+    memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows);
+
+    char* p = pRes->urow[i];
+    for (int32_t k = 0; k < pRes->numOfRows; ++k) {
+      char* dst = pRes->buffer[i] + k * pInfo->field.bytes;
+
+      if (isNull(p, TSDB_DATA_TYPE_NCHAR)) {
+        memcpy(dst, p, varDataTLen(p));
+      } else {
+        char* json = parseTagDatatoJson(p);
+        if(json){
+          memcpy(varDataVal(dst), json, strlen(json));
+          varDataSetLen(dst, strlen(json));
+          tfree(json);
+        }else{
+          tscError("construct json error");
+        }
+      }
+      p += pInfo->field.bytes;
+    }
+
     memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
   }
 }
@@ -5145,6 +5174,65 @@ char* cloneCurrentDBName(SSqlObj* pSql) {
   return p;
 }
 
+char* parseTagDatatoJson(void *p){
+  char* string = NULL;
+  cJSON *json = cJSON_CreateObject();
+  if (json == NULL)
+  {
+    goto end;
+  }
+
+  int16_t nCols = kvRowNCols(p);
+  ASSERT(nCols%2 == 1);
+  char tagJsonKey[TSDB_MAX_TAGS_LEN] = {0};
+  for (int j = 0; j < nCols; ++j) {
+    SColIdx * pColIdx = kvRowColIdxAt(p, j);
+    void* val = (kvRowColVal(p, pColIdx));
+    if (j == 0){        // json value is the first
+      int8_t jsonVal = *(int8_t*)val;
+      ASSERT(jsonVal == TSDB_DATA_BINARY_PLACEHOLDER);
+      continue;
+    }
+    if (j%2 == 0) { // json key
+      memset(tagJsonKey, 0, TSDB_MAX_TAGS_LEN);
+      int32_t length = taosUcs4ToMbs(varDataVal(val), varDataLen(val), tagJsonKey);
+      if (length == 0) {
+        tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val);
+        goto end;
+      }
+    }else{  // json value
+      char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0};
+      if(*(char*)val == cJSON_String){
+        int32_t length = taosUcs4ToMbs(varDataVal(val + CHAR_BYTES), varDataLen(val + CHAR_BYTES), tagJsonValue);
+        if (length == 0) {
+          tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val);
+          goto end;
+        }
+        cJSON* value = cJSON_CreateString(tagJsonValue);
+        if (value == NULL)
+        {
+          goto end;
+        }
+        cJSON_AddItemToObject(json, tagJsonKey, value);
+      }else if(*(char*)val == cJSON_Number){
+        double jsonVd = *(double*)(val + CHAR_BYTES);
+        cJSON* value = cJSON_CreateNumber(jsonVd);
+        if (value == NULL)
+        {
+          goto end;
+        }
+        cJSON_AddItemToObject(json, tagJsonKey, value);
+      }else{
+        tscError("unsupportted json value");
+      }
+    }
+  }
+  string = cJSON_PrintUnformatted(json);
+end:
+  cJSON_Delete(json);
+  return string;
+}
+
 int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId){
   cJSON *root = cJSON_Parse(json);
   if (root == NULL){

src/common/src/ttypes.c

@@ -384,7 +384,7 @@ tDataTypeDescriptor tDataTypes[16] = {
   {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES,  "SMALLINT UNSIGNED",  0,          UINT16_MAX,     tsCompressSmallint,  tsDecompressSmallint,  getStatics_u16},
   {TSDB_DATA_TYPE_UINT,      12, INT_BYTES,    "INT UNSIGNED",       0,          UINT32_MAX,     tsCompressInt,       tsDecompressInt,       getStatics_u32},
   {TSDB_DATA_TYPE_UBIGINT,   15, LONG_BYTES,   "BIGINT UNSIGNED",    0,          UINT64_MAX,     tsCompressBigint,    tsDecompressBigint,    getStatics_u64},
-  {TSDB_DATA_TYPE_JSON,      4,  CHAR_BYTES,   "JSON",               INT8_MIN,          INT8_MAX,              tsCompressTinyint,    tsDecompressTinyint,    getStatics_i8},
+  {TSDB_DATA_TYPE_JSON,      4,TSDB_MAX_NCHAR_LEN,   "JSON", INT8_MIN,          TSDB_MAX_NCHAR_LEN,              tsCompressString,    tsDecompressString,    getStatics_nchr},
 };
 
 char tTokenTypeSwitcher[13] = {

src/kit/shell/src/shellEngine.c

@@ -676,6 +676,7 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:
       shellPrintNChar(val, length, width);
       break;
     case TSDB_DATA_TYPE_TIMESTAMP:

src/query/src/qExecutor.c

@@ -4290,6 +4290,7 @@ static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows,
                                                                  colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0);
 }
 
+}
 static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) {
   SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
   SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
@@ -4312,7 +4313,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
         *compLen += compSizes[col];
         compSizes[col] = htonl(compSizes[col]);
       } else {
-        memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows);
+        memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows);   // todo json
         data += pColRes->info.bytes * pRes->info.rows;
       }
     }
@@ -4325,6 +4326,10 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
         *compLen += compSizes[col];
         compSizes[col] = htonl(compSizes[col]);
       } else {
+        if(pColRes->info.type == TSDB_DATA_TYPE_JSON){      // todo json
+          //pColRes->info.bytes =
+
+        }
         memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows);
         data += pColRes->info.bytes * numOfRows;
       }

src/tsdb/src/tsdbMeta.c

@@ -210,7 +210,13 @@ void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_
     return NULL;  // No matched tag volumn
   }
 
-  char *val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId);
+  char *val = NULL;
+  if (type == TSDB_DATA_TYPE_JSON){
+    val = kvRowValues(((STable*)pTable)->tagVal);
+  }else{
+    val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId);
+  }
+
   assert(type == pCol->type && bytes >= pCol->bytes);
 
   // if (val != NULL && IS_VAR_DATA_TYPE(type)) {