fea:add select json logic

include/util/tdef.h

@@ -232,7 +232,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAGS           128
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
-#define TSDB_MAX_JSON_TAG_LEN 16384
+#define TSDB_MAX_JSON_TAG_LEN (16384 + VARSTR_HEADER_SIZE)
 
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32

source/client/src/clientImpl.c

@@ -268,7 +268,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
 
     if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) {
       pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
-    } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+    } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
       pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
     }
 
@@ -745,6 +745,105 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
+#include "cJSON.h"
+static char* parseTagDatatoJson(void *p){
+  char* string = NULL;
+  cJSON *json = cJSON_CreateObject();
+  if (json == NULL)
+  {
+    goto end;
+  }
+
+  int16_t nCols = kvRowNCols(p);
+  char tagJsonKey[256] = {0};
+  for (int j = 0; j < nCols; ++j) {
+    SColIdx * pColIdx = kvRowColIdxAt(p, j);
+    void* val = (kvRowColVal(p, pColIdx));
+    if (j == 0){
+      if(*(char*)val == TSDB_DATA_TYPE_NULL){
+        string = taosMemoryCalloc(1, 8);
+        sprintf(varDataVal(string), "%s", TSDB_DATA_NULL_STR_L);
+        varDataSetLen(string, strlen(varDataVal(string)));
+        goto end;
+      }
+      continue;
+    }
+
+    // json key  encode by binary
+    memset(tagJsonKey, 0, sizeof(tagJsonKey));
+    memcpy(tagJsonKey, varDataVal(val), varDataLen(val));
+    // json value
+    val += varDataTLen(val);
+    char* realData = POINTER_SHIFT(val, CHAR_BYTES);
+    char type = *(char*)val;
+    if(type == TSDB_DATA_TYPE_NULL) {
+      cJSON* value = cJSON_CreateNull();
+      if (value == NULL)
+      {
+        goto end;
+      }
+      cJSON_AddItemToObject(json, tagJsonKey, value);
+    }else if(type == TSDB_DATA_TYPE_NCHAR) {
+      cJSON* value = NULL;
+      if (varDataLen(realData) > 0){
+        char *tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1);
+        int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(realData), varDataLen(realData), tagJsonValue);
+        if (length < 0) {
+          tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+                    (char*)val);
+          taosMemoryFree(tagJsonValue);
+          goto end;
+        }
+        value = cJSON_CreateString(tagJsonValue);
+        taosMemoryFree(tagJsonValue);
+        if (value == NULL)
+        {
+          goto end;
+        }
+      }else if(varDataLen(realData) == 0){
+        value = cJSON_CreateString("");
+      }else{
+        ASSERT(0);
+      }
+
+      cJSON_AddItemToObject(json, tagJsonKey, value);
+    }else if(type == TSDB_DATA_TYPE_DOUBLE){
+      double jsonVd = *(double*)(realData);
+      cJSON* value = cJSON_CreateNumber(jsonVd);
+      if (value == NULL)
+      {
+        goto end;
+      }
+      cJSON_AddItemToObject(json, tagJsonKey, value);
+    }else if(type == TSDB_DATA_TYPE_BIGINT){
+      int64_t jsonVd = *(int64_t*)(realData);
+      cJSON* value = cJSON_CreateNumber((double)jsonVd);
+      if (value == NULL)
+      {
+        goto end;
+      }
+      cJSON_AddItemToObject(json, tagJsonKey, value);
+    }else if (type == TSDB_DATA_TYPE_BOOL) {
+      char jsonVd = *(char*)(realData);
+      cJSON* value = cJSON_CreateBool(jsonVd);
+      if (value == NULL)
+      {
+        goto end;
+      }
+      cJSON_AddItemToObject(json, tagJsonKey, value);
+    }
+    else{
+      tscError("unsupportted json value");
+    }
+
+  }
+  string = cJSON_PrintUnformatted(json);
+end:
+  cJSON_Delete(json);
+  return string;
+}
+#include "tdataformat.h"
+
 static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) {
   for (int32_t i = 0; i < numOfCols; ++i) {
     int32_t type = pResultInfo->fields[i].type;
@@ -775,9 +874,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
 
       pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
       pResultInfo->row[i] = pResultInfo->pCol[i].pData;
-    }
-
-    if (type == TSDB_DATA_TYPE_JSON) {
+    }else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) {
       char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
       if (p == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
@@ -789,48 +886,51 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
       for (int32_t j = 0; j < numOfRows; ++j) {
         if (pCol->offset[j] != -1) {
           char* pStart = pCol->offset[j] + pCol->pData;
-
-          int32_t jsonInnerType = *pStart;
-          char*   jsonInnerData = pStart + CHAR_BYTES;
-          char    dst[TSDB_MAX_JSON_TAG_LEN] = {0};
-          if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
-            sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
-            varDataSetLen(dst, strlen(varDataVal(dst)));
-          } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) {
-            int32_t length =
-                taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
-
-            if (length <= 0) {
-              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
-                       varDataVal(jsonInnerData));
-              length = 0;
-            }
-            varDataSetLen(dst, length);
-          } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) {  // value -> "value"
-            *(char*)varDataVal(dst) = '\"';
-            int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
-                                           varDataVal(dst) + CHAR_BYTES);
-            if (length <= 0) {
-              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
-                       varDataVal(jsonInnerData));
-              length = 0;
-            }
-            varDataSetLen(dst, length + CHAR_BYTES * 2);
-            *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
-          } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
-            double jsonVd = *(double*)(jsonInnerData);
-            sprintf(varDataVal(dst), "%.9lf", jsonVd);
-            varDataSetLen(dst, strlen(varDataVal(dst)));
-          } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) {
-            int64_t jsonVd = *(int64_t*)(jsonInnerData);
-            sprintf(varDataVal(dst), "%" PRId64, jsonVd);
-            varDataSetLen(dst, strlen(varDataVal(dst)));
-          } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
-            sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
-            varDataSetLen(dst, strlen(varDataVal(dst)));
-          } else {
-            ASSERT(0);
-          }
+          char dst[TSDB_MAX_JSON_TAG_LEN] = {0};
+          char *jsonString = parseTagDatatoJson(pStart);
+          STR_TO_VARSTR(dst, jsonString);
+          taosMemoryFree(jsonString);
+//          int32_t jsonInnerType = *pStart;
+//          char*   jsonInnerData = pStart + CHAR_BYTES;
+//          char    dst[TSDB_MAX_JSON_TAG_LEN] = {0};
+//          if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
+//            sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
+//            varDataSetLen(dst, strlen(varDataVal(dst)));
+//          } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) {
+//            int32_t length =
+//                taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
+//
+//            if (length <= 0) {
+//              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+//                       varDataVal(jsonInnerData));
+//              length = 0;
+//            }
+//            varDataSetLen(dst, length);
+//          } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) {  // value -> "value"
+//            *(char*)varDataVal(dst) = '\"';
+//            int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
+//                                           varDataVal(dst) + CHAR_BYTES);
+//            if (length <= 0) {
+//              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+//                       varDataVal(jsonInnerData));
+//              length = 0;
+//            }
+//            varDataSetLen(dst, length + CHAR_BYTES * 2);
+//            *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
+//          } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
+//            double jsonVd = *(double*)(jsonInnerData);
+//            sprintf(varDataVal(dst), "%.9lf", jsonVd);
+//            varDataSetLen(dst, strlen(varDataVal(dst)));
+//          } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) {
+//            int64_t jsonVd = *(int64_t*)(jsonInnerData);
+//            sprintf(varDataVal(dst), "%" PRId64, jsonVd);
+//            varDataSetLen(dst, strlen(varDataVal(dst)));
+//          } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
+//            sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
+//            varDataSetLen(dst, strlen(varDataVal(dst)));
+//          } else {
+//            ASSERT(0);
+//          }
 
           if (len + varDataTLen(dst) > colLength[i]) {
             p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));

source/common/src/tdatablock.c

@@ -116,18 +116,21 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
 
   int32_t type = pColumnInfoData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
-    int32_t dataLen = varDataTLen(pData);
+    int32_t dataLen = 0;
     if (type == TSDB_DATA_TYPE_JSON) {
-      if (*pData == TSDB_DATA_TYPE_NULL) {
-        dataLen = 0;
-      } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
-        dataLen = varDataTLen(pData + CHAR_BYTES);
-      } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
-        dataLen = LONG_BYTES;
-      } else if (*pData == TSDB_DATA_TYPE_BOOL) {
-        dataLen = CHAR_BYTES;
-      }
-      dataLen += CHAR_BYTES;
+//      if (*pData == TSDB_DATA_TYPE_NULL) {
+//        dataLen = 0;
+//      } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
+//        dataLen = varDataTLen(pData + CHAR_BYTES);
+//      } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+//        dataLen = LONG_BYTES;
+//      } else if (*pData == TSDB_DATA_TYPE_BOOL) {
+//        dataLen = CHAR_BYTES;
+//      }
+//      dataLen += CHAR_BYTES;
+      dataLen = kvRowLen(pData);
+    }else {
+      dataLen = varDataTLen(pData);
     }
 
     SVarColAttr* pAttr = &pColumnInfoData->varmeta;

source/libs/executor/src/scanoperator.c

@@ -1467,7 +1467,12 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
         STR_TO_VARSTR(str, mr.me.name);
         colDataAppend(pDst, count, str, false);
       } else { // it is a tag value
-        const char* p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId);
+        const char* p = NULL;
+        if(pDst->info.type == TSDB_DATA_TYPE_JSON){
+          p = mr.me.ctbEntry.pTags;
+        }else{
+          p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId);
+        }
         colDataAppend(pDst, count, p, (p == NULL));
       }
     }

tools/shell/src/shellEngine.c

@@ -315,6 +315,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:
       memcpy(buf, val, length);
       buf[length] = 0;
       taosFprintfFile(pFile, "\'%s\'", buf);
@@ -384,19 +385,25 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) {
   while (pos < length) {
     TdWchar wc;
     int32_t bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
-    if (bytes == 0) {
+    if (bytes <= 0) {
       break;
     }
-    pos += bytes;
-    if (pos > length) {
+
+    if (pos + bytes > length) {
       break;
     }
-
+    int w = 0;
 #ifdef WINDOWS
-    int32_t w = bytes;
+    w = bytes;
 #else
-    int32_t w = taosWcharWidth(wc);
+    if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
+      w = bytes;
+    }else{
+      w = taosWcharWidth(wc);
+    }
 #endif
+    pos += bytes;
+
     if (w <= 0) {
       continue;
     }
@@ -496,6 +503,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:
       shellPrintNChar(val, length, width);
       break;
     case TSDB_DATA_TYPE_TIMESTAMP:
@@ -604,7 +612,6 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
     case TSDB_DATA_TYPE_DOUBLE:
       return TMAX(25, width);
 
-    case TSDB_DATA_TYPE_JSON:
     case TSDB_DATA_TYPE_BINARY:
       if (field->bytes > shell.args.displayWidth) {
         return TMAX(shell.args.displayWidth, width);
@@ -612,7 +619,8 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
         return TMAX(field->bytes, width);
       }
 
-    case TSDB_DATA_TYPE_NCHAR: {
+    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON: {
       int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
       if (bytes > shell.args.displayWidth) {
         return TMAX(shell.args.displayWidth, width);