refactor: add json tag function for sql parser

include/common/tdataformat.h

@@ -608,7 +608,7 @@ void    tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
 void    tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow  tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
-static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) {
+static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
     SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
@@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
 
   pBuilder->nCols++;
 
-  int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
   if (tlen > pBuilder->alloc - pBuilder->size) {
     while (tlen > pBuilder->alloc - pBuilder->size) {
       pBuilder->alloc *= 2;

include/common/ttypes.h

@@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor {
                      int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 
-extern tDataTypeDescriptor tDataTypes[15];
+extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
 
 bool isValidDataType(int32_t type);
 

include/util/taoserror.h

@@ -599,6 +599,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST    TAOS_DEF_ERROR_CODE(0, 0x2624)
 #define TSDB_CODE_PAR_INVALID_OPTION_UNIT       TAOS_DEF_ERROR_CODE(0, 0x2625)
 #define TSDB_CODE_PAR_INVALID_KEEP_UNIT         TAOS_DEF_ERROR_CODE(0, 0x2626)
+#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG         TAOS_DEF_ERROR_CODE(0, 0x2627)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR              TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tdef.h

@@ -254,6 +254,14 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAGS           128
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
+#define TSDB_MAX_JSON_TAG_LEN   16384
+#define TSDB_JSON_PLACEHOLDER      0x7F
+#define TSDB_JSON_null             0x00
+#define TSDB_JSON_KEY_NULL         0x00
+#define TSDB_JSON_NOT_NULL         0x01
+#define TSDB_JSON_NULL             0x00
+
+
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32
 #define TSDB_USET_PASSWORD_LEN 129

include/util/tjson.h

@@ -76,6 +76,7 @@ char* tjsonToString(const SJson* pJson);
 char* tjsonToUnformattedString(const SJson* pJson);
 
 SJson* tjsonParse(const char* pStr);
+bool   tjsonIsObject(const SJson* pJson);
 
 #ifdef __cplusplus
 }

include/util/tutil.h

@@ -26,8 +26,6 @@ extern "C" {
 #endif
 
 int32_t strdequote(char *src);
-int32_t strndequote(char *dst, const char *z, int32_t len);
-int32_t strRmquote(char *z, int32_t len);
 size_t  strtrim(char *src);
 char   *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
 char  **strsplit(char *src, const char *delim, int32_t *num);

source/common/src/ttypes.c

@@ -365,7 +365,7 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i
   *maxIndex = 0;
 }
 
-tDataTypeDescriptor tDataTypes[15] = {
+tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
     {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
     {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
     {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint,
@@ -388,6 +388,7 @@ tDataTypeDescriptor tDataTypes[15] = {
     {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
     {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint,
      getStatics_u64},
+    {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
 };
 
 char tTokenTypeSwitcher[13] = {

source/common/src/tvariant.c

@@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) {
     }
     case TSDB_DATA_TYPE_BINARY: {
       pVar->pz = strndup(z, n);
-      pVar->nLen = strRmquote(pVar->pz, n);
+      //pVar->nLen = strRmquote(pVar->pz, n);
       break;
     }
     case TSDB_DATA_TYPE_TIMESTAMP: {

source/libs/parser/inc/parUtil.h

@@ -47,6 +47,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta);
 int32_t  getNumOfColumns(const STableMeta* pTableMeta);
 int32_t  getNumOfTags(const STableMeta* pTableMeta);
 STableComInfo getTableInfo(const STableMeta* pTableMeta);
+int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId);
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
 

source/libs/parser/src/parAstCreater.c

@@ -47,6 +47,7 @@ void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt) {
 }
 
 static void trimEscape(SToken* pName) {
+  // todo need to deal with `ioo``ii` -> ioo`ii
   if (NULL != pName && pName->n > 1 && '`' == pName->z[0]) {
     pName->z += 1;
     pName->n -= 2;

source/libs/parser/src/parInsert.c

@@ -377,35 +377,15 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha
     return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
   }
 
-  if (IS_NUMERIC_TYPE(type) && pToken->n == 0) {
-    return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z);
-  }
-
   // Remove quotation marks
   if (TK_NK_STRING == pToken->type) {
     if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) {
       return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z);
     }
 
-    // delete escape character: \\, \', \"
-    char delim = pToken->z[0];
-    int32_t cnt = 0;
-    int32_t j = 0;
-    for (uint32_t k = 1; k < pToken->n - 1; ++k) {
-      if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) {
-        tmpTokenBuf[j] = pToken->z[k + 1];
-        cnt++;
-        j++;
-        k++;
-        continue;
-      }
-      tmpTokenBuf[j] = pToken->z[k];
-      j++;
-    }
-
-    tmpTokenBuf[j] = 0;
+    int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
     pToken->z = tmpTokenBuf;
-    pToken->n -= 2 + cnt;
+    pToken->n = len;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -582,6 +562,13 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int
       return func(pMsgBuf, pToken->z, pToken->n, param);
     }
 
+    case TSDB_DATA_TYPE_JSON: {
+      if(pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+        return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z);
+      }
+      return func(pMsgBuf, pToken->z, pToken->n, param);
+    }
+
     case TSDB_DATA_TYPE_TIMESTAMP: {
       int64_t tmpVal;
       if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) {
@@ -702,8 +689,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
     qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar);
   }
 
-  memset(&pColList->boundColumns[pColList->numOfBound], 0,
-         sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
+  if(pColList->numOfCols > pColList->numOfBound){
+    memset(&pColList->boundColumns[pColList->numOfBound], 0,
+           sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -720,22 +709,31 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi
   int8_t  type = pa->schema->type;
   int16_t colId = pa->schema->colId;
 
+  if(TSDB_DATA_TYPE_JSON == type){
+    return parseJsontoTagData(value, pa->builder, pMsgBuf, colId);
+  }
+
+  if (value == NULL) {  // it is a null data
+    // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx);
+    return TSDB_CODE_SUCCESS;
+  }
+
   if (TSDB_DATA_TYPE_BINARY == type) {
     STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len);
-    tdAddColToKVRow(pa->builder, colId, type, pa->buf);
+    tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf));
   } else if (TSDB_DATA_TYPE_NCHAR == type) {
     // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
     int32_t output = 0;
     if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
       char buf[512] = {0};
       snprintf(buf, tListLen(buf), "%s", strerror(errno));
-      return buildSyntaxErrMsg(pMsgBuf, buf, value);;
+      return buildSyntaxErrMsg(pMsgBuf, buf, value);
     }
 
     varDataSetLen(pa->buf, output);
-    tdAddColToKVRow(pa->builder, colId, type, pa->buf);
+    tdAddColToKVRow(pa->builder, colId, varDataTLen(pa->buf));
   } else {
-    tdAddColToKVRow(pa->builder, colId, type, value);
+    tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]);
   }
 
   return TSDB_CODE_SUCCESS;

source/libs/parser/src/parTranslater.c

@@ -1542,6 +1542,17 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
+  SNode* pNode;
+  FOREACH(pNode, pStmt->pTags) {
+    SColumnDefNode* pCol = (SColumnDefNode*)pNode;
+    if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){
+      return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs) {
   if (NULL == pFuncs) {
     return TSDB_CODE_SUCCESS;
@@ -1577,6 +1588,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
   if (TSDB_CODE_SUCCESS == code) {
     code = checkRangeOption(pCxt, "delay", pStmt->pOptions->pDelay, TSDB_MIN_DB_DELAY, TSDB_MAX_DB_DELAY);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = checkTableTags(pCxt, pStmt);
+  }
   return code;
 }
 
@@ -2893,11 +2907,19 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS
   }
   SVariant var;
   valueNodeToVariant(pVal, &var);
+  if(pSchema->type == TSDB_DATA_TYPE_JSON){
+    if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz);
+    }
+    return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId);
+  }
+
   char    tagVal[TSDB_MAX_TAGS_LEN] = {0};
   int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true);
   if (TSDB_CODE_SUCCESS == code) {
-    tdAddColToKVRow(pBuilder, pSchema->colId, pSchema->type, tagVal);
+    tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]);
   }
+
   return code;
 }
 

source/libs/parser/src/parUtil.c

@@ -14,6 +14,7 @@
  */
 
 #include "parUtil.h"
+#include "cJSON.h"
 
 static char* getSyntaxErrFormat(int32_t errCode) {
   switch (errCode) {
@@ -95,6 +96,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "Invalid option %s unit: %c, only m, h, d allowed";
     case TSDB_CODE_PAR_INVALID_KEEP_UNIT:
       return "Invalid option keep unit: %c, %c, %c, only m, h, d allowed";
+    case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG:
+      return "Only one tag if there is a json tag";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:
@@ -195,24 +198,180 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) {
 }
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
-  // delete escape character: \\, \', \"
+  if (len <=0 || dlen <= 0) return 0;
+
   char delim = src[0];
-  int32_t cnt = 0;
   int32_t j = 0;
   for (uint32_t k = 1; k < len - 1; ++k) {
     if (j >= dlen) {
-      break;
+      dst[j - 1] = '\0';
+      return j;
     }
-    if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) {
+    if (src[k] == delim && src[k + 1] == delim) {   // deal with "", ''
       dst[j] = src[k + 1];
-      cnt++;
       j++;
       k++;
       continue;
     }
+
+    if (src[k] == '\\') {   // deal with escape character
+      if(src[k+1] == 'n'){
+        dst[j] = '\n';
+      }else if(src[k+1] == 'r'){
+        dst[j] = '\r';
+      }else if(src[k+1] == 't'){
+        dst[j] = '\t';
+      }else if(src[k+1] == '\\'){
+        dst[j] = '\\';
+      }else if(src[k+1] == '\''){
+        dst[j] = '\'';
+      }else if(src[k+1] == '"'){
+        dst[j] = '"';
+      }else if(src[k+1] == '%' || src[k+1] == '_'){
+        dst[j++] = src[k];
+        dst[j] = src[k+1];
+      }else{
+        dst[j] = src[k+1];
+      }
+      j++;
+      k++;
+      continue;
+    }
+
     dst[j] = src[k];
     j++;
   }
   dst[j] = '\0';
   return j;
 }
+
+static bool isValidateTag(char *input) {
+  if (!input) return false;
+  for (size_t i = 0; i < strlen(input); ++i) {
+    if (isprint(input[i]) == 0) return false;
+  }
+  return true;
+}
+
+int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
+  // set json NULL data
+  uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL;
+  uint8_t jsonNULL = TSDB_JSON_NULL;
+  int jsonIndex = startColId + 1;
+  tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES);   // add json null type
+  if (!json || strtrim(json) == 0 || strncasecmp(json, "null", 4) == 0){
+    tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);   // add json null value
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // set json real data
+  cJSON *root = cJSON_Parse(json);
+  if (root == NULL){
+    return buildSyntaxErrMsg(pMsgBuf, "json parse error", json);
+  }
+
+  int size = cJSON_GetArraySize(root);
+  if(!cJSON_IsObject(root)){
+    return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json);
+  }
+
+  int retCode = 0;
+  SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
+  for(int i = 0; i < size; i++) {
+    cJSON* item = cJSON_GetArrayItem(root, i);
+    if (!item) {
+      qError("json inner error:%d", i);
+      return buildSyntaxErrMsg(pMsgBuf, "json inner error", json);
+      goto end;
+    }
+
+    char *jsonKey = item->string;
+    if(!isValidateTag(jsonKey)){
+      retCode =  buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
+      goto end;
+    }
+//    if(strlen(jsonKey) > TSDB_MAX_JSON_KEY_LEN){
+//      tscError("json key too long error");
+//      retCode =  tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL);
+//      goto end;
+//    }
+    size_t keyLen = strlen(jsonKey);
+    if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
+      continue;
+    }
+
+    if(taosHashGetSize(keyHash) == 0){
+      uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES);   // add json type
+    }
+    // json key encode by binary
+    void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1);
+    if(!tagKey) {
+      retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      goto end;
+    }
+    strncpy(varDataVal(tagKey), jsonKey, keyLen);
+    taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless
+
+    varDataSetLen(tagKey, keyLen);
+    tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey));   // add json key
+    taosMemoryFree(tagKey);
+
+    if(item->type == cJSON_String){     // add json value  format: type|data
+      char *jsonValue = item->valuestring;
+      int32_t valLen = (int32_t)strlen(jsonValue);
+      char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1);
+      if(!tagVal) {
+        retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
+        goto end;
+      } ;
+      tagVal[0] = TSDB_DATA_TYPE_NCHAR;
+      char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES);
+      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData),
+                                                  (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
+        qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
+        retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
+        taosMemoryFree(tagVal);
+        goto end;
+      }
+
+      varDataSetLen(tagData, valLen);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData));
+      taosMemoryFree(tagVal);
+    }else if(item->type == cJSON_Number){
+      if(!isfinite(item->valuedouble)){
+        qError("json value is invalidate");
+        retCode =  buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
+        goto end;
+      }
+      char tagVal[LONG_BYTES + CHAR_BYTES] = {0};
+      tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT
+                                                                        : TSDB_DATA_TYPE_DOUBLE;
+      char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
+      if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble;
+      else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES);
+    }else if(item->type == cJSON_True || item->type == cJSON_False){
+      char tagVal[CHAR_BYTES + CHAR_BYTES] = {0};
+      tagVal[0] = TSDB_DATA_TYPE_BOOL;
+      tagVal[1] = (char)(item->valueint);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES);
+    }else if(item->type == cJSON_NULL){
+      char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL};
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES);
+    }
+    else{
+      retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
+      goto end;
+    }
+  }
+
+  if(taosHashGetSize(keyHash) == 0){  // set json NULL true
+    tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);
+  }
+
+end:
+  taosHashCleanup(keyHash);
+  cJSON_Delete(root);
+  return retCode;
+}
\ No newline at end of file

source/libs/scalar/src/filter.c

@@ -57,7 +57,7 @@ OptrStr gOptrStr[] = {
   {OP_TYPE_IS_NOT_UNKNOWN,                 "not unknown"},
 
   // json operator
-  {OP_TYPE_JSON_GET_VALUE,                 "json get"},
+  {OP_TYPE_JSON_GET_VALUE,                 "->"},
   {OP_TYPE_JSON_CONTAINS,                  "json contains"}
 };
 

source/util/src/tjson.c

@@ -276,3 +276,4 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void
 }
 
 SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); }
+bool tjsonIsObject(const SJson* pJson) { return cJSON_IsObject(pJson); }
\ No newline at end of file

source/util/src/tutil.c

@@ -47,65 +47,6 @@ int32_t strdequote(char *z) {
   return j + 1;  // only one quote, do nothing
 }
 
-int32_t strRmquote(char *z, int32_t len) {
-  // delete escape character: \\, \', \"
-  char delim = z[0];
-  if (delim != '\'' && delim != '\"') {
-    return len;
-  }
-
-  int32_t cnt = 0;
-  int32_t j = 0;
-  for (uint32_t k = 1; k < len - 1; ++k) {
-    if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) {
-      if (z[k] == '\\' && z[k + 1] == '_') {
-        // match '_' self
-      } else {
-        z[j] = z[k + 1];
-        cnt++;
-        j++;
-        k++;
-        continue;
-      }
-    }
-
-    z[j] = z[k];
-    j++;
-  }
-
-  z[j] = 0;
-
-  return len - 2 - cnt;
-}
-
-int32_t strndequote(char *dst, const char *z, int32_t len) {
-  assert(dst != NULL);
-  if (z == NULL || len == 0) {
-    return 0;
-  }
-
-  int32_t quote = z[0];
-  int32_t i = 1, j = 0;
-
-  while (z[i] != 0) {
-    if (z[i] == quote) {
-      if (z[i + 1] == quote) {
-        dst[j++] = (char)quote;
-        i++;
-      } else {
-        dst[j++] = 0;
-        return (j - 1);
-      }
-    } else {
-      dst[j++] = z[i];
-    }
-
-    i++;
-  }
-
-  return j + 1;  // only one quote, do nothing
-}
-
 size_t strtrim(char *z) {
   int32_t i = 0;
   int32_t j = 0;

tools/shell/src/shellEngine.c

@@ -226,63 +226,27 @@ int32_t shellRunCommand(TAOS *con, char *command) {
     }
   }
 
-  bool esc = false;
-  char quote = 0, *cmd = command, *p = command;
+  char quote = 0, *cmd = command;
   for (char c = *command++; c != 0; c = *command++) {
-    if (esc) {
-      switch (c) {
-        case 'n':
-          c = '\n';
-          break;
-        case 'r':
-          c = '\r';
-          break;
-        case 't':
-          c = '\t';
-          break;
-        case 'G':
-          *p++ = '\\';
-          break;
-        case '\'':
-        case '"':
-          if (quote) {
-            *p++ = '\\';
-          }
-          break;
-      }
-      *p++ = c;
-      esc = false;
+    if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) {
+      command ++;
       continue;
     }
 
-    if (c == '\\') {
-      if (quote != 0 && (*command == '_' || *command == '\\')) {
-        // DO nothing
-      } else {
-        esc = true;
-        continue;
-      }
-    }
-
     if (quote == c) {
       quote = 0;
-    } else if (quote == 0 && (c == '\'' || c == '"')) {
+    } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) {
       quote = c;
-    }
-
-    *p++ = c;
-    if (c == ';' && quote == 0) {
-      c = *p;
-      *p = 0;
+    } else if (c == ';' && quote == 0) {
+      c = *command;
+      *command = 0;
       if (shellRunSingleCommand(con, cmd) < 0) {
         return -1;
       }
-      *p = c;
-      p = cmd;
+      *command = c;
+      cmd = command;
     }
   }
-
-  *p = 0;
   return shellRunSingleCommand(con, cmd);
 }
 
@@ -573,19 +537,23 @@ static void shellPrintNChar(const char *str, int length, int width) {
   while (pos < length) {
     TdWchar wc;
     int     bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
-    if (bytes == 0) {
+    if (bytes <= 0) {
       break;
     }
-    pos += bytes;
-    if (pos > length) {
+    if (pos + bytes > length) {
       break;
     }
-
+    int w = 0;
 #ifdef WINDOWS
-    int w = bytes;
+    w = bytes;
 #else
-    int w = taosWcharWidth(wc);
+    if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
+      w = bytes;
+    }else{
+      w = taosWcharWidth(wc);
+    }
 #endif
+    pos += bytes;
     if (w <= 0) {
       continue;
     }
@@ -679,6 +647,7 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_JSON:
       shellPrintNChar(val, length, width);
       break;
     case TSDB_DATA_TYPE_TIMESTAMP: