From eed7ebdde1ca045d07caaa5f1114eae9a67dd71f Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Sun, 9 Jan 2022 18:10:31 +0800
Subject: [PATCH] 
 [TD-12252]<feature>(connector,query,insert,other,tools,taosAdapter):deal with
 escape character

---
 src/client/inc/tscUtil.h              |   2 -
 src/client/src/tscParseInsert.c       |  49 ++++-------
 src/client/src/tscParseLineProtocol.c |  42 +++++-----
 src/client/src/tscParseOpenTSDB.c     |  20 ++---
 src/client/src/tscSQLParser.c         |  63 +++++++-------
 src/client/src/tscSql.c               |   2 +-
 src/client/src/tscUtil.c              |  98 ++--------------------
 src/common/src/tname.c                |   6 +-
 src/common/src/tvariant.c             |   2 +-
 src/inc/taosdef.h                     |   4 +-
 src/kit/shell/src/shellCommand.c      |   4 +-
 src/kit/shell/src/shellEngine.c       |  85 +++++++------------
 src/query/src/qSqlParser.c            |   2 +-
 src/util/inc/tutil.h                  |   5 +-
 src/util/src/ttokenizer.c             |   7 ++
 src/util/src/tutil.c                  | 115 +++++++++++++-------------
 16 files changed, 186 insertions(+), 320 deletions(-)

diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h
index dd9db51795..b583ee0b01 100644
--- a/src/client/inc/tscUtil.h
+++ b/src/client/inc/tscUtil.h
@@ -258,8 +258,6 @@ void tscColumnListCopyAll(SArray* dst, const SArray* src);
 
 void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar, bool convertJson);
 
-void tscDequoteAndTrimToken(SStrToken* pToken);
-void tscRmEscapeAndTrimToken(SStrToken* pToken);
 int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded);
 
 void tscIncStreamExecutionCount(void* pStream);
diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c
index de974bec46..cff03f568a 100644
--- a/src/client/src/tscParseInsert.c
+++ b/src/client/src/tscParseInsert.c
@@ -481,32 +481,12 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
 
     // Remove quotation marks
     if (TK_STRING == sToken.type) {
-      // delete escape character: \\, \', \"
-      char delim = sToken.z[0];
-
-      int32_t cnt = 0;
-      int32_t j = 0;
       if (sToken.n >= TSDB_MAX_BYTES_PER_ROW) {
         return tscSQLSyntaxErrMsg(pInsertParam->msg, "too long string", sToken.z);
       }
-
-      for (uint32_t k = 1; k < sToken.n - 1; ++k) {
-        if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) {
-          tmpTokenBuf[j] = sToken.z[k + 1];
-
-          cnt++;
-          j++;
-          k++;
-          continue;
-        }
-
-        tmpTokenBuf[j] = sToken.z[k];
-        j++;
-      }
-
-      tmpTokenBuf[j] = 0;
+      strncpy(tmpTokenBuf, sToken.z, sToken.n);
+      sToken.n = stringProcess(tmpTokenBuf, sToken.n);
       sToken.z = tmpTokenBuf;
-      sToken.n -= 2 + cnt;
     }
 
     bool    isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX);
@@ -1057,10 +1037,12 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
         break;
       }
 
+      char* tmp = NULL;
       // Remove quotation marks
       if (TK_STRING == sToken.type) {
-        sToken.z++;
-        sToken.n -= 2;
+        tmp = strndup(sToken.z, sToken.n);
+        sToken.n = stringProcess(tmp, sToken.n);
+        sToken.z = tmp;
       }
 
       char tagVal[TSDB_MAX_TAGS_LEN] = {0};
@@ -1068,6 +1050,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
       if (code != TSDB_CODE_SUCCESS) {
         tdDestroyKVRowBuilder(&kvRowBuilder);
         tscDestroyBoundColumnInfo(&spd);
+        tfree(tmp);
         return code;
       }
 
@@ -1078,18 +1061,18 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
         if(sToken.n > TSDB_MAX_JSON_TAGS_LEN/TSDB_NCHAR_SIZE){
           tdDestroyKVRowBuilder(&kvRowBuilder);
           tscDestroyBoundColumnInfo(&spd);
+          tfree(tmp);
           return tscSQLSyntaxErrMsg(pInsertParam->msg, "json tag too long", NULL);
         }
-        char* json = strndup(sToken.z, sToken.n);
-        code = parseJsontoTagData(json, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId);
+        code = parseJsontoTagData(sToken.z, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId);
         if (code != TSDB_CODE_SUCCESS) {
           tdDestroyKVRowBuilder(&kvRowBuilder);
           tscDestroyBoundColumnInfo(&spd);
-          tfree(json);
+          tfree(tmp);
           return code;
         }
-        tfree(json);
       }
+      tfree(tmp);
     }
     tscDestroyBoundColumnInfo(&spd);
 
@@ -1246,12 +1229,8 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
     strncpy(tmpTokenBuf, sToken.z, sToken.n);
     sToken.z = tmpTokenBuf;
 
-    if (TK_STRING == sToken.type) {
-      tscDequoteAndTrimToken(&sToken);
-    }
-
-    if (TK_ID == sToken.type) {
-      tscRmEscapeAndTrimToken(&sToken);
+    if (TK_STRING == sToken.type || TK_ID == sToken.type) {
+      sToken.n = stringProcess(sToken.z, sToken.n);
     }
 
     if (sToken.type == TK_RP) {
@@ -1371,7 +1350,7 @@ _clean:
 static int32_t getFileFullPath(SStrToken* pToken, char* output) {
   char path[PATH_MAX] = {0};
   strncpy(path, pToken->z, pToken->n);
-  strdequote(path);
+  stringProcess(path, strlen(path));
 
   wordexp_t full_path;
   if (wordexp(path, &full_path, 0) != 0) {
diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c
index 9f69a8a66d..ea74e4e018 100644
--- a/src/client/src/tscParseLineProtocol.c
+++ b/src/client/src/tscParseLineProtocol.c
@@ -20,7 +20,7 @@
 #include "tscParseLine.h"
 
 typedef struct  {
-  char sTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE];
+  char sTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE];
   SHashObj* tagHash;
   SHashObj* fieldHash;
   SArray* tags; //SArray<SSchema>
@@ -68,13 +68,13 @@ typedef enum {
 } ESchemaAction;
 
 typedef struct {
-  char sTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE];
+  char sTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE];
   SArray* tags; //SArray<SSchema>
   SArray* fields; //SArray<SSchema>
 } SCreateSTableActionInfo;
 
 typedef struct {
-  char sTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE];
+  char sTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE];
   SSchema* field;
 } SAlterSTableActionInfo;
 
@@ -161,14 +161,14 @@ static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableNa
   }
 
   SStringBuilder sb; memset(&sb, 0, sizeof(sb));
-  char sTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char sTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   strncpy(sTableName, point->stableName, strlen(point->stableName));
   //strtolower(sTableName, point->stableName);
   taosStringBuilderAppendString(&sb, sTableName);
   for (int j = 0; j < point->tagNum; ++j) {
     taosStringBuilderAppendChar(&sb, ',');
     TAOS_SML_KV* tagKv = point->tags + j;
-    char tagName[TSDB_COL_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+    char tagName[TSDB_COL_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
     strncpy(tagName, tagKv->key, strlen(tagKv->key));
     //strtolower(tagName, tagKv->key);
     taosStringBuilderAppendString(&sb, tagName);
@@ -192,8 +192,8 @@ static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableNa
 
 static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) {
   tscDebug("SML:0x%"PRIx64" taos_sml_insert build child table name", info->id);
-  char childTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE];
-  int32_t tableNameLen = TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE;
+  char childTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE];
+  int32_t tableNameLen = TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE;
   getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info);
   point->childTableName = calloc(1, tableNameLen+1);
   strncpy(point->childTableName, childTableName, tableNameLen);
@@ -251,15 +251,15 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint,
       size_t nameLen = strlen(tsSmlTagNullName);
       strncpy(tagNullName, tsSmlTagNullName, nameLen);
       addEscapeCharToString(tagNullName, (int32_t)nameLen);
-      size_t* pTagNullIdx = taosHashGet(pStableSchema->tagHash, tagNullName, nameLen + TS_ESCAPE_CHAR_SIZE);
+      size_t* pTagNullIdx = taosHashGet(pStableSchema->tagHash, tagNullName, nameLen + TS_BACKQUOTE_CHAR_SIZE);
       if (!pTagNullIdx) {
         SSchema tagNull = {0};
         tagNull.type  = TSDB_DATA_TYPE_NCHAR;
         tagNull.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
-        strncpy(tagNull.name, tagNullName, nameLen + TS_ESCAPE_CHAR_SIZE);
+        strncpy(tagNull.name, tagNullName, nameLen + TS_BACKQUOTE_CHAR_SIZE);
         taosArrayPush(pStableSchema->tags, &tagNull);
         size_t tagNullIdx = taosArrayGetSize(pStableSchema->tags) - 1;
-        taosHashPut(pStableSchema->tagHash, tagNull.name, nameLen + TS_ESCAPE_CHAR_SIZE, &tagNullIdx, sizeof(tagNullIdx));
+        taosHashPut(pStableSchema->tagHash, tagNull.name, nameLen + TS_BACKQUOTE_CHAR_SIZE, &tagNullIdx, sizeof(tagNullIdx));
       }
     }
 
@@ -295,7 +295,7 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint,
 
 static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[],
                                        SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) {
-  char fieldName[TSDB_COL_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char fieldName[TSDB_COL_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   strcpy(fieldName, pointColField->name);
 
   size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName));
@@ -315,7 +315,7 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash
         action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE;
       }
       memset(&action->alterSTable, 0,  sizeof(SAlterSTableActionInfo));
-      memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE);
+      memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE);
       action->alterSTable.field = pointColField;
       *actionNeeded = true;
     }
@@ -326,7 +326,7 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash
       action->action = SCHEMA_ACTION_ADD_COLUMN;
     }
     memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo));
-    memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE);
+    memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE);
     action->alterSTable.field = pointColField;
     *actionNeeded = true;
   }
@@ -572,7 +572,7 @@ static int32_t getSuperTableMetaFromLocalCache(TAOS* taos, char* tableName, STab
   pSql->fp = NULL;
 
   registerSqlObj(pSql);
-  char tableNameBuf[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char tableNameBuf[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   memcpy(tableNameBuf, tableName, strlen(tableName));
   SStrToken tableToken = {.z = tableNameBuf, .n = (uint32_t)strlen(tableName), .type = TK_ID};
   tGetToken(tableNameBuf, &tableToken.type);
@@ -689,7 +689,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo*
       SSchemaAction schemaAction = {0};
       schemaAction.action = SCHEMA_ACTION_CREATE_STABLE;
       memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo));
-      memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE);
+      memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE);
       schemaAction.createSTable.tags = pointSchema->tags;
       schemaAction.createSTable.fields = pointSchema->fields;
       applySchemaAction(taos, &schemaAction, info);
@@ -726,7 +726,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo*
 
       SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0);
       SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0);
-      memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN + TS_ESCAPE_CHAR_SIZE);
+      memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE);
 
       for (int j = 1; j < pointFieldSize; ++j) {
         SSchema* pointCol = taosArrayGet(pointSchema->fields, j);
@@ -1398,7 +1398,7 @@ char* addEscapeCharToString(char *str, int32_t len) {
     return NULL;
   }
   memmove(str + 1, str, len);
-  str[0] = str[len + 1] = TS_ESCAPE_CHAR;
+  str[0] = str[len + 1] = TS_BACKQUOTE_CHAR;
   str[len + 2] = '\0';
   return str;
 }
@@ -2129,7 +2129,7 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
 
-  pKV->key = calloc(len + TS_ESCAPE_CHAR_SIZE + 1, 1);
+  pKV->key = calloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1);
   memcpy(pKV->key, key, len + 1);
   addEscapeCharToString(pKV->key, len);
   tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
@@ -2227,7 +2227,7 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index
   const char *cur = *index;
   int16_t len = 0;
 
-  pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE, 1);
+  pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1);
   if (pSml->stableName == NULL){
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
@@ -2313,7 +2313,7 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs,
   }
 
   size_t childTableNameLen = strlen(tsSmlChildTableName);
-  char childTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char childTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   if (childTableNameLen != 0) {
     memcpy(childTableName, tsSmlChildTableName, childTableNameLen);
     addEscapeCharToString(childTableName, (int32_t)(childTableNameLen));
@@ -2332,7 +2332,7 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs,
     }
 
     if (!isField && childTableNameLen != 0 && strcasecmp(pkv->key, childTableName) == 0)  {
-      smlData->childTableName = malloc(pkv->length + TS_ESCAPE_CHAR_SIZE + 1);
+      smlData->childTableName = malloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1);
       memcpy(smlData->childTableName, pkv->value, pkv->length);
       addEscapeCharToString(smlData->childTableName, (int32_t)pkv->length);
       free(pkv->key);
diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c
index 76d3d38d83..4b2738e567 100644
--- a/src/client/src/tscParseOpenTSDB.c
+++ b/src/client/src/tscParseOpenTSDB.c
@@ -37,7 +37,7 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index,
   const char *cur = *index;
   uint16_t len = 0;
 
-  pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE, 1);
+  pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1);
   if (pSml->stableName == NULL) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
@@ -125,7 +125,7 @@ static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char
   }
   tfree(value);
 
-  (*pTS)->key = tcalloc(sizeof(key) + TS_ESCAPE_CHAR_SIZE, 1);
+  (*pTS)->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1);
   memcpy((*pTS)->key, key, sizeof(key));
   addEscapeCharToString((*pTS)->key, (int32_t)strlen(key));
 
@@ -196,7 +196,7 @@ static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const ch
   }
   tfree(value);
 
-  pVal->key = tcalloc(sizeof(key) + TS_ESCAPE_CHAR_SIZE, 1);
+  pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1);
   memcpy(pVal->key, key, sizeof(key));
   addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key));
   *num_kvs += 1;
@@ -240,7 +240,7 @@ static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj
     return TSDB_CODE_TSC_DUP_TAG_NAMES;
   }
 
-  pKV->key = tcalloc(len + TS_ESCAPE_CHAR_SIZE + 1, 1);
+  pKV->key = tcalloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1);
   memcpy(pKV->key, key, len + 1);
   addEscapeCharToString(pKV->key, len);
   //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
@@ -307,7 +307,7 @@ static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs,
   pkv = *pKVs;
 
   size_t childTableNameLen = strlen(tsSmlChildTableName);
-  char childTbName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char childTbName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   if (childTableNameLen != 0) {
     memcpy(childTbName, tsSmlChildTableName, childTableNameLen);
     addEscapeCharToString(childTbName, (int32_t)(childTableNameLen));
@@ -324,7 +324,7 @@ static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs,
       return ret;
     }
     if (childTableNameLen != 0 && strcasecmp(pkv->key, childTbName) == 0) {
-      *childTableName = tcalloc(pkv->length + TS_ESCAPE_CHAR_SIZE + 1, 1);
+      *childTableName = tcalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1, 1);
       memcpy(*childTableName, pkv->value, pkv->length);
       (*childTableName)[pkv->length] = '\0';
       addEscapeCharToString(*childTableName, pkv->length);
@@ -500,7 +500,7 @@ static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlL
       return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
   }
 
-  pSml->stableName = tcalloc(stableLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
+  pSml->stableName = tcalloc(stableLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char));
   if (pSml->stableName == NULL){
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
@@ -879,7 +879,7 @@ static int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *nu
     return ret;
   }
 
-  pVal->key = tcalloc(sizeof(key) + TS_ESCAPE_CHAR_SIZE, 1);
+  pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1);
   memcpy(pVal->key, key, sizeof(key));
   addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key));
 
@@ -910,7 +910,7 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs,
         return TSDB_CODE_TSC_INVALID_JSON;
       }
       size_t idLen = strlen(id->valuestring);
-      *childTableName = tcalloc(idLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
+      *childTableName = tcalloc(idLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char));
       memcpy(*childTableName, id->valuestring, idLen);
       addEscapeCharToString(*childTableName, (int32_t)idLen);
 
@@ -948,7 +948,7 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs,
       tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters in JSON", info->id, TSDB_COL_NAME_LEN - 1);
       return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
     }
-    pkv->key = tcalloc(keyLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
+    pkv->key = tcalloc(keyLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char));
     strncpy(pkv->key, tag->string, keyLen);
     addEscapeCharToString(pkv->key, (int32_t)keyLen);
     //value
diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c
index 508adb1050..c2492b2bd0 100644
--- a/src/client/src/tscSQLParser.c
+++ b/src/client/src/tscSQLParser.c
@@ -322,7 +322,7 @@ static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) {
 
 static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) {
   int64_t     time = 0;
-  strdequote(pVar->pz);
+  stringProcess(pVar->pz, pVar->nLen);
 
   char*           seg = strnchr(pVar->pz, '-', pVar->nLen, false);
   if (seg != NULL) {
@@ -359,7 +359,7 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) {
     return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
   }
 
-  strdequote(pPwd->z);
+  stringProcess(pPwd->z, pPwd->n);
   pPwd->n = (uint32_t)strtrim(pPwd->z);  // trim space before and after passwords
 
   if (pPwd->n <= 0) {
@@ -477,7 +477,7 @@ int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       if (validateColumnName(createInfo->name.z) != TSDB_CODE_SUCCESS) {
         return  invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
       }
-      strdequote(createInfo->name.z);
+      stringProcess(createInfo->name.z, createInfo->name.n);
 
       if (strlen(createInfo->name.z) >= TSDB_FUNC_NAME_LEN) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
@@ -485,7 +485,7 @@ int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
       createInfo->path.z[createInfo->path.n] = 0;
 
-      strdequote(createInfo->path.z);
+      stringProcess(createInfo->path.z, createInfo->path.n);
 
       if (strlen(createInfo->path.z) >= PATH_MAX) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
@@ -543,7 +543,7 @@ int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
       t0->z[t0->n] = 0;
 
-      strdequote(t0->z);
+      stringProcess(t0->z, t0->n);
 
       if (strlen(t0->z) >= TSDB_FUNC_NAME_LEN) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
@@ -628,7 +628,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         }
       } else if (pInfo->type == TSDB_SQL_DROP_DNODE) {
         if (pzName->type == TK_STRING) {
-          pzName->n = strdequote(pzName->z);
+          pzName->n = stringProcess(pzName->z, pzName->n);
         }
         strncpy(pCmd->payload, pzName->z, pzName->n);
       } else {  // drop user/account
@@ -718,7 +718,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
       SStrToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0);
       if (id->type == TK_STRING) {
-        id->n = strdequote(id->z);
+        id->n = stringProcess(id->z, id->n);
       }
       break;
     }
@@ -834,7 +834,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       SStrToken* t0 = taosArrayGet(pMiscInfo->a, 0);
       SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1);
 
-      t0->n = strdequote(t0->z);
+      t0->n = stringProcess(t0->z, t0->n);
       strncpy(pCfg->ep, t0->z, t0->n);
 
       if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) {
@@ -3435,7 +3435,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken
   pToken->z = tmpTokenBuf;
 
   if (pToken->type == TK_ID) {
-    tscRmEscapeAndTrimToken(pToken);
+    pToken->n = stringProcess(pToken->z, pToken->n);
   }
 
   for (int16_t i = 0; i < numOfCols; ++i) {
@@ -3596,11 +3596,11 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     // show table/stable like 'xxxx', set the like pattern for show tables
     SStrToken* pPattern = &pShowInfo->pattern;
     if (pPattern->type != 0) {
-      if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) {
+      if (pPattern->type == TK_ID && pPattern->z[0] == TS_BACKQUOTE_CHAR) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
       }
       
-      pPattern->n = strdequote(pPattern->z);
+      pPattern->n = stringProcess(pPattern->z, pPattern->n);
 
       if (pPattern->n <= 0) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
@@ -3618,7 +3618,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     }
 
     if (pShowInfo->prefix.type == TK_STRING) {
-      pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z);
+      pShowInfo->prefix.n = stringProcess(pShowInfo->prefix.z, pShowInfo->prefix.n);
     }
   } 
   return TSDB_CODE_SUCCESS;
@@ -4932,7 +4932,7 @@ static int32_t validateNullExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
     }
 
     char *v = strndup(pRight->exprToken.z, pRight->exprToken.n);
-    int32_t len = strRmquote(v, pRight->exprToken.n);
+    int32_t len = stringProcess(v, pRight->exprToken.n);
     if (len > 0) {
       uint32_t type = 0;
       tGetToken(v, &type);
@@ -6059,7 +6059,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t
   int64_t val = 0;
   bool    parsed = false;
   if (pRight->value.nType == TSDB_DATA_TYPE_BINARY) {
-    pRight->value.nLen = strdequote(pRight->value.pz);
+    pRight->value.nLen = stringProcess(pRight->value.pz, pRight->value.nLen);
 
     char* seg = strnchr(pRight->value.pz, '-', pRight->value.nLen, false);
     if (seg != NULL) {
@@ -7015,7 +7015,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     //handle Escape character backstick
     bool inEscape = false;
-    if (name.z[0] == TS_ESCAPE_CHAR && name.z[name.n - 1] == TS_ESCAPE_CHAR) {
+    if (name.z[0] == TS_BACKQUOTE_CHAR && name.z[name.n - 1] == TS_BACKQUOTE_CHAR) {
       inEscape = true;
       name.type = TK_ID;
     }
@@ -7034,7 +7034,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     int32_t nameLen = pItem->pVar.nLen;
     if (inEscape) {
       memmove(name1, name1 + 1, nameLen);
-      name1[nameLen - TS_ESCAPE_CHAR_SIZE] = '\0';
+      name1[nameLen - TS_BACKQUOTE_CHAR_SIZE] = '\0';
     }
 
     TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes);
@@ -7055,7 +7055,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     //handle Escape character backstick
     bool inEscape = false;
-    if (name.z[0] == TS_ESCAPE_CHAR && name.z[name.n - 1] == TS_ESCAPE_CHAR) {
+    if (name.z[0] == TS_BACKQUOTE_CHAR && name.z[name.n - 1] == TS_BACKQUOTE_CHAR) {
       inEscape = true;
       name.type = TK_ID;
     }
@@ -7096,8 +7096,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     if (inEscape) {
       memmove(name.z, name.z + 1, name.n);
-      name.z[name.n - TS_ESCAPE_CHAR_SIZE] = '\0';
-      name.n -= TS_ESCAPE_CHAR_SIZE;
+      name.z[name.n - TS_BACKQUOTE_CHAR_SIZE] = '\0';
+      name.n -= TS_BACKQUOTE_CHAR_SIZE;
     }
 
     TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes);
@@ -7115,10 +7115,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
     SStrToken    name = {.type = TK_STRING, .z = pItem->name, .n = (uint32_t)strlen(pItem->name)};
     //handle Escape character backstick
-    if (name.z[0] == TS_ESCAPE_CHAR && name.z[name.n - 1] == TS_ESCAPE_CHAR) {
+    if (name.z[0] == TS_BACKQUOTE_CHAR && name.z[name.n - 1] == TS_BACKQUOTE_CHAR) {
       memmove(name.z, name.z + 1, name.n);
-      name.z[name.n - TS_ESCAPE_CHAR_SIZE] = '\0';
-      name.n -= TS_ESCAPE_CHAR_SIZE;
+      name.z[name.n - TS_BACKQUOTE_CHAR_SIZE] = '\0';
+      name.n -= TS_BACKQUOTE_CHAR_SIZE;
     }
     if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
       return invalidOperationMsg(pMsg, msg17);
@@ -7315,7 +7315,7 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) {
     SStrToken* pValToken = taosArrayGet(pOptions->a, 2);
     int32_t vnodeId = 0;
     int32_t dnodeId = 0;
-    strdequote(pValToken->z);
+    stringProcess(pValToken->z, pValToken->n);
     bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeId, &dnodeId);
     if (!parseOk) {
       return TSDB_CODE_TSC_INVALID_OPERATION;  // options value is invalid
@@ -7417,7 +7417,7 @@ int32_t validateColumnName(char* name) {
   }
 
   if (token.type == TK_STRING) {
-    strdequote(token.z);
+    token.n = stringProcess(token.z, token.n);
     strntolower(token.z, token.z, token.n);
     token.n = (uint32_t)strtrim(token.z);
 
@@ -7428,7 +7428,7 @@ int32_t validateColumnName(char* name) {
 
     return validateColumnName(token.z);
   } else if (token.type == TK_ID) {
-    strRmquoteEscape(name, token.n);
+    stringProcess(name, token.n);
     return TSDB_CODE_SUCCESS;
   } else {
     if (isNumber(&token)) {
@@ -7579,7 +7579,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo
 
   SStrToken* pToken = &pCreateDbInfo->precision;
   if (pToken->n > 0) {
-    pToken->n = strdequote(pToken->z);
+    pToken->n = stringProcess(pToken->z, pToken->n);
 
     if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 &&
         strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) {
@@ -8634,12 +8634,8 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
         strncpy(tmpTokenBuf, sToken->z, sToken->n);
         sToken->z = tmpTokenBuf;
 
-        if (TK_STRING == sToken->type) {
-          tscDequoteAndTrimToken(sToken);
-        }
-
-        if (TK_ID == sToken->type) {
-          tscRmEscapeAndTrimToken(sToken);
+        if (TK_STRING == sToken->type || TK_ID == sToken->type) {
+          sToken->n = stringProcess(sToken->z, sToken->n);
         }
 
         tVariantListItem* pItem = taosArrayGet(pValList, i);
@@ -9578,8 +9574,6 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
     }
 
-    tscDequoteAndTrimToken(oriName);
-
     bool dbIncluded = false;
     char      buf[TSDB_TABLE_FNAME_LEN];
     SStrToken sTblToken;
@@ -9601,7 +9595,6 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
 
-      tscDequoteAndTrimToken(aliasName);
       if (tscValidateName(aliasName, false, NULL) != TSDB_CODE_SUCCESS || aliasName->n >= TSDB_TABLE_NAME_LEN) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
       }
diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c
index 49711b6dee..eb189632d1 100644
--- a/src/client/src/tscSql.c
+++ b/src/client/src/tscSql.c
@@ -137,7 +137,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     char tmp[TSDB_DB_NAME_LEN] = {0};
     tstrncpy(tmp, db, sizeof(tmp));
 
-    strdequote(tmp);
+    stringProcess(tmp, strlen(tmp));
     strtolower(pObj->db, tmp);
   }
 
diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c
index af3d2ce4d0..cfe7fb800c 100644
--- a/src/client/src/tscUtil.c
+++ b/src/client/src/tscUtil.c
@@ -2918,7 +2918,7 @@ void tscColumnListDestroy(SArray* pColumnList) {
  *
  */
 static int32_t validateQuoteToken(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded) {
-  tscDequoteAndTrimToken(pToken);
+  pToken->n = stringProcess(pToken->z, pToken->n);
 
   int32_t k = tGetToken(pToken->z, &pToken->type);
 
@@ -2932,94 +2932,6 @@ static int32_t validateQuoteToken(SStrToken* pToken, bool escapeEnabled, bool *d
   return TSDB_CODE_SUCCESS;
 }
 
-void tscDequoteAndTrimToken(SStrToken* pToken) {
-  uint32_t first = 0, last = pToken->n;
-
-  // trim leading spaces
-  while (first < last) {
-    char c = pToken->z[first];
-    if (c != ' ' && c != '\t') {
-      break;
-    }
-    first++;
-  }
-
-  // trim ending spaces
-  while (first < last) {
-    char c = pToken->z[last - 1];
-    if (c != ' ' && c != '\t') {
-      break;
-    }
-    last--;
-  }
-
-  // there are still at least two characters
-  if (first < last - 1) {
-    char c = pToken->z[first];
-    // dequote
-    if ((c == '\'' || c == '"') && c == pToken->z[last - 1]) {
-      first++;
-      last--;
-    }
-  }
-
-  // left shift the string and pad spaces
-  for (uint32_t i = 0; i + first < last; i++) {
-    pToken->z[i] = pToken->z[first + i];
-  }
-  for (uint32_t i = last - first; i < pToken->n; i++) {
-    pToken->z[i] = ' ';
-  }
-
-  // adjust token length
-  pToken->n = last - first;
-}
-
-void tscRmEscapeAndTrimToken(SStrToken* pToken) {
-  uint32_t first = 0, last = pToken->n;
-
-  // trim leading spaces
-  while (first < last) {
-    char c = pToken->z[first];
-    if (c != ' ' && c != '\t') {
-      break;
-    }
-    first++;
-  }
-
-  // trim ending spaces
-  while (first < last) {
-    char c = pToken->z[last - 1];
-    if (c != ' ' && c != '\t') {
-      break;
-    }
-    last--;
-  }
-
-  // there are still at least two characters
-  if (first < last - 1) {
-    char c = pToken->z[first];
-    // dequote
-    if ((c == '`') && c == pToken->z[last - 1]) {
-      first++;
-      last--;
-    }
-  }
-
-  // left shift the string and pad spaces
-  for (uint32_t i = 0; i + first < last; i++) {
-    pToken->z[i] = pToken->z[first + i];
-  }
-  for (uint32_t i = last - first; i < pToken->n; i++) {
-    pToken->z[i] = ' ';
-  }
-
-  // adjust token length
-  pToken->n = last - first;
-}
-
-
-
 int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded) {
   if (pToken == NULL || pToken->z == NULL
      || (pToken->type != TK_STRING && pToken->type != TK_ID)) {
@@ -3027,7 +2939,7 @@ int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded)
   }
 
   if ((!escapeEnabled) && pToken->type == TK_ID) {
-    if (pToken->z[0] == TS_ESCAPE_CHAR) {
+    if (pToken->z[0] == TS_BACKQUOTE_CHAR) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
   }
@@ -3045,7 +2957,7 @@ int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded)
 
     if (pToken->type == TK_STRING) {
 
-      tscDequoteAndTrimToken(pToken);
+      pToken->n = stringProcess(pToken->z, pToken->n);
       // tscStrToLower(pToken->z, pToken->n);
       strntolower(pToken->z, pToken->z, pToken->n);
       //pToken->n = (uint32_t)strtrim(pToken->z);
@@ -3065,7 +2977,7 @@ int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded)
         return tscValidateName(pToken, escapeEnabled, NULL);
       }
     } else if (pToken->type == TK_ID) {
-      tscRmEscapeAndTrimToken(pToken);
+      pToken->n = stringProcess(pToken->z, pToken->n);
 
       if (pToken->n == 0) {
         return TSDB_CODE_TSC_INVALID_OPERATION;
@@ -3126,7 +3038,7 @@ int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded)
     }
 
     if (escapeEnabled && pToken->type == TK_ID) {
-      tscRmEscapeAndTrimToken(pToken);
+      pToken->n = stringProcess(pToken->z, pToken->n);
     }
 
     // re-build the whole name string
diff --git a/src/common/src/tname.c b/src/common/src/tname.c
index 5d7e8ce542..6727fc1961 100644
--- a/src/common/src/tname.c
+++ b/src/common/src/tname.c
@@ -50,7 +50,7 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const
   } else {
     size_t tlen = MIN(sizeof(s.name), exprStr->n + 1);
     tstrncpy(s.name, exprStr->z, tlen);
-    strdequote(s.name);
+    stringProcess(s.name, strlen(s.name));
   }
 
   return s;
@@ -163,7 +163,7 @@ char *tableNameGetPosition(SStrToken* pToken, char target) {
       return pToken->z + i;
     }
   
-    if (*(pToken->z + i) == TS_ESCAPE_CHAR) {
+    if (*(pToken->z + i) == TS_BACKQUOTE_CHAR) {
       if (!inQuote) {
         inEscape = !inEscape;
       }
@@ -223,7 +223,7 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
   char* r = tableNameGetPosition(pToken, sep);  
 
   if (r != NULL) {  // record the table name token    
-    if (pToken->z[0] == TS_ESCAPE_CHAR && *(r - 1) == TS_ESCAPE_CHAR) {
+    if (pToken->z[0] == TS_BACKQUOTE_CHAR && *(r - 1) == TS_BACKQUOTE_CHAR) {
       pTable->n = (uint32_t)(r - pToken->z - 2);
       pTable->z = pToken->z + 1;
     } else {
diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c
index 3c9d622947..8a46875bf5 100644
--- a/src/common/src/tvariant.c
+++ b/src/common/src/tvariant.c
@@ -87,7 +87,7 @@ void tVariantCreateExt(tVariant *pVar, SStrToken *token, int32_t optrType, bool
 
     case TSDB_DATA_TYPE_BINARY: {
       pVar->pz = strndup(token->z, token->n);
-      pVar->nLen = needRmquoteEscape ? strRmquoteEscape(pVar->pz, token->n) : token->n;
+      pVar->nLen = needRmquoteEscape ? stringProcess(pVar->pz, token->n) : token->n;
       break;
     }
     case TSDB_DATA_TYPE_TIMESTAMP: {
diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h
index b7c628a118..c5d65b831a 100644
--- a/src/inc/taosdef.h
+++ b/src/inc/taosdef.h
@@ -108,8 +108,8 @@ extern const int32_t TYPE_BYTES[16];
 #define TSDB_ERR   -1
 
 #define TS_PATH_DELIMITER "."
-#define TS_ESCAPE_CHAR '`'
-#define TS_ESCAPE_CHAR_SIZE 2
+#define TS_BACKQUOTE_CHAR '`'
+#define TS_BACKQUOTE_CHAR_SIZE 2
 
 #define TSDB_TIME_PRECISION_MILLI 0
 #define TSDB_TIME_PRECISION_MICRO 1
diff --git a/src/kit/shell/src/shellCommand.c b/src/kit/shell/src/shellCommand.c
index 67e0c94989..d78e152dbd 100644
--- a/src/kit/shell/src/shellCommand.c
+++ b/src/kit/shell/src/shellCommand.c
@@ -51,8 +51,8 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) {
     if (str[pos] > 0 || countPrefixOnes((unsigned char )str[pos]) > 1) break;
   }
 
-  int rc = mbtowc(&wc, str + pos, MB_CUR_MAX);
-  assert(rc == *size);
+  mbtowc(&wc, str + pos, MB_CUR_MAX);
+  // assert(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk
 
   *width = wcwidth(wc);
 }
diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c
index 0177204176..c37479d79b 100644
--- a/src/kit/shell/src/shellEngine.c
+++ b/src/kit/shell/src/shellEngine.c
@@ -81,9 +81,9 @@ extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *aut
 TAOS *shellInit(SShellArguments *_args) {
   printf("\n");
   if (!_args->is_use_passwd) {
-#ifdef TD_WINDOWS
+#ifdef WINDOWS
     strcpy(tsOsName, "Windows");
-#elif defined(TD_DARWIN)
+#elif defined(DARWIN)
     strcpy(tsOsName, "Darwin");
 #endif
     printf(CLIENT_VERSION, tsOsName, taos_get_client_info());
@@ -239,64 +239,27 @@ int32_t shellRunCommand(TAOS* con, char* command) {
     }
   }
 
-  bool esc = false;
-  char quote = 0, *cmd = command, *p = command;
+  char quote = 0, *cmd = command;
   for (char c = *command++; c != 0; c = *command++) {
-    if (esc) {
-      switch (c) {
-        case 'n':
-          c = '\n';
-          break;
-        case 'r':
-          c = '\r';
-          break;
-        case 't':
-          c = '\t';
-          break;
-        case 'G':
-          *p++ = '\\';
-          break;
-        case '\'':
-        case '"':
-        case '`':
-          if (quote) {
-            *p++ = '\\';
-          }
-          break;
-      }
-      *p++ = c;
-      esc = false;
+    if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) {
+      command ++;
       continue;
     }
 
-    if (c == '\\') {
-      if (quote != 0 && (*command == '_' || *command == '%' || *command == '\\')) {
-        //DO nothing 
-      } else {
-        esc = true;
-        continue;
-      }
-    }
-
     if (quote == c) {
       quote = 0;
     } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) {
       quote = c;
-    }
-
-    *p++ = c;
-    if (c == ';' && quote == 0) {
-      c = *p;
-      *p = 0;
+    } else if (c == ';' && quote == 0) {
+      c = *command;
+      *command = 0;
       if (shellRunSingleCommand(con, cmd) < 0) {
         return -1;
       }
-      *p = c;
-      p = cmd;
+      *command = c;
+      cmd = command;
     }
   }
-
-  *p = 0;
   return shellRunSingleCommand(con, cmd);
 }
 
@@ -411,7 +374,14 @@ int regex_match(const char *s, const char *reg, int cflags) {
   } else if (reti == REG_NOMATCH) {
     regfree(&regex);
     return 0;
-  } else {
+  }
+#ifdef DARWIN
+  else if (reti == REG_ILLSEQ){
+    regfree(&regex);
+    return 0;
+  }
+#endif
+  else {
     regerror(reti, &regex, msgbuf, sizeof(msgbuf));
     fprintf(stderr, "Regex match failed: %s\n", msgbuf);
     regfree(&regex);
@@ -609,20 +579,25 @@ static void shellPrintNChar(const char *str, int length, int width) {
     if (bytes <= 0) {
       break;
     }
-    pos += bytes;
-    if (pos > length) {
-      break;
-    }
-
+    int w = 0;
 #ifdef WINDOWS
-    int w = bytes;
+    w = bytes;
 #else
-    int w = wcwidth(wc);
+    if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
+      w = bytes;
+    }else{
+      w = wcwidth(wc);
+    }
 #endif
     if (w <= 0) {
       continue;
     }
 
+    pos += bytes;
+    if (pos > length) {
+      break;
+    }
+
     if (width <= 0) {
       printf("%lc", wc);
       continue;
diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c
index 3074894071..eaec7b23ad 100644
--- a/src/query/src/qSqlParser.c
+++ b/src/query/src/qSqlParser.c
@@ -96,7 +96,7 @@ SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct,
       strncpy(item.aliasName, pToken->z, pToken->n);
       item.aliasName[pToken->n] = 0;
 
-      strdequote(item.aliasName);
+      stringProcess(item.aliasName, strlen(item.aliasName));
     }
 
     taosArrayPush(pList, &item);
diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h
index cbf7006f8f..8756ed49de 100644
--- a/src/util/inc/tutil.h
+++ b/src/util/inc/tutil.h
@@ -25,9 +25,8 @@ extern "C" {
 #include "tcrc32c.h"
 #include "taosdef.h"
 
-int32_t strdequote(char *src);
-int32_t strRmquote(char *z, int32_t len);
-int32_t strRmquoteEscape(char *z, int32_t len);
+int32_t strDealWithEscape(char *z, int32_t len);
+int32_t stringProcess(char *z, int32_t len);
 size_t  strtrim(char *src);
 char *  tstrstr(char *src, char *dst, bool ignoreInEsc);
 char *  strnchr(char *haystack, char needle, int32_t len, bool skipquote);
diff --git a/src/util/src/ttokenizer.c b/src/util/src/ttokenizer.c
index 5876d82bea..ef1e7890eb 100644
--- a/src/util/src/ttokenizer.c
+++ b/src/util/src/ttokenizer.c
@@ -448,6 +448,13 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) {
     }
     case '`': {
       for (i = 1; z[i]; i++) {
+        if(isprint(z[i]) == 0){
+          break;
+        }
+        if (z[i] == '`' && z[i+1] == '`') {
+          i++;
+          continue;
+        }
         if (z[i] == '`') {
           i++;
           *tokenId = TK_ID;
diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c
index 4a72697f79..e5a326661e 100644
--- a/src/util/src/tutil.c
+++ b/src/util/src/tutil.c
@@ -26,74 +26,79 @@ bool isInteger(double x){
   return (x == truncated);
 }
 
-int32_t strdequote(char *z) {
+int32_t strDealWithEscape(char *z, int32_t len){
   if (z == NULL) {
     return 0;
   }
 
-  int32_t quote = z[0];
-  if (quote != '\'' && quote != '"') {
-    return (int32_t)strlen(z);
-  }
-
-  int32_t i = 1, j = 0;
-
-  while (z[i] != 0) {
-    if (z[i] == quote) {
-      if (z[i + 1] == quote) {
-        z[j++] = (char)quote;
-        i++;
-      } else {
-        z[j++] = 0;
-        return (j - 1);
+  int32_t j = 0;
+  for (int32_t i = 0; i < len; i++) {
+    if (z[i] == '\\') {   // deal with escape character
+      if(z[i+1] == 'n'){
+        z[j++] = '\n';
+      }else if(z[i+1] == 'r'){
+        z[j++] = '\r';
+      }else if(z[i+1] == 't'){
+        z[j++] = '\t';
+      }else if(z[i+1] == '\\'){
+        z[j++] = '\\';
+      }else if(z[i+1] == '\''){
+        z[j++] = '\'';
+      }else if(z[i+1] == '"'){
+        z[j++] = '"';
+      }else if(z[i+1] == '%'){
+        z[j++] = z[i];
+        z[j++] = z[i+1];
+      }else if(z[i+1] == '_'){
+        z[j++] = z[i];
+        z[j++] = z[i+1];
+      }else{
+        z[j++] = z[i+1];
       }
-    } else {
-      z[j++] = z[i];
+
+      i++;
+      continue;
     }
 
-    i++;
+    z[j++] = z[i];
   }
-
-  return j + 1;  // only one quote, do nothing
+  z[j] = 0;
+  return j;
 }
 
-// delete escape character: \\, \', \"
-int32_t strRmquote(char *z, int32_t len){
-  char delim = 0;
-  int32_t cnt = 0;
-  int32_t j = 0;
-  for (size_t k = 0; k < len; ++k) {
-    if (!delim && (z[k] == '\'' || z[k] == '"')){   // find the start ' or "
-      delim = z[k];
-    }
-
-    if ((z[k] == '\\' && z[k + 1] == '_') || (z[k] == '\\' && z[k + 1] == '%')) {
-      //match '_' '%' self
-    }else if(z[k] == '\\'){
-      z[j] = z[k + 1];
-      cnt++;
-      j++;
-      k++;
-      continue;
-    }else if(z[k] == delim){
-      continue;
+/*
+ * remove the quotation marks at both ends
+ * "fsd" => fsd
+ * "f""sd" =>f"sd
+ *  'fsd' => fsd
+ * 'f''sd' =>f'sd
+ *  `fsd => fsd
+ * `f``sd` =>f`sd
+ */
+static int32_t strdequote(char *z, int32_t n){
+  if(z == NULL || n < 2) return n;
+  int32_t quote = z[0];
+  int32_t i = 1, j = 0;
+  while (i < n) {
+    if (i < n - 1 && z[i] == quote && z[i + 1] == quote) { // two consecutive quotation marks keep one
+      z[j++] = (char)quote;
+      i += 2;
+    } else {
+      z[j++] = z[i++];
     }
-    z[j] = z[k];
-    j++;
   }
-  z[j] = 0;
-  return j;
+  z[j - 1] = 0;
+  return j - 1;
 }
 
-int32_t strRmquoteEscape(char *z, int32_t len) {
-  if (len <= 0) return len;
+int32_t stringProcess(char *z, int32_t len) {
+  if (z == NULL || len < 2) return len;
 
-  if (z[0] == '\'' || z[0] == '\"') {
-    return strRmquote(z, len);
-  } else if (len > 1 && z[0] == TS_ESCAPE_CHAR && z[len - 1] == TS_ESCAPE_CHAR) {
-    memmove(z, z + 1, len - 2);
-    z[len - 2] = '\0';
-    return len - 2;
+  if ((z[0] == '\'' && z[len - 1] == '\'')|| (z[0] == '"' && z[len - 1] == '"')) {
+    int32_t n = strdequote(z, len);
+    return strDealWithEscape(z, n);
+  } else if (z[0] == TS_BACKQUOTE_CHAR && z[len - 1] == TS_BACKQUOTE_CHAR) {
+    return strdequote(z, len);
   }
 
   return len;
@@ -192,7 +197,7 @@ char *tstrstr(char *src, char *dst, bool ignoreInEsc) {
   char *str = src, *res = NULL;
   
   for (int32_t i = 0; i < len; ++i) {
-    if (src[i] == TS_ESCAPE_CHAR || src[i] == '\'' || src[i] == '\"') {
+    if (src[i] == TS_BACKQUOTE_CHAR || src[i] == '\'' || src[i] == '\"') {
       if (!inEsc) {
         escChar = src[i];
         src[i] = 0;
@@ -218,8 +223,6 @@ char *tstrstr(char *src, char *dst, bool ignoreInEsc) {
   return str ? strstr(str, dst) : NULL;
 }
 
-
-
 char* strtolower(char *dst, const char *src) {
   int esc = 0;
   char quote = 0, *p = dst, c;
-- 
GitLab