Merge pull request #9696 from taosdata/feature/TD-12252-new

[TD-12252]<feature>(connector,query,insert,other,tools,taosAdapter):d…

documentation20/cn/12.taos-sql/docs.md

@@ -1851,3 +1851,24 @@ TDengine 中的表（列）名命名规则如下：
      ```mysql 
      select jtag->'key' from (select jtag from stable) where jtag->'key'>0
      ```
+## 转义字符说明
+- 转义字符表
+
+  | 字符序列    | **代表的字符**  |
+    | :--------:     |   -------    |
+  | `\'`             |  单引号'      | 
+  | `\"`             |  双引号"      |
+  | \n             |  换行符       |
+  | \r             |  回车符       |
+  | \t             |  tab符       |
+  | `\\`             |  斜杠\        |
+  | `\%`             |  % 规则见下    |
+  | `\%`             |  _ 规则见下    |
+
+- 转义字符使用规则
+  1. 标识符里有转义字符（数据库名、表名、列名）
+    1. 普通标识符：    直接提示错误的标识符，因为标识符规定必须是数字、字母和下划线，并且不能以数字开头。
+    2. 反引号``标识符： 保持原样，不转义
+  2. 数据里有转义字符
+    1. 遇到上面定义的转义字符会转义（%和_见下面说明），如果没有匹配的转义字符会忽略掉转义符\。
+    2. 对于%和_，因为在like里这两个字符是通配符，所以在模式匹配like里用`\%`%和`\_`表示字符里本身的%和_，如果在like模式匹配上下文之外使用`\%`或`\_`，则它们的计算结果为字符串`\%`和`\_`，而不是%和_。
\ No newline at end of file

documentation20/en/12.taos-sql/docs.md

@@ -1335,3 +1335,24 @@ Is not null supports all types of columns. Non-null expression is < > "" and onl
       
      select jtag->'key' from (select jtag from stable) where jtag->'key'>0
      ```
+## Escape character description
+- Special Character Escape Sequences
+
+  | Escape Sequence    | **Character Represented by Sequence**  |
+      | :--------:     |   -------------------   |
+  | `\'`             |  A single quote (') character    | 
+  | `\"`             |  A double quote (") character      |
+  | \n             |  A newline (linefeed) character       |
+  | \r             |  A carriage return character       |
+  | \t             |  A tab character       |
+  | `\\`             |  A backslash (\) character        |
+  | `\%`            |  A % character; see note following the table    |
+  | `\_`             |  A _ character; see note following the table    |
+
+- Escape character usage rules
+  - The escape characters that in a identifier (database name, table name, column name)
+    1. Normal identifier：    The wrong identifier is prompted directly, because the identifier must be numbers, letters and underscores, and cannot start with a number.
+    2. Backquote`` identifier： Keep it as it is.
+  - The escape characters that in a data
+    3. The escape character defined above will be escaped (% and _ see the description below). If there is no matching escape character, the escape character will be ignored.
+    4. The `\%` and `\_` sequences are used to search for literal instances of % and _ in pattern-matching contexts where they would otherwise be interpreted as wildcard characters.If you use `\%` or `\_` outside of pattern-matching contexts, they evaluate to the strings `\%` and `\_`, not to % and _.
\ No newline at end of file

src/client/inc/tscUtil.h

@@ -258,8 +258,6 @@ void tscColumnListCopyAll(SArray* dst, const SArray* src);
 
 void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar, bool convertJson);
 
-void tscDequoteAndTrimToken(SStrToken* pToken);
-void tscRmEscapeAndTrimToken(SStrToken* pToken);
 int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded);
 
 void tscIncStreamExecutionCount(void* pStream);

src/client/src/tscParseInsert.c

@@ -481,32 +481,12 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
 
     // Remove quotation marks
     if (TK_STRING == sToken.type) {
-      // delete escape character: \\, \', \"
-      char delim = sToken.z[0];
-
-      int32_t cnt = 0;
-      int32_t j = 0;
       if (sToken.n >= TSDB_MAX_BYTES_PER_ROW) {
         return tscSQLSyntaxErrMsg(pInsertParam->msg, "too long string", sToken.z);
       }
-
-      for (uint32_t k = 1; k < sToken.n - 1; ++k) {
-        if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) {
-          tmpTokenBuf[j] = sToken.z[k + 1];
-
-          cnt++;
-          j++;
-          k++;
-          continue;
-        }
-
-        tmpTokenBuf[j] = sToken.z[k];
-        j++;
-      }
-
-      tmpTokenBuf[j] = 0;
+      strncpy(tmpTokenBuf, sToken.z, sToken.n);
+      sToken.n = stringProcess(tmpTokenBuf, sToken.n);
       sToken.z = tmpTokenBuf;
-      sToken.n -= 2 + cnt;
     }
 
     bool    isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX);
@@ -1057,10 +1037,12 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
         break;
       }
 
+      char* tmp = NULL;
       // Remove quotation marks
       if (TK_STRING == sToken.type) {
-        sToken.z++;
-        sToken.n -= 2;
+        tmp = strndup(sToken.z, sToken.n);
+        sToken.n = stringProcess(tmp, sToken.n);
+        sToken.z = tmp;
       }
 
       char tagVal[TSDB_MAX_TAGS_LEN] = {0};
@@ -1068,6 +1050,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
       if (code != TSDB_CODE_SUCCESS) {
         tdDestroyKVRowBuilder(&kvRowBuilder);
         tscDestroyBoundColumnInfo(&spd);
+        tfree(tmp);
         return code;
       }
 
@@ -1078,18 +1061,18 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
         if(sToken.n > TSDB_MAX_JSON_TAGS_LEN/TSDB_NCHAR_SIZE){
           tdDestroyKVRowBuilder(&kvRowBuilder);
           tscDestroyBoundColumnInfo(&spd);
+          tfree(tmp);
           return tscSQLSyntaxErrMsg(pInsertParam->msg, "json tag too long", NULL);
         }
-        char* json = strndup(sToken.z, sToken.n);
-        code = parseJsontoTagData(json, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId);
+        code = parseJsontoTagData(sToken.z, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId);
         if (code != TSDB_CODE_SUCCESS) {
           tdDestroyKVRowBuilder(&kvRowBuilder);
           tscDestroyBoundColumnInfo(&spd);
-          tfree(json);
+          tfree(tmp);
           return code;
         }
-        tfree(json);
       }
+      tfree(tmp);
     }
     tscDestroyBoundColumnInfo(&spd);
 
@@ -1246,12 +1229,8 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
     strncpy(tmpTokenBuf, sToken.z, sToken.n);
     sToken.z = tmpTokenBuf;
 
-    if (TK_STRING == sToken.type) {
-      tscDequoteAndTrimToken(&sToken);
-    }
-
-    if (TK_ID == sToken.type) {
-      tscRmEscapeAndTrimToken(&sToken);
+    if (TK_STRING == sToken.type || TK_ID == sToken.type) {
+      sToken.n = stringProcess(sToken.z, sToken.n);
     }
 
     if (sToken.type == TK_RP) {
@@ -1371,7 +1350,7 @@ _clean:
 static int32_t getFileFullPath(SStrToken* pToken, char* output) {
   char path[PATH_MAX] = {0};
   strncpy(path, pToken->z, pToken->n);
-  strdequote(path);
+  stringProcess(path, (int32_t)strlen(path));
 
   wordexp_t full_path;
   if (wordexp(path, &full_path, 0) != 0) {

src/client/src/tscParseLineProtocol.c

@@ -20,7 +20,7 @@
 #include "tscParseLine.h"
 
 typedef struct  {
-  char sTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE];
+  char sTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE];
   SHashObj* tagHash;
   SHashObj* fieldHash;
   SArray* tags; //SArray<SSchema>
@@ -68,13 +68,13 @@ typedef enum {
 } ESchemaAction;
 
 typedef struct {
-  char sTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE];
+  char sTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE];
   SArray* tags; //SArray<SSchema>
   SArray* fields; //SArray<SSchema>
 } SCreateSTableActionInfo;
 
 typedef struct {
-  char sTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE];
+  char sTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE];
   SSchema* field;
 } SAlterSTableActionInfo;
 
@@ -161,14 +161,14 @@ static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableNa
   }
 
   SStringBuilder sb; memset(&sb, 0, sizeof(sb));
-  char sTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char sTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   strncpy(sTableName, point->stableName, strlen(point->stableName));
   //strtolower(sTableName, point->stableName);
   taosStringBuilderAppendString(&sb, sTableName);
   for (int j = 0; j < point->tagNum; ++j) {
     taosStringBuilderAppendChar(&sb, ',');
     TAOS_SML_KV* tagKv = point->tags + j;
-    char tagName[TSDB_COL_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+    char tagName[TSDB_COL_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
     strncpy(tagName, tagKv->key, strlen(tagKv->key));
     //strtolower(tagName, tagKv->key);
     taosStringBuilderAppendString(&sb, tagName);
@@ -192,8 +192,8 @@ static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableNa
 
 static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) {
   tscDebug("SML:0x%"PRIx64" taos_sml_insert build child table name", info->id);
-  char childTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE];
-  int32_t tableNameLen = TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE;
+  char childTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE];
+  int32_t tableNameLen = TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE;
   getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info);
   point->childTableName = calloc(1, tableNameLen+1);
   strncpy(point->childTableName, childTableName, tableNameLen);
@@ -251,15 +251,15 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint,
       size_t nameLen = strlen(tsSmlTagNullName);
       strncpy(tagNullName, tsSmlTagNullName, nameLen);
       addEscapeCharToString(tagNullName, (int32_t)nameLen);
-      size_t* pTagNullIdx = taosHashGet(pStableSchema->tagHash, tagNullName, nameLen + TS_ESCAPE_CHAR_SIZE);
+      size_t* pTagNullIdx = taosHashGet(pStableSchema->tagHash, tagNullName, nameLen + TS_BACKQUOTE_CHAR_SIZE);
       if (!pTagNullIdx) {
         SSchema tagNull = {0};
         tagNull.type  = TSDB_DATA_TYPE_NCHAR;
         tagNull.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
-        strncpy(tagNull.name, tagNullName, nameLen + TS_ESCAPE_CHAR_SIZE);
+        strncpy(tagNull.name, tagNullName, nameLen + TS_BACKQUOTE_CHAR_SIZE);
         taosArrayPush(pStableSchema->tags, &tagNull);
         size_t tagNullIdx = taosArrayGetSize(pStableSchema->tags) - 1;
-        taosHashPut(pStableSchema->tagHash, tagNull.name, nameLen + TS_ESCAPE_CHAR_SIZE, &tagNullIdx, sizeof(tagNullIdx));
+        taosHashPut(pStableSchema->tagHash, tagNull.name, nameLen + TS_BACKQUOTE_CHAR_SIZE, &tagNullIdx, sizeof(tagNullIdx));
       }
     }
 
@@ -295,7 +295,7 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint,
 
 static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[],
                                        SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) {
-  char fieldName[TSDB_COL_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char fieldName[TSDB_COL_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   strcpy(fieldName, pointColField->name);
 
   size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName));
@@ -315,7 +315,7 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash
         action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE;
       }
       memset(&action->alterSTable, 0,  sizeof(SAlterSTableActionInfo));
-      memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE);
+      memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE);
       action->alterSTable.field = pointColField;
       *actionNeeded = true;
     }
@@ -326,7 +326,7 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash
       action->action = SCHEMA_ACTION_ADD_COLUMN;
     }
     memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo));
-    memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE);
+    memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE);
     action->alterSTable.field = pointColField;
     *actionNeeded = true;
   }
@@ -572,7 +572,7 @@ static int32_t getSuperTableMetaFromLocalCache(TAOS* taos, char* tableName, STab
   pSql->fp = NULL;
 
   registerSqlObj(pSql);
-  char tableNameBuf[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char tableNameBuf[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   memcpy(tableNameBuf, tableName, strlen(tableName));
   SStrToken tableToken = {.z = tableNameBuf, .n = (uint32_t)strlen(tableName), .type = TK_ID};
   tGetToken(tableNameBuf, &tableToken.type);
@@ -689,7 +689,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo*
       SSchemaAction schemaAction = {0};
       schemaAction.action = SCHEMA_ACTION_CREATE_STABLE;
       memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo));
-      memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE);
+      memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE);
       schemaAction.createSTable.tags = pointSchema->tags;
       schemaAction.createSTable.fields = pointSchema->fields;
       applySchemaAction(taos, &schemaAction, info);
@@ -726,7 +726,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo*
 
       SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0);
       SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0);
-      memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN + TS_ESCAPE_CHAR_SIZE);
+      memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE);
 
       for (int j = 1; j < pointFieldSize; ++j) {
         SSchema* pointCol = taosArrayGet(pointSchema->fields, j);
@@ -1398,7 +1398,7 @@ char* addEscapeCharToString(char *str, int32_t len) {
     return NULL;
   }
   memmove(str + 1, str, len);
-  str[0] = str[len + 1] = TS_ESCAPE_CHAR;
+  str[0] = str[len + 1] = TS_BACKQUOTE_CHAR;
   str[len + 2] = '\0';
   return str;
 }
@@ -2129,7 +2129,7 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
 
-  pKV->key = calloc(len + TS_ESCAPE_CHAR_SIZE + 1, 1);
+  pKV->key = calloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1);
   memcpy(pKV->key, key, len + 1);
   addEscapeCharToString(pKV->key, len);
   tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
@@ -2227,7 +2227,7 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index
   const char *cur = *index;
   int16_t len = 0;
 
-  pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE, 1);
+  pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1);
   if (pSml->stableName == NULL){
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
@@ -2313,7 +2313,7 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs,
   }
 
   size_t childTableNameLen = strlen(tsSmlChildTableName);
-  char childTableName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char childTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   if (childTableNameLen != 0) {
     memcpy(childTableName, tsSmlChildTableName, childTableNameLen);
     addEscapeCharToString(childTableName, (int32_t)(childTableNameLen));
@@ -2332,7 +2332,7 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs,
     }
 
     if (!isField && childTableNameLen != 0 && strcasecmp(pkv->key, childTableName) == 0)  {
-      smlData->childTableName = malloc(pkv->length + TS_ESCAPE_CHAR_SIZE + 1);
+      smlData->childTableName = malloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1);
       memcpy(smlData->childTableName, pkv->value, pkv->length);
       addEscapeCharToString(smlData->childTableName, (int32_t)pkv->length);
       free(pkv->key);

src/client/src/tscParseOpenTSDB.c

@@ -37,7 +37,7 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index,
   const char *cur = *index;
   uint16_t len = 0;
 
-  pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE, 1);
+  pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1);
   if (pSml->stableName == NULL) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
@@ -125,7 +125,7 @@ static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char
   }
   tfree(value);
 
-  (*pTS)->key = tcalloc(sizeof(key) + TS_ESCAPE_CHAR_SIZE, 1);
+  (*pTS)->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1);
   memcpy((*pTS)->key, key, sizeof(key));
   addEscapeCharToString((*pTS)->key, (int32_t)strlen(key));
 
@@ -196,7 +196,7 @@ static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const ch
   }
   tfree(value);
 
-  pVal->key = tcalloc(sizeof(key) + TS_ESCAPE_CHAR_SIZE, 1);
+  pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1);
   memcpy(pVal->key, key, sizeof(key));
   addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key));
   *num_kvs += 1;
@@ -240,7 +240,7 @@ static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj
     return TSDB_CODE_TSC_DUP_TAG_NAMES;
   }
 
-  pKV->key = tcalloc(len + TS_ESCAPE_CHAR_SIZE + 1, 1);
+  pKV->key = tcalloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1);
   memcpy(pKV->key, key, len + 1);
   addEscapeCharToString(pKV->key, len);
   //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
@@ -307,7 +307,7 @@ static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs,
   pkv = *pKVs;
 
   size_t childTableNameLen = strlen(tsSmlChildTableName);
-  char childTbName[TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE] = {0};
+  char childTbName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0};
   if (childTableNameLen != 0) {
     memcpy(childTbName, tsSmlChildTableName, childTableNameLen);
     addEscapeCharToString(childTbName, (int32_t)(childTableNameLen));
@@ -324,7 +324,7 @@ static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs,
       return ret;
     }
     if (childTableNameLen != 0 && strcasecmp(pkv->key, childTbName) == 0) {
-      *childTableName = tcalloc(pkv->length + TS_ESCAPE_CHAR_SIZE + 1, 1);
+      *childTableName = tcalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1, 1);
       memcpy(*childTableName, pkv->value, pkv->length);
       (*childTableName)[pkv->length] = '\0';
       addEscapeCharToString(*childTableName, pkv->length);
@@ -500,7 +500,7 @@ static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlL
       return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
   }
 
-  pSml->stableName = tcalloc(stableLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
+  pSml->stableName = tcalloc(stableLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char));
   if (pSml->stableName == NULL){
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
@@ -879,7 +879,7 @@ static int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *nu
     return ret;
   }
 
-  pVal->key = tcalloc(sizeof(key) + TS_ESCAPE_CHAR_SIZE, 1);
+  pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1);
   memcpy(pVal->key, key, sizeof(key));
   addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key));
 
@@ -910,7 +910,7 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs,
         return TSDB_CODE_TSC_INVALID_JSON;
       }
       size_t idLen = strlen(id->valuestring);
-      *childTableName = tcalloc(idLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
+      *childTableName = tcalloc(idLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char));
       memcpy(*childTableName, id->valuestring, idLen);
       addEscapeCharToString(*childTableName, (int32_t)idLen);
 
@@ -948,7 +948,7 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs,
       tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters in JSON", info->id, TSDB_COL_NAME_LEN - 1);
       return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
     }
-    pkv->key = tcalloc(keyLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
+    pkv->key = tcalloc(keyLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char));
     strncpy(pkv->key, tag->string, keyLen);
     addEscapeCharToString(pkv->key, (int32_t)keyLen);
     //value

src/client/src/tscSQLParser.c

@@ -322,7 +322,7 @@ static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) {
 
 static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) {
   int64_t     time = 0;
-  strdequote(pVar->pz);
+  stringProcess(pVar->pz, pVar->nLen);
 
   char*           seg = strnchr(pVar->pz, '-', pVar->nLen, false);
   if (seg != NULL) {
@@ -359,7 +359,7 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) {
     return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
   }
 
-  strdequote(pPwd->z);
+  stringProcess(pPwd->z, pPwd->n);
   pPwd->n = (uint32_t)strtrim(pPwd->z);  // trim space before and after passwords
 
   if (pPwd->n <= 0) {
@@ -477,7 +477,7 @@ int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       if (validateColumnName(createInfo->name.z) != TSDB_CODE_SUCCESS) {
         return  invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
       }
-      strdequote(createInfo->name.z);
+      stringProcess(createInfo->name.z, createInfo->name.n);
 
       if (strlen(createInfo->name.z) >= TSDB_FUNC_NAME_LEN) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
@@ -485,7 +485,7 @@ int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
       createInfo->path.z[createInfo->path.n] = 0;
 
-      strdequote(createInfo->path.z);
+      stringProcess(createInfo->path.z, createInfo->path.n);
 
       if (strlen(createInfo->path.z) >= PATH_MAX) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
@@ -543,7 +543,7 @@ int32_t handleUserDefinedFunc(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
       t0->z[t0->n] = 0;
 
-      strdequote(t0->z);
+      stringProcess(t0->z, t0->n);
 
       if (strlen(t0->z) >= TSDB_FUNC_NAME_LEN) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
@@ -628,7 +628,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         }
       } else if (pInfo->type == TSDB_SQL_DROP_DNODE) {
         if (pzName->type == TK_STRING) {
-          pzName->n = strdequote(pzName->z);
+          pzName->n = stringProcess(pzName->z, pzName->n);
         }
         strncpy(pCmd->payload, pzName->z, pzName->n);
       } else {  // drop user/account
@@ -718,7 +718,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
       SStrToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0);
       if (id->type == TK_STRING) {
-        id->n = strdequote(id->z);
+        id->n = stringProcess(id->z, id->n);
       }
       break;
     }
@@ -834,7 +834,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       SStrToken* t0 = taosArrayGet(pMiscInfo->a, 0);
       SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1);
 
-      t0->n = strdequote(t0->z);
+      t0->n = stringProcess(t0->z, t0->n);
       strncpy(pCfg->ep, t0->z, t0->n);
 
       if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) {
@@ -3435,7 +3435,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken
   pToken->z = tmpTokenBuf;
 
   if (pToken->type == TK_ID) {
-    tscRmEscapeAndTrimToken(pToken);
+    pToken->n = stringProcess(pToken->z, pToken->n);
   }
 
   for (int16_t i = 0; i < numOfCols; ++i) {
@@ -3596,11 +3596,11 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     // show table/stable like 'xxxx', set the like pattern for show tables
     SStrToken* pPattern = &pShowInfo->pattern;
     if (pPattern->type != 0) {
-      if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) {
+      if (pPattern->type == TK_ID && pPattern->z[0] == TS_BACKQUOTE_CHAR) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
       }
       
-      pPattern->n = strdequote(pPattern->z);
+      pPattern->n = stringProcess(pPattern->z, pPattern->n);
 
       if (pPattern->n <= 0) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
@@ -3618,7 +3618,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     }
 
     if (pShowInfo->prefix.type == TK_STRING) {
-      pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z);
+      pShowInfo->prefix.n = stringProcess(pShowInfo->prefix.z, pShowInfo->prefix.n);
     }
   } 
   return TSDB_CODE_SUCCESS;
@@ -4932,7 +4932,7 @@ static int32_t validateNullExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
     }
 
     char *v = strndup(pRight->exprToken.z, pRight->exprToken.n);
-    int32_t len = strRmquote(v, pRight->exprToken.n);
+    int32_t len = stringProcess(v, pRight->exprToken.n);
     if (len > 0) {
       uint32_t type = 0;
       tGetToken(v, &type);
@@ -6059,7 +6059,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t
   int64_t val = 0;
   bool    parsed = false;
   if (pRight->value.nType == TSDB_DATA_TYPE_BINARY) {
-    pRight->value.nLen = strdequote(pRight->value.pz);
+    pRight->value.nLen = stringProcess(pRight->value.pz, pRight->value.nLen);
 
     char* seg = strnchr(pRight->value.pz, '-', pRight->value.nLen, false);
     if (seg != NULL) {
@@ -7015,7 +7015,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     //handle Escape character backstick
     bool inEscape = false;
-    if (name.z[0] == TS_ESCAPE_CHAR && name.z[name.n - 1] == TS_ESCAPE_CHAR) {
+    if (name.z[0] == TS_BACKQUOTE_CHAR && name.z[name.n - 1] == TS_BACKQUOTE_CHAR) {
       inEscape = true;
       name.type = TK_ID;
     }
@@ -7034,7 +7034,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     int32_t nameLen = pItem->pVar.nLen;
     if (inEscape) {
       memmove(name1, name1 + 1, nameLen);
-      name1[nameLen - TS_ESCAPE_CHAR_SIZE] = '\0';
+      name1[nameLen - TS_BACKQUOTE_CHAR_SIZE] = '\0';
     }
 
     TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes);
@@ -7055,7 +7055,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     //handle Escape character backstick
     bool inEscape = false;
-    if (name.z[0] == TS_ESCAPE_CHAR && name.z[name.n - 1] == TS_ESCAPE_CHAR) {
+    if (name.z[0] == TS_BACKQUOTE_CHAR && name.z[name.n - 1] == TS_BACKQUOTE_CHAR) {
       inEscape = true;
       name.type = TK_ID;
     }
@@ -7096,8 +7096,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     if (inEscape) {
       memmove(name.z, name.z + 1, name.n);
-      name.z[name.n - TS_ESCAPE_CHAR_SIZE] = '\0';
-      name.n -= TS_ESCAPE_CHAR_SIZE;
+      name.z[name.n - TS_BACKQUOTE_CHAR_SIZE] = '\0';
+      name.n -= TS_BACKQUOTE_CHAR_SIZE;
     }
 
     TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes);
@@ -7115,10 +7115,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
     SStrToken    name = {.type = TK_STRING, .z = pItem->name, .n = (uint32_t)strlen(pItem->name)};
     //handle Escape character backstick
-    if (name.z[0] == TS_ESCAPE_CHAR && name.z[name.n - 1] == TS_ESCAPE_CHAR) {
+    if (name.z[0] == TS_BACKQUOTE_CHAR && name.z[name.n - 1] == TS_BACKQUOTE_CHAR) {
       memmove(name.z, name.z + 1, name.n);
-      name.z[name.n - TS_ESCAPE_CHAR_SIZE] = '\0';
-      name.n -= TS_ESCAPE_CHAR_SIZE;
+      name.z[name.n - TS_BACKQUOTE_CHAR_SIZE] = '\0';
+      name.n -= TS_BACKQUOTE_CHAR_SIZE;
     }
     if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
       return invalidOperationMsg(pMsg, msg17);
@@ -7315,7 +7315,7 @@ int32_t validateDNodeConfig(SMiscInfo* pOptions) {
     SStrToken* pValToken = taosArrayGet(pOptions->a, 2);
     int32_t vnodeId = 0;
     int32_t dnodeId = 0;
-    strdequote(pValToken->z);
+    stringProcess(pValToken->z, pValToken->n);
     bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeId, &dnodeId);
     if (!parseOk) {
       return TSDB_CODE_TSC_INVALID_OPERATION;  // options value is invalid
@@ -7417,7 +7417,7 @@ int32_t validateColumnName(char* name) {
   }
 
   if (token.type == TK_STRING) {
-    strdequote(token.z);
+    token.n = stringProcess(token.z, token.n);
     strntolower(token.z, token.z, token.n);
     token.n = (uint32_t)strtrim(token.z);
 
@@ -7428,7 +7428,7 @@ int32_t validateColumnName(char* name) {
 
     return validateColumnName(token.z);
   } else if (token.type == TK_ID) {
-    strRmquoteEscape(name, token.n);
+    stringProcess(name, token.n);
     return TSDB_CODE_SUCCESS;
   } else {
     if (isNumber(&token)) {
@@ -7579,7 +7579,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo
 
   SStrToken* pToken = &pCreateDbInfo->precision;
   if (pToken->n > 0) {
-    pToken->n = strdequote(pToken->z);
+    pToken->n = stringProcess(pToken->z, pToken->n);
 
     if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 &&
         strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) {
@@ -8634,12 +8634,8 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
         strncpy(tmpTokenBuf, sToken->z, sToken->n);
         sToken->z = tmpTokenBuf;
 
-        if (TK_STRING == sToken->type) {
-          tscDequoteAndTrimToken(sToken);
-        }
-
-        if (TK_ID == sToken->type) {
-          tscRmEscapeAndTrimToken(sToken);
+        if (TK_STRING == sToken->type || TK_ID == sToken->type) {
+          sToken->n = stringProcess(sToken->z, sToken->n);
         }
 
         tVariantListItem* pItem = taosArrayGet(pValList, i);
@@ -9578,8 +9574,6 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
     }
 
-    tscDequoteAndTrimToken(oriName);
-
     bool dbIncluded = false;
     char      buf[TSDB_TABLE_FNAME_LEN];
     SStrToken sTblToken;
@@ -9601,7 +9595,6 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
 
-      tscDequoteAndTrimToken(aliasName);
       if (tscValidateName(aliasName, false, NULL) != TSDB_CODE_SUCCESS || aliasName->n >= TSDB_TABLE_NAME_LEN) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
       }

src/client/src/tscSql.c

@@ -137,7 +137,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
     char tmp[TSDB_DB_NAME_LEN] = {0};
     tstrncpy(tmp, db, sizeof(tmp));
 
-    strdequote(tmp);
+    stringProcess(tmp, (int32_t)strlen(tmp));
     strtolower(pObj->db, tmp);
   }
 

src/client/src/tscUtil.c

@@ -2918,7 +2918,7 @@ void tscColumnListDestroy(SArray* pColumnList) {
  *
  */
 static int32_t validateQuoteToken(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded) {
-  tscDequoteAndTrimToken(pToken);
+  if(pToken->z[0] != TS_BACKQUOTE_CHAR) pToken->n = stringProcess(pToken->z, pToken->n);
 
   int32_t k = tGetToken(pToken->z, &pToken->type);
 
@@ -2932,94 +2932,6 @@ static int32_t validateQuoteToken(SStrToken* pToken, bool escapeEnabled, bool *d
   return TSDB_CODE_SUCCESS;
 }
 
-void tscDequoteAndTrimToken(SStrToken* pToken) {
-  uint32_t first = 0, last = pToken->n;
-
-  // trim leading spaces
-  while (first < last) {
-    char c = pToken->z[first];
-    if (c != ' ' && c != '\t') {
-      break;
-    }
-    first++;
-  }
-
-  // trim ending spaces
-  while (first < last) {
-    char c = pToken->z[last - 1];
-    if (c != ' ' && c != '\t') {
-      break;
-    }
-    last--;
-  }
-
-  // there are still at least two characters
-  if (first < last - 1) {
-    char c = pToken->z[first];
-    // dequote
-    if ((c == '\'' || c == '"') && c == pToken->z[last - 1]) {
-      first++;
-      last--;
-    }
-  }
-
-  // left shift the string and pad spaces
-  for (uint32_t i = 0; i + first < last; i++) {
-    pToken->z[i] = pToken->z[first + i];
-  }
-  for (uint32_t i = last - first; i < pToken->n; i++) {
-    pToken->z[i] = ' ';
-  }
-
-  // adjust token length
-  pToken->n = last - first;
-}
-
-void tscRmEscapeAndTrimToken(SStrToken* pToken) {
-  uint32_t first = 0, last = pToken->n;
-
-  // trim leading spaces
-  while (first < last) {
-    char c = pToken->z[first];
-    if (c != ' ' && c != '\t') {
-      break;
-    }
-    first++;
-  }
-
-  // trim ending spaces
-  while (first < last) {
-    char c = pToken->z[last - 1];
-    if (c != ' ' && c != '\t') {
-      break;
-    }
-    last--;
-  }
-
-  // there are still at least two characters
-  if (first < last - 1) {
-    char c = pToken->z[first];
-    // dequote
-    if ((c == '`') && c == pToken->z[last - 1]) {
-      first++;
-      last--;
-    }
-  }
-
-  // left shift the string and pad spaces
-  for (uint32_t i = 0; i + first < last; i++) {
-    pToken->z[i] = pToken->z[first + i];
-  }
-  for (uint32_t i = last - first; i < pToken->n; i++) {
-    pToken->z[i] = ' ';
-  }
-
-  // adjust token length
-  pToken->n = last - first;
-}
-
-
-
 int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded) {
   if (pToken == NULL || pToken->z == NULL
      || (pToken->type != TK_STRING && pToken->type != TK_ID)) {
@@ -3027,7 +2939,7 @@ int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded)
   }
 
   if ((!escapeEnabled) && pToken->type == TK_ID) {
-    if (pToken->z[0] == TS_ESCAPE_CHAR) {
+    if (pToken->z[0] == TS_BACKQUOTE_CHAR) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
   }
@@ -3045,7 +2957,7 @@ int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded)
 
     if (pToken->type == TK_STRING) {
 
-      tscDequoteAndTrimToken(pToken);
+      if(pToken->z[0] != TS_BACKQUOTE_CHAR) pToken->n = stringProcess(pToken->z, pToken->n);
       // tscStrToLower(pToken->z, pToken->n);
       strntolower(pToken->z, pToken->z, pToken->n);
       //pToken->n = (uint32_t)strtrim(pToken->z);
@@ -3065,7 +2977,7 @@ int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded)
         return tscValidateName(pToken, escapeEnabled, NULL);
       }
     } else if (pToken->type == TK_ID) {
-      tscRmEscapeAndTrimToken(pToken);
+      if(pToken->z[0] == TS_BACKQUOTE_CHAR) pToken->n = stringProcess(pToken->z, pToken->n);
 
       if (pToken->n == 0) {
         return TSDB_CODE_TSC_INVALID_OPERATION;
@@ -3126,7 +3038,7 @@ int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded)
     }
 
     if (escapeEnabled && pToken->type == TK_ID) {
-      tscRmEscapeAndTrimToken(pToken);
+      if(pToken->z[0] == TS_BACKQUOTE_CHAR) pToken->n = stringProcess(pToken->z, pToken->n);
     }
 
     // re-build the whole name string

src/common/src/tname.c

@@ -50,7 +50,7 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const
   } else {
     size_t tlen = MIN(sizeof(s.name), exprStr->n + 1);
     tstrncpy(s.name, exprStr->z, tlen);
-    strdequote(s.name);
+    stringProcess(s.name, (int32_t)strlen(s.name));
   }
 
   return s;
@@ -163,7 +163,7 @@ char *tableNameGetPosition(SStrToken* pToken, char target) {
       return pToken->z + i;
     }
   
-    if (*(pToken->z + i) == TS_ESCAPE_CHAR) {
+    if (*(pToken->z + i) == TS_BACKQUOTE_CHAR) {
       if (!inQuote) {
         inEscape = !inEscape;
       }
@@ -223,7 +223,7 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
   char* r = tableNameGetPosition(pToken, sep);  
 
   if (r != NULL) {  // record the table name token    
-    if (pToken->z[0] == TS_ESCAPE_CHAR && *(r - 1) == TS_ESCAPE_CHAR) {
+    if (pToken->z[0] == TS_BACKQUOTE_CHAR && *(r - 1) == TS_BACKQUOTE_CHAR) {
       pTable->n = (uint32_t)(r - pToken->z - 2);
       pTable->z = pToken->z + 1;
     } else {

src/common/src/tvariant.c

@@ -87,7 +87,7 @@ void tVariantCreateExt(tVariant *pVar, SStrToken *token, int32_t optrType, bool
 
     case TSDB_DATA_TYPE_BINARY: {
       pVar->pz = strndup(token->z, token->n);
-      pVar->nLen = needRmquoteEscape ? strRmquoteEscape(pVar->pz, token->n) : token->n;
+      pVar->nLen = needRmquoteEscape ? stringProcess(pVar->pz, token->n) : token->n;
       break;
     }
     case TSDB_DATA_TYPE_TIMESTAMP: {

src/inc/taosdef.h

@@ -108,8 +108,8 @@ extern const int32_t TYPE_BYTES[16];
 #define TSDB_ERR   -1
 
 #define TS_PATH_DELIMITER "."
-#define TS_ESCAPE_CHAR '`'
-#define TS_ESCAPE_CHAR_SIZE 2
+#define TS_BACKQUOTE_CHAR '`'
+#define TS_BACKQUOTE_CHAR_SIZE 2
 
 #define TSDB_TIME_PRECISION_MILLI 0
 #define TSDB_TIME_PRECISION_MICRO 1

src/kit/shell/src/shellCommand.c

@@ -51,8 +51,8 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) {
     if (str[pos] > 0 || countPrefixOnes((unsigned char )str[pos]) > 1) break;
   }
 
-  int rc = mbtowc(&wc, str + pos, MB_CUR_MAX);
-  assert(rc == *size);
+  mbtowc(&wc, str + pos, MB_CUR_MAX);
+  // assert(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk
 
   *width = wcwidth(wc);
 }

src/kit/shell/src/shellEngine.c

@@ -81,9 +81,9 @@ extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *aut
 TAOS *shellInit(SShellArguments *_args) {
   printf("\n");
   if (!_args->is_use_passwd) {
-#ifdef TD_WINDOWS
+#ifdef WINDOWS
     strcpy(tsOsName, "Windows");
-#elif defined(TD_DARWIN)
+#elif defined(DARWIN)
     strcpy(tsOsName, "Darwin");
 #endif
     printf(CLIENT_VERSION, tsOsName, taos_get_client_info());
@@ -239,64 +239,27 @@ int32_t shellRunCommand(TAOS* con, char* command) {
     }
   }
 
-  bool esc = false;
-  char quote = 0, *cmd = command, *p = command;
+  char quote = 0, *cmd = command;
   for (char c = *command++; c != 0; c = *command++) {
-    if (esc) {
-      switch (c) {
-        case 'n':
-          c = '\n';
-          break;
-        case 'r':
-          c = '\r';
-          break;
-        case 't':
-          c = '\t';
-          break;
-        case 'G':
-          *p++ = '\\';
-          break;
-        case '\'':
-        case '"':
-        case '`':
-          if (quote) {
-            *p++ = '\\';
-          }
-          break;
-      }
-      *p++ = c;
-      esc = false;
+    if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) {
+      command ++;
       continue;
     }
 
-    if (c == '\\') {
-      if (quote != 0 && (*command == '_' || *command == '%' || *command == '\\')) {
-        //DO nothing 
-      } else {
-        esc = true;
-        continue;
-      }
-    }
-
     if (quote == c) {
       quote = 0;
     } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) {
       quote = c;
-    }
-
-    *p++ = c;
-    if (c == ';' && quote == 0) {
-      c = *p;
-      *p = 0;
+    } else if (c == ';' && quote == 0) {
+      c = *command;
+      *command = 0;
       if (shellRunSingleCommand(con, cmd) < 0) {
         return -1;
       }
-      *p = c;
-      p = cmd;
+      *command = c;
+      cmd = command;
     }
   }
-
-  *p = 0;
   return shellRunSingleCommand(con, cmd);
 }
 
@@ -411,7 +374,14 @@ int regex_match(const char *s, const char *reg, int cflags) {
   } else if (reti == REG_NOMATCH) {
     regfree(&regex);
     return 0;
-  } else {
+  }
+#ifdef DARWIN
+  else if (reti == REG_ILLSEQ){
+    regfree(&regex);
+    return 0;
+  }
+#endif
+  else {
     regerror(reti, &regex, msgbuf, sizeof(msgbuf));
     fprintf(stderr, "Regex match failed: %s\n", msgbuf);
     regfree(&regex);
@@ -609,20 +579,25 @@ static void shellPrintNChar(const char *str, int length, int width) {
     if (bytes <= 0) {
       break;
     }
-    pos += bytes;
-    if (pos > length) {
-      break;
-    }
-
+    int w = 0;
 #ifdef WINDOWS
-    int w = bytes;
+    w = bytes;
 #else
-    int w = wcwidth(wc);
+    if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
+      w = bytes;
+    }else{
+      w = wcwidth(wc);
+    }
 #endif
     if (w <= 0) {
       continue;
     }
 
+    pos += bytes;
+    if (pos > length) {
+      break;
+    }
+
     if (width <= 0) {
       printf("%lc", wc);
       continue;

src/query/src/qSqlParser.c

@@ -96,7 +96,7 @@ SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct,
       strncpy(item.aliasName, pToken->z, pToken->n);
       item.aliasName[pToken->n] = 0;
 
-      strdequote(item.aliasName);
+      stringProcess(item.aliasName, (int32_t)strlen(item.aliasName));
     }
 
     taosArrayPush(pList, &item);

src/util/inc/tutil.h

@@ -25,9 +25,8 @@ extern "C" {
 #include "tcrc32c.h"
 #include "taosdef.h"
 
-int32_t strdequote(char *src);
-int32_t strRmquote(char *z, int32_t len);
-int32_t strRmquoteEscape(char *z, int32_t len);
+int32_t strDealWithEscape(char *z, int32_t len);
+int32_t stringProcess(char *z, int32_t len);
 size_t  strtrim(char *src);
 char *  tstrstr(char *src, char *dst, bool ignoreInEsc);
 char *  strnchr(char *haystack, char needle, int32_t len, bool skipquote);

src/util/src/tcompare.c

@@ -390,6 +390,7 @@ int WCSPatternMatch(const uint32_t *patterStr, const uint32_t *str, size_t size,
   uint32_t c, c1;
   uint32_t matchOne = (uint32_t) L'_';  // "_"
   uint32_t matchAll = (uint32_t) L'%';  // "%"
+  uint32_t escape   = (uint32_t) L'\\';  // "\"
 
   int32_t i = 0;
   int32_t j = 0;
@@ -427,6 +428,8 @@ int WCSPatternMatch(const uint32_t *patterStr, const uint32_t *str, size_t size,
     c1 = str[j++];
 
     if (j <= size) {
+      if (c == escape && patterStr[i] == matchOne && c1 == matchOne) { i++; continue; }
+      if (c == escape && patterStr[i] == matchAll && c1 == matchAll) { i++; continue; }
       if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
         continue;
       }

src/util/src/ttokenizer.c

@@ -448,6 +448,13 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) {
     }
     case '`': {
       for (i = 1; z[i]; i++) {
+//        if(isprint(z[i]) == 0){
+//          break;
+//        }
+//        if (z[i] == '`' && z[i+1] == '`') {
+//          i++;
+//          continue;
+//        }
         if (z[i] == '`') {
           i++;
           *tokenId = TK_ID;

src/util/src/tutil.c

@@ -26,74 +26,81 @@ bool isInteger(double x){
   return (x == truncated);
 }
 
-int32_t strdequote(char *z) {
+int32_t strDealWithEscape(char *z, int32_t len){
   if (z == NULL) {
     return 0;
   }
 
-  int32_t quote = z[0];
-  if (quote != '\'' && quote != '"') {
-    return (int32_t)strlen(z);
-  }
-
-  int32_t i = 1, j = 0;
-
-  while (z[i] != 0) {
-    if (z[i] == quote) {
-      if (z[i + 1] == quote) {
-        z[j++] = (char)quote;
-        i++;
-      } else {
-        z[j++] = 0;
-        return (j - 1);
+  int32_t j = 0;
+  for (int32_t i = 0; i < len; i++) {
+    if (z[i] == '\\') {   // deal with escape character
+      if(z[i+1] == 'n'){
+        z[j++] = '\n';
+      }else if(z[i+1] == 'r'){
+        z[j++] = '\r';
+      }else if(z[i+1] == 't'){
+        z[j++] = '\t';
+      }else if(z[i+1] == '\\'){
+        z[j++] = '\\';
+      }else if(z[i+1] == '\''){
+        z[j++] = '\'';
+      }else if(z[i+1] == '"'){
+        z[j++] = '"';
+      }else if(z[i+1] == '%'){
+        z[j++] = z[i];
+        z[j++] = z[i+1];
+      }else if(z[i+1] == '_'){
+        z[j++] = z[i];
+        z[j++] = z[i+1];
+      }else{
+        z[j++] = z[i+1];
       }
-    } else {
-      z[j++] = z[i];
+
+      i++;
+      continue;
     }
 
-    i++;
+    z[j++] = z[i];
   }
-
-  return j + 1;  // only one quote, do nothing
+  z[j] = 0;
+  return j;
 }
 
-// delete escape character: \\, \', \"
-int32_t strRmquote(char *z, int32_t len){
-  char delim = 0;
-  int32_t cnt = 0;
-  int32_t j = 0;
-  for (size_t k = 0; k < len; ++k) {
-    if (!delim && (z[k] == '\'' || z[k] == '"')){   // find the start ' or "
-      delim = z[k];
-    }
-
-    if ((z[k] == '\\' && z[k + 1] == '_') || (z[k] == '\\' && z[k + 1] == '%')) {
-      //match '_' '%' self
-    }else if(z[k] == '\\'){
-      z[j] = z[k + 1];
-      cnt++;
-      j++;
-      k++;
-      continue;
-    }else if(z[k] == delim){
-      continue;
+/*
+ * remove the quotation marks at both ends
+ * "fsd" => fsd
+ * "f""sd" =>f"sd
+ *  'fsd' => fsd
+ * 'f''sd' =>f'sd
+ *  `fsd => fsd
+ * `f``sd` =>f`sd
+ */
+static int32_t strdequote(char *z, int32_t n){
+  if(z == NULL || n < 2) return n;
+  int32_t quote = z[0];
+  z[0] = 0;
+  z[n - 1] = 0;
+  int32_t i = 1, j = 0;
+  while (i < n) {
+    if (i < n - 1 && z[i] == quote && z[i + 1] == quote) { // two consecutive quotation marks keep one
+      z[j++] = (char)quote;
+      i += 2;
+    } else {
+      z[j++] = z[i++];
     }
-    z[j] = z[k];
-    j++;
   }
-  z[j] = 0;
-  return j;
+  z[j - 1] = 0;
+  return j - 1;
 }
 
-int32_t strRmquoteEscape(char *z, int32_t len) {
-  if (len <= 0) return len;
+int32_t stringProcess(char *z, int32_t len) {
+  if (z == NULL || len < 2) return len;
 
-  if (z[0] == '\'' || z[0] == '\"') {
-    return strRmquote(z, len);
-  } else if (len > 1 && z[0] == TS_ESCAPE_CHAR && z[len - 1] == TS_ESCAPE_CHAR) {
-    memmove(z, z + 1, len - 2);
-    z[len - 2] = '\0';
-    return len - 2;
+  if ((z[0] == '\'' && z[len - 1] == '\'')|| (z[0] == '"' && z[len - 1] == '"')) {
+    int32_t n = strdequote(z, len);
+    return strDealWithEscape(z, n);
+  } else if (z[0] == TS_BACKQUOTE_CHAR && z[len - 1] == TS_BACKQUOTE_CHAR) {
+    return strdequote(z, len);
   }
 
   return len;
@@ -134,7 +141,6 @@ size_t strtrim(char *z) {
   } else if (j != i) {
     z[i] = 0;
   }
-  
   return i;
 }
 
@@ -190,9 +196,9 @@ char *tstrstr(char *src, char *dst, bool ignoreInEsc) {
   bool inEsc = false;
   char escChar = 0;
   char *str = src, *res = NULL;
-  
+
   for (int32_t i = 0; i < len; ++i) {
-    if (src[i] == TS_ESCAPE_CHAR || src[i] == '\'' || src[i] == '\"') {
+    if (src[i] == TS_BACKQUOTE_CHAR || src[i] == '\'' || src[i] == '\"') {
       if (!inEsc) {
         escChar = src[i];
         src[i] = 0;
@@ -209,7 +215,7 @@ char *tstrstr(char *src, char *dst, bool ignoreInEsc) {
 
         str = src + i + 1;
       }
-      
+
       inEsc = !inEsc;
       continue;
     }
@@ -218,8 +224,6 @@ char *tstrstr(char *src, char *dst, bool ignoreInEsc) {
   return str ? strstr(str, dst) : NULL;
 }
 
-
-
 char* strtolower(char *dst, const char *src) {
   int esc = 0;
   char quote = 0, *p = dst, c;

src/util/tests/stringTest.cpp

@@ -6,56 +6,80 @@
 #include "taos.h"
 #include "tutil.h"
 
-TEST(testCase, str_rmquote_test) {
-  char    t1[] = "\"\".dd";
-  int32_t len = strRmquote(t1, strlen(t1));
+TEST(testCase, str_escape_test) {
+  char    t1[] = "\"\\\".dd";
+  int32_t len = strDealWithEscape(t1, strlen(t1));
   printf("t1:%s, len:%d\n", t1, len);
-  EXPECT_EQ(3, len);
-  EXPECT_STRCASEEQ(t1, ".dd");
+  EXPECT_EQ(5, len);
+  EXPECT_STRCASEEQ(t1, "\"\".dd");
 
-  char    t2[] = "\"fsd\\\"fs\".dd";
-  len = strRmquote(t2, strlen(t2));
+  char    t2[] = "'\\\'.dd";
+   len = strDealWithEscape(t2, strlen(t2));
   printf("t2:%s, len:%d\n", t2, len);
-  EXPECT_EQ(9, len);
-  EXPECT_STRCASEEQ(t2, "fsd\"fs.dd");
+  EXPECT_EQ(5, len);
+  EXPECT_STRCASEEQ(t2, "''.dd");
 
-  char    t3[] = "fs\\_d\\%.d\\d";
-  len = strRmquote(t3, strlen(t3));
+  char    t3[] = "\\\\.dd";
+   len = strDealWithEscape(t3, strlen(t3));
   printf("t3:%s, len:%d\n", t3, len);
-  EXPECT_EQ(10, len);
-  EXPECT_STRCASEEQ(t3, "fs\\_d\\%.dd");
+  EXPECT_EQ(4, len);
+  EXPECT_STRCASEEQ(t3, "\\.dd");
 
-  char    t4[] = "\"fs\\_d\\%\".dd";
-  len = strRmquote(t4, strlen(t4));
+  char    t4[] = "'\\n.dd";
+   len = strDealWithEscape(t4, strlen(t4));
   printf("t4:%s, len:%d\n", t4, len);
-  EXPECT_EQ(10, len);
-  EXPECT_STRCASEEQ(t4, "fs\\_d\\%.dd");
-
-  char    t5[] = "\"fs\\_d\\%\"";
-  len = strRmquote(t5, strlen(t5));
-  printf("t5:%s, len:%d\n", t5, len);
-  EXPECT_EQ(7, len);
-  EXPECT_STRCASEEQ(t5, "fs\\_d\\%");
-
-  char    t6[] = "'fs\\_d\\%'";
-  len = strRmquote(t6, strlen(t6));
-  printf("t6:%s, len:%d\n", t6, len);
-  EXPECT_EQ(7, len);
-  EXPECT_STRCASEEQ(t6, "fs\\_d\\%");
+  EXPECT_EQ(4, len);
+  EXPECT_STRCASEEQ(t4, "\n.dd");
+
+//  char    t2[] = "\"fsd\\\"fs\".dd";
+//  len = strDealWithEscape(t2, strlen(t2));
+//  printf("t2:%s, len:%d\n", t2, len);
+//  EXPECT_EQ(11, len);
+//  EXPECT_STRCASEEQ(t2, "\"fsd\"fs\".dd");
+//
+//  char    t3[] = "fs\\_d\\%.d\\d";
+//  len = strRmquote(t3, strlen(t3));
+//  printf("t3:%s, len:%d\n", t3, len);
+//  EXPECT_EQ(10, len);
+//  EXPECT_STRCASEEQ(t3, "fs\\_d\\%.dd");
+//
+//  char    t4[] = "\"fs\\_d\\%\".dd";
+//  len = strRmquote(t4, strlen(t4));
+//  printf("t4:%s, len:%d\n", t4, len);
+//  EXPECT_EQ(10, len);
+//  EXPECT_STRCASEEQ(t4, "fs\\_d\\%.dd");
+//
+//  char    t5[] = "\"fs\\_d\\%\"";
+//  len = strRmquote(t5, strlen(t5));
+//  printf("t5:%s, len:%d\n", t5, len);
+//  EXPECT_EQ(7, len);
+//  EXPECT_STRCASEEQ(t5, "fs\\_d\\%");
+//
+//  char    t6[] = "'fs\\_d\\%'";
+//  len = strRmquote(t6, strlen(t6));
+//  printf("t6:%s, len:%d\n", t6, len);
+//  EXPECT_EQ(7, len);
+//  EXPECT_STRCASEEQ(t6, "fs\\_d\\%");
 }
 
 TEST(testCase, string_dequote_test) {
-  char    t1[] = "'abc'";
-  int32_t len = strdequote(t1);
+  char    t1[] = "'ab''c'";
+  int32_t len = stringProcess(t1, strlen(t1));
 
-  EXPECT_EQ(3, len);
-  EXPECT_STRCASEEQ(t1, "abc");
+  EXPECT_EQ(4, len);
+  EXPECT_STRCASEEQ(t1, "ab'c");
+
+  char t2[] = "\"ab\"\"c\"";
+  len = stringProcess(t2, strlen(t2));
+
+  EXPECT_EQ(4, len);
+  EXPECT_STRCASEEQ(t1, "ab\"c");
 
-  char t2[] = "\"abc\"";
-  len = strdequote(t2);
+  char t3[] = "`ab``c`";
+  len = stringProcess(t3, strlen(t3));
 
   EXPECT_EQ(3, len);
-  EXPECT_STRCASEEQ(t1, "abc");
+  EXPECT_STRCASEEQ(t1, "ab`c");
 
   char t21[] = " abc ";
   int32_t lx = strtrim(t21);

tests/develop-test/2-query/escape.py

+###################################################################
+#           Copyright (c) 2021 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def caseDescription(self):
+        '''
+        case1: [TD-12251] json type containing single quotes cannot be inserted
+        case2: [TD-12334] '\' escape unknown
+        case3: [TD-11071] escape table creation problem
+        case5: [TD-12815] like wildcards (% _)  are not supported nchar type
+        '''
+        return
+
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        self._conn = conn
+
+    def run(self):
+        print("running {}".format(__file__))
+        tdSql.execute("drop database if exists escape")
+        tdSql.execute("create database if not exists escape")
+        tdSql.execute('use escape')
+
+        # [TD-12251]
+        tdSql.execute('create stable st (ts timestamp,t int) tags(metrics json)')
+        tdSql.execute(r"insert into t1 using st tags('{\"a\":\"a\",\"b\":\"\'a\'=b\"}') values(now,1)")
+        tdSql.query('select * from st')
+        tdSql.checkData(0, 2, '''{"a":"a","b":"'a'=b"}''')
+
+        # [TD-12334]
+        tdSql.execute('create table car (ts timestamp, s int) tags(j int)')
+        tdSql.execute(r'create table `zz\ ` using car tags(11)')
+        tdSql.execute(r'create table `zz\\ ` using car tags(11)')
+        tdSql.execute(r'create table `zz\\\ ` using car tags(11)')
+        tdSql.query(r'select tbname from car where tbname like "zz\\\\ "')
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, r"zz\\ ")
+
+        tdSql.query(r'show tables like "zz\\\\ "')
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, r"zz\\ ")
+
+        tdSql.query(r'show tables like "zz\\ "')
+        tdSql.checkRows(1)
+
+        # [TD-11071]
+        tdSql.execute('create table es (ts timestamp, s int) tags(j int)')
+        tdSql.execute(r'create table `zz\t` using es tags(11)')
+        tdSql.execute(r'create table `zz\\n` using es tags(11)')
+        tdSql.execute(r'create table `zz\r\ ` using es tags(11)')
+        tdSql.execute(r'create table `      ` using es tags(11)')
+        tdSql.query(r'select tbname from es')
+        tdSql.checkData(0, 0, r'zz\t')
+        tdSql.checkData(1, 0, r'zz\\n')
+        tdSql.checkData(2, 0, r'zz\r\ ')
+        tdSql.checkData(3, 0, r'      ')
+
+        # [TD-6232]
+        tdSql.execute('create table tt(ts timestamp, i nchar(128))')
+        tdSql.execute(r"insert into tt values(1591060628000, '\t')")
+        tdSql.execute(r"insert into tt values(1591060638000, '\n')")
+        tdSql.execute(r"insert into tt values(1591060648000, '\r')")
+        tdSql.execute(r"insert into tt values(1591060658000, '\\')")
+        tdSql.execute(r"insert into tt values(1591060668000, '\"')")
+        tdSql.execute(r"insert into tt values(1591060678000, '\'')")
+        tdSql.execute(r"insert into tt values(1591060688000, '\%')")
+        tdSql.execute(r"insert into tt values(1591060698000, '\_')")
+        tdSql.execute(r"insert into tt values(1591060708000, '\9')")
+
+        tdSql.query(r"select * from tt where i='\t'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='\n'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='\r'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='\\'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='\"'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='\''")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='\%'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='\_'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='\9'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i='9'")
+        tdSql.checkRows(1)
+
+        # [TD-12815] like wildcard(%, _) are not supported nchar
+        tdSql.execute(r"insert into tt values(1591070708000, 'h%d')")
+        tdSql.execute(r"insert into tt values(1591080708000, 'h_j')")
+        tdSql.query(r"select * from tt where i like 'h\%d'")
+        tdSql.checkRows(1)
+        tdSql.query(r"select * from tt where i like 'h\_j'")
+        tdSql.checkRows(1)
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/develop-test/fulltest-query.sh

@@ -2,4 +2,5 @@ python3 ./test.py -f 2-query/ts_hidden_column.py
 python3 ./test.py -f 2-query/union-order.py
 python3 ./test.py -f 2-query/session_two_stage.py
 python3 ./test.py -f 2-query/timeline_agg_func_groupby.py
-python3 ./test.py -f 2-query/ts_2016.py
\ No newline at end of file
+python3 ./test.py -f 2-query/ts_2016.py
+python3 ./test.py -f 2-query/escape.py
\ No newline at end of file