Merge pull request #8425 from taosdata/enhance/TD-10700

[TD-10700]<enhance>: The escape char backstick can be used for both tag name and column name

src/client/inc/tscParseLine.h

@@ -67,6 +67,7 @@ typedef struct {
   int64_t affectedRows;
 } SSmlLinesInfo;
 
+void addEscapeCharToString(char *str, int32_t len);
 int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info);
 bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info);
 bool isValidInteger(char *str);

src/client/inc/tscUtil.h

@@ -251,6 +251,7 @@ void tscColumnListCopyAll(SArray* dst, const SArray* src);
 void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar);
 
 void tscDequoteAndTrimToken(SStrToken* pToken);
+void tscRmEscapeAndTrimToken(SStrToken* pToken);
 int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded);
 
 void tscIncStreamExecutionCount(void* pStream);

src/client/src/tscParseInsert.c

@@ -1251,10 +1251,18 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
     sToken = tStrGetToken(str, &index, false);
     str += index;
 
+    char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character backstick(`)
+    strncpy(tmpTokenBuf, sToken.z, sToken.n);
+    sToken.z = tmpTokenBuf;
+
     if (TK_STRING == sToken.type) {
       tscDequoteAndTrimToken(&sToken);
     }
 
+    if (TK_ID == sToken.type) {
+      tscRmEscapeAndTrimToken(&sToken);
+    }
+
     if (sToken.type == TK_RP) {
       if (end != NULL) {  // set the end position
         *end = str;

src/client/src/tscParseLineProtocol.c

@@ -499,6 +499,7 @@ static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSc
   for (int i=0; i<tableMeta->tableInfo.numOfColumns; ++i) {
     SSchema field;
     tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1);
+    addEscapeCharToString(field.name, (int16_t)strlen(field.name));
     field.type = tableMeta->schema[i].type;
     field.bytes = tableMeta->schema[i].bytes;
     taosArrayPush(schema->fields, &field);
@@ -510,6 +511,7 @@ static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSc
     int j = i + tableMeta->tableInfo.numOfColumns;
     SSchema field;
     tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1);
+    addEscapeCharToString(field.name, (int16_t)strlen(field.name));
     field.type = tableMeta->schema[j].type;
     field.bytes = tableMeta->schema[j].bytes;
     taosArrayPush(schema->tags, &field);
@@ -1175,6 +1177,15 @@ static void escapeSpecialCharacter(uint8_t field, const char **pos) {
   *pos = cur;
 }
 
+void addEscapeCharToString(char *str, int32_t len) {
+  if (str == NULL) {
+    return;
+  }
+  memmove(str + 1, str, len);
+  str[0] = str[len + 1] = TS_ESCAPE_CHAR;
+  str[len + 2] = '\0';
+}
+
 bool isValidInteger(char *str) {
   char *c = str;
   if (*c != '+' && *c != '-' && !isdigit(*c)) {
@@ -1883,13 +1894,8 @@ bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) {
 static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) {
   const char *cur = *index;
   char key[TSDB_COL_NAME_LEN + 1];  // +1 to avoid key[len] over write
-  uint16_t len = 0;
+  int16_t len = 0;
 
-  //key field cannot start with digit
-  if (isdigit(*cur)) {
-    tscError("SML:0x%"PRIx64" Tag key cannot start with digit", info->id);
-    return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
-  }
   while (*cur != '\0') {
     if (len > TSDB_COL_NAME_LEN - 1) {
       tscError("SML:0x%"PRIx64" Key field cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1);
@@ -1918,9 +1924,11 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
 
-  pKV->key = calloc(len + 1, 1);
+  pKV->key = calloc(len + TS_ESCAPE_CHAR_SIZE + 1, 1);
   memcpy(pKV->key, key, len + 1);
-  //tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
+  strntolower_s(pKV->key, pKV->key, (int32_t)len);
+  addEscapeCharToString(pKV->key, len);
+  tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
   *index = cur + 1;
   return TSDB_CODE_SUCCESS;
 }
@@ -1931,7 +1939,7 @@ static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index,
   const char *start, *cur;
   int32_t ret = TSDB_CODE_SUCCESS;
   char *value = NULL;
-  uint16_t len = 0;
+  int16_t len = 0;
   bool searchQuote = false;
   start = cur = *index;
 
@@ -2012,18 +2020,12 @@ error:
 static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index,
                                    uint8_t *has_tags, SSmlLinesInfo* info) {
   const char *cur = *index;
-  uint16_t len = 0;
+  int16_t len = 0;
 
-  pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + 1, 1);    // +1 to avoid 1772 line over write
+  pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE, 1);
   if (pSml->stableName == NULL){
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
-  if (isdigit(*cur)) {
-    tscError("SML:0x%"PRIx64" Measurement field cannnot start with digit", info->id);
-    free(pSml->stableName);
-    pSml->stableName = NULL;
-    return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
-  }
 
   while (*cur != '\0') {
     if (len > TSDB_TABLE_NAME_LEN - 1) {
@@ -2060,7 +2062,7 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index
     pSml->stableName = NULL;
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
-  pSml->stableName[len] = '\0';
+  addEscapeCharToString(pSml->stableName, len);
   *index = cur + 1;
   tscDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len);
 
@@ -2116,17 +2118,11 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs,
       tscError("SML:0x%"PRIx64" Unable to parse value", info->id);
       goto error;
     }
-    if (!isField && (strcasecmp(pkv->key, "ID") == 0)) {
-      ret = isValidChildTableName(pkv->value, pkv->length, info);
-      if (ret) {
-        free(pkv->key);
-        free(pkv->value);
-        goto error;
-      }
-      smlData->childTableName = malloc( pkv->length + 1);
+    if (!isField && (strcasecmp(pkv->key, "`ID`") == 0)) {
+      smlData->childTableName = malloc(pkv->length + TS_ESCAPE_CHAR_SIZE + 1);
       memcpy(smlData->childTableName, pkv->value, pkv->length);
       strntolower_s(smlData->childTableName, smlData->childTableName, (int32_t)pkv->length);
-      smlData->childTableName[pkv->length] = '\0';
+      addEscapeCharToString(smlData->childTableName, (int32_t)pkv->length);
       free(pkv->key);
       free(pkv->value);
     } else {

src/client/src/tscParseOpenTSDB.c

@@ -37,15 +37,17 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index,
   const char *cur = *index;
   uint16_t len = 0;
 
-  pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN, 1);
+  pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE, 1);
   if (pSml->stableName == NULL) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
+  /*
   if (isdigit(*cur)) {
     tscError("OTD:0x%"PRIx64" Metric cannot start with digit", info->id);
     tfree(pSml->stableName);
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
+  */
 
   while (*cur != '\0') {
     if (len > TSDB_TABLE_NAME_LEN - 1) {
@@ -63,7 +65,7 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index,
       }
     }
 
-    pSml->stableName[len] = *cur;
+    pSml->stableName[len] = tolower(*cur);
 
     cur++;
     len++;
@@ -73,7 +75,7 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index,
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
 
-  pSml->stableName[len] = '\0';
+  addEscapeCharToString(pSml->stableName, len);
   *index = cur + 1;
   tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len);
 
@@ -207,10 +209,10 @@ static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj
   uint16_t len = 0;
 
   //key field cannot start with digit
-  if (isdigit(*cur)) {
-    tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id);
-    return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
-  }
+  //if (isdigit(*cur)) {
+  //  tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id);
+  //  return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
+  //}
   while (*cur != '\0') {
     if (len > TSDB_COL_NAME_LEN - 1) {
       tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1);
@@ -236,8 +238,10 @@ static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj
     return TSDB_CODE_TSC_DUP_TAG_NAMES;
   }
 
-  pKV->key = tcalloc(len + 1, 1);
+  pKV->key = tcalloc(len + TS_ESCAPE_CHAR_SIZE + 1, 1);
   memcpy(pKV->key, key, len + 1);
+  strntolower_s(pKV->key, pKV->key, (int32_t)len);
+  addEscapeCharToString(pKV->key, len);
   //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
   *index = cur + 1;
   return TSDB_CODE_SUCCESS;
@@ -312,15 +316,12 @@ static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs,
       tscError("OTD:0x%"PRIx64" Unable to parse value", info->id);
       return ret;
     }
-    if ((strcasecmp(pkv->key, "ID") == 0)) {
-      ret = isValidChildTableName(pkv->value, pkv->length, info);
-      if (ret) {
-        return ret;
-      }
-      *childTableName = malloc(pkv->length + 1);
+    if ((strcasecmp(pkv->key, "`ID`") == 0)) {
+      *childTableName = tcalloc(pkv->length + TS_ESCAPE_CHAR_SIZE + 1, 1);
       memcpy(*childTableName, pkv->value, pkv->length);
       (*childTableName)[pkv->length] = '\0';
       strntolower_s(*childTableName, *childTableName, (int32_t)pkv->length);
+      addEscapeCharToString(*childTableName, pkv->length);
       tfree(pkv->key);
       tfree(pkv->value);
     } else {
@@ -493,19 +494,22 @@ static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlL
       return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
   }
 
-  pSml->stableName = tcalloc(stableLen + 1, sizeof(char));
+  pSml->stableName = tcalloc(stableLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
   if (pSml->stableName == NULL){
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
+  /*
   if (isdigit(metric->valuestring[0])) {
     tscError("OTD:0x%"PRIx64" Metric cannot start with digit in JSON", info->id);
     tfree(pSml->stableName);
     return TSDB_CODE_TSC_INVALID_JSON;
   }
+  */
 
   tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1);
   strntolower_s(pSml->stableName, pSml->stableName, (int32_t)stableLen);
+  addEscapeCharToString(pSml->stableName, stableLen);
 
   return TSDB_CODE_SUCCESS;
 
@@ -896,13 +900,10 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs,
       return TSDB_CODE_TSC_INVALID_JSON;
     }
     size_t idLen = strlen(id->valuestring);
-    ret = isValidChildTableName(id->valuestring, (int16_t)idLen, info);
-    if (ret != TSDB_CODE_SUCCESS) {
-      return ret;
-    }
-    *childTableName = tcalloc(idLen + 1, sizeof(char));
+    *childTableName = tcalloc(idLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
     memcpy(*childTableName, id->valuestring, idLen);
     strntolower_s(*childTableName, *childTableName, (int32_t)idLen);
+    addEscapeCharToString(*childTableName, idLen);
 
     //check duplicate IDs
     cJSON_DeleteItemFromObject(tags, "ID");
@@ -936,8 +937,10 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs,
       tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters in JSON", info->id, TSDB_COL_NAME_LEN - 1);
       return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
     }
-    pkv->key = tcalloc(keyLen + 1, sizeof(char));
+    pkv->key = tcalloc(keyLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char));
     strncpy(pkv->key, tag->string, keyLen);
+    strntolower_s(pkv->key, pkv->key, (int32_t)keyLen);
+    addEscapeCharToString(pkv->key, keyLen);
     //value
     ret = parseValueFromJSON(tag, pkv, info);
     if (ret != TSDB_CODE_SUCCESS) {

src/client/src/tscSQLParser.c

@@ -3174,6 +3174,14 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken
 
   int16_t columnIndex = COLUMN_INDEX_INITIAL_VAL;
 
+  char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // create tmp buf to avoid alter orginal sqlstr
+  strncpy(tmpTokenBuf, pToken->z, pToken->n);
+  pToken->z = tmpTokenBuf;
+
+  if (pToken->type == TK_ID) {
+    tscRmEscapeAndTrimToken(pToken);
+  }
+
   for (int16_t i = 0; i < numOfCols; ++i) {
     if (pToken->n != strlen(pSchema[i].name)) {
       continue;
@@ -6313,6 +6321,13 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
     SStrToken    name = {.type = TK_STRING, .z = pItem->name, .n = (uint32_t)strlen(pItem->name)};
+    //handle Escape character backstick
+    if (name.z[0] == TS_ESCAPE_CHAR && name.z[name.n - 1] == TS_ESCAPE_CHAR) {
+      memmove(name.z, name.z + 1, name.n);
+      name.z[name.n - TS_ESCAPE_CHAR_SIZE] = '\0';
+      name.n -= TS_ESCAPE_CHAR_SIZE;
+    }
+
     if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
       return invalidOperationMsg(pMsg, msg17);
     }
@@ -6646,6 +6661,9 @@ int32_t validateColumnName(char* name) {
     }
 
     return validateColumnName(token.z);
+  } else if (token.type == TK_ID) {
+    strRmquoteEscape(name, token.n);
+    return TSDB_CODE_SUCCESS;
   } else {
     if (isNumber(&token)) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
@@ -7711,7 +7729,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
     SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j);
 
     SStrToken* pToken = &pCreateTableInfo->stableName;
-    
+
     bool dbIncluded = false;
     char      buf[TSDB_TABLE_FNAME_LEN];
     SStrToken sTblToken;
@@ -7771,10 +7789,19 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
 
       for (int32_t i = 0; i < nameSize; ++i) {
         SStrToken* sToken = taosArrayGet(pNameList, i);
+
+        char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // create tmp buf to avoid alter orginal sqlstr
+        strncpy(tmpTokenBuf, sToken->z, sToken->n);
+        sToken->z = tmpTokenBuf;
+
         if (TK_STRING == sToken->type) {
           tscDequoteAndTrimToken(sToken);
         }
 
+        if (TK_ID == sToken->type) {
+          tscRmEscapeAndTrimToken(sToken);
+        }
+
         tVariantListItem* pItem = taosArrayGet(pValList, i);
 
         findColumnIndex = false;

src/inc/taosdef.h

@@ -99,6 +99,7 @@ extern const int32_t TYPE_BYTES[15];
 
 #define TS_PATH_DELIMITER "."
 #define TS_ESCAPE_CHAR '`'
+#define TS_ESCAPE_CHAR_SIZE 2
 
 #define TSDB_TIME_PRECISION_MILLI 0
 #define TSDB_TIME_PRECISION_MICRO 1

src/util/src/ttokenizer.c

@@ -628,7 +628,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
         t0.n = 0;
         return t0;
       }
-    
+
       t = str[++(*i)];
     }
 

src/util/src/tutil.c

@@ -53,13 +53,13 @@ int32_t strdequote(char *z) {
 }
 
 
-int32_t strRmquote(char *z, int32_t len){  
+int32_t strRmquote(char *z, int32_t len){
     // delete escape character: \\, \', \"
     char delim = z[0];
     if (delim != '\'' && delim != '\"') {
       return len;
     }
-  
+
     int32_t cnt = 0;
     int32_t j = 0;
     for (uint32_t k = 1; k < len - 1; ++k) {
@@ -74,23 +74,24 @@ int32_t strRmquote(char *z, int32_t len){
           continue;
         }
       }
-  
+
       z[j] = z[k];
       j++;
     }
-  
+
     z[j] = 0;
-    
+
     return len - 2 - cnt;
 }
 
 int32_t strRmquoteEscape(char *z, int32_t len) {
   if (len <= 0) return len;
-  
+
   if (z[0] == '\'' || z[0] == '\"') {
     return strRmquote(z, len);
   } else if (len > 1 && z[0] == TS_ESCAPE_CHAR && z[len - 1] == TS_ESCAPE_CHAR) {
     memmove(z, z + 1, len - 2);
+    z[len - 2] = '\0';
     return len - 2;
   }
 

tests/pytest/insert/insertJSONPayload.py

@@ -36,7 +36,7 @@ class TDTestCase:
         print("============= step0 : test metric  ================")
         payload = ['''
         {
-	    "metric":	"`.stb.0.`",
+	    "metric":	".stb.0.",
 	    "timestamp":	1626006833610,
 	    "value":	10,
 	    "tags":	{
@@ -664,6 +664,183 @@ class TDTestCase:
         tdSql.checkData(9, 1, "BINARY")
         tdSql.checkData(10, 1, "NCHAR")
 
+        ### special characters ###
+
+        payload = ['''
+        {
+	    "metric":	 "1234",
+	    "timestamp": 1626006833,
+	    "value":     1,
+	    "tags":	{
+		"id":	        "123",
+		"456":	        true,
+		"int":	        false,
+		"double":       1,
+		"into":         1,
+		"from":         2,
+		"!@#$.%^&*()":	"123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"
+	    }
+        }
+        ''']
+        code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value)
+        print("schemaless_insert result {}".format(code))
+
+        tdSql.query("describe `1234`")
+        tdSql.checkRows(8)
+
+        tdSql.query("select * from `123`")
+        tdSql.checkRows(1)
+
+        payload = ['''
+        {
+	    "metric":	 "int",
+	    "timestamp": 1626006833,
+	    "value":     1,
+	    "tags":	{
+		"id":	        "and",
+		"456":	        true,
+		"int":	        false,
+		"double":       1,
+		"into":         1,
+		"from":         2,
+		"!@#$.%^&*()":	"123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"
+	    }
+        }
+        ''']
+        code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value)
+        print("schemaless_insert result {}".format(code))
+
+        tdSql.query("describe `int`")
+        tdSql.checkRows(8)
+
+        tdSql.query("select * from `and`")
+        tdSql.checkRows(1)
+
+        payload = ['''
+        {
+	    "metric":	 "double",
+	    "timestamp": 1626006833,
+	    "value":     1,
+	    "tags":	{
+		"id":	        "for",
+		"456":	        true,
+		"int":	        false,
+		"double":       1,
+		"into":         1,
+		"from":         2,
+		"!@#$.%^&*()":	"123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"
+	    }
+        }
+        ''']
+        code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value)
+        print("schemaless_insert result {}".format(code))
+
+        tdSql.query("describe `double`")
+        tdSql.checkRows(8)
+
+        tdSql.query("select * from `for`")
+        tdSql.checkRows(1)
+
+        payload = ['''
+        {
+	    "metric":	 "from",
+	    "timestamp": 1626006833,
+	    "value":     1,
+	    "tags":	{
+		"id":	        "!@#.^&",
+		"456":	        true,
+		"int":	        false,
+		"double":       1,
+		"into":         1,
+		"from":         2,
+		"!@#$.%^&*()":	"123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"
+	    }
+        }
+        ''']
+        code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value)
+        print("schemaless_insert result {}".format(code))
+
+        tdSql.query("describe `from`")
+        tdSql.checkRows(8)
+
+        tdSql.query("select * from `!@#.^&`")
+        tdSql.checkRows(1)
+
+        payload = ['''
+        {
+	    "metric":	 "!@#$.%^&*()",
+	    "timestamp": 1626006833,
+	    "value":     1,
+	    "tags":	{
+		"id":	        "none",
+		"456":	        true,
+		"int":	        false,
+		"double":       1,
+		"into":         1,
+		"from":         2,
+		"!@#$.%^&*()":	"123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"
+	    }
+        }
+        ''']
+        code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value)
+        print("schemaless_insert result {}".format(code))
+
+        tdSql.query("describe `!@#$.%^&*()`")
+        tdSql.checkRows(8)
+
+        tdSql.query("select * from `none`")
+        tdSql.checkRows(1)
+
+        payload = ['''
+        {
+	    "metric":	"STABLE",
+	    "timestamp":	{
+		"value":	1626006833,
+		"type":	"s"
+	    },
+	    "value":	{
+		"value":	"hello",
+		"type":	"nchar"
+	    },
+	    "tags":	{
+                "id":   "KEY",
+		"456":	{
+			"value":	true,
+			"type":	"bool"
+		},
+		"int":	{
+			"value":	127,
+			"type":	"tinyint"
+		},
+		"double":{
+			"value":	32767,
+			"type":	"smallint"
+		},
+		"into":	{
+			"value":	2147483647,
+			"type":	"int"
+		},
+		"INSERT":	{
+			"value":	9.2233720368547758e+18,
+			"type":	"bigint"
+		},
+		"!@#$.%^&*()":	{
+			"value":	11.12345,
+			"type":	"float"
+		}
+	    }
+        }
+        ''']
+
+        code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value)
+        print("schemaless_insert result {}".format(code))
+
+        tdSql.query("describe `stable`")
+        tdSql.checkRows(8)
+
+        tdSql.query("select * from `key`")
+        tdSql.checkRows(1)
+
 
     def stop(self):
         tdSql.close()

tests/pytest/insert/insertTelnetLines.py

@@ -32,10 +32,10 @@ class TDTestCase:
         ### metric ###
         print("============= step1 : test metric  ================")
         lines0 = [
-                        "stb0_0 1626006833639 4i8 host=\"host0\" interface=\"eth0\"",
-                        "stb0_1 1626006833639 4i8 host=\"host0\" interface=\"eth0\"",
-                        "stb0_2 1626006833639 4i8 host=\"host0\" interface=\"eth0\"",
-                        "`.stb0.3.` 1626006833639 4i8 host=\"host0\" interface=\"eth0\"",
+                        "stb0_0 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"",
+                        "stb0_1 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"",
+                        "stb0_2 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"",
+                        ".stb0.3. 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"",
                    ]
 
         code = self._conn.schemaless_insert(lines0, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value)
@@ -304,6 +304,56 @@ class TDTestCase:
 
         tdSql.checkData(0, 0, "child_table1")
 
+        ### special characters and keywords ###
+        print("============= step4 : test special characters and keywords  ================")
+        lines4_1 = [
+                        "1234 1626006833610ms 1 id=123 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false",
+                        "int 1626006833610ms 2 id=and 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false",
+                        "double 1626006833610ms 2 id=for 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false",
+                        "from 1626006833610ms 2 id=!@#.^& 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false",
+                        "!@#$.%^&*() 1626006833610ms 2 id=none 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false",
+                        "STABLE 1626006833610ms 2 id=KEY 456=true int=true double=false TAG=1 FROM=2 COLUMN=false",
+                     ]
+
+        code = self._conn.schemaless_insert(lines4_1, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value)
+        print("schemaless_insert result {}".format(code))
+
+        tdSql.query('describe `1234`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `int`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `double`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `from`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `!@#$.%^&*()`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `stable`')
+        tdSql.checkRows(8)
+
+        tdSql.query('select * from `123`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `and`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `for`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `!@#.^&`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `none`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `key`')
+        tdSql.checkRows(1)
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/insert/line_insert.py

@@ -85,6 +85,53 @@ class TDTestCase:
 
         tdSql.query('select tbname, * from childtable')
         tdSql.checkRows(1)
+
+        ###Special Character and keyss
+        self._conn.schemaless_insert([
+                                "1234,id=3456,abc=4i64,def=3i64 123=3i64,int=2i64,bool=false,into=5f64,column=7u64,!@#$.%^&*()=false 1626006933641",
+                                "int,id=and,123=4i64,smallint=5f64,double=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false abc=false 1626006933654",
+                                "double,id=for,123=4i64,smallint=5f64,double=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false abc=false 1626006933664",
+                                "from,id=!@#$.%^,123=4i64,smallint=5f64,double=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false abc=false 1626006933674",
+                                "!@#$.%^&*(),id=none,123=4i64,smallint=5f64,double=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false abc=false 1626006933684",
+                                "STABLE,id=CREATE,123=4i64,smallint=5f64,DOUBLE=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false SELECT=false 1626006933684",
+                                ], TDSmlProtocolType.LINE.value, TDSmlTimestampType.MILLI_SECOND.value)
+        tdSql.execute('reset query cache')
+
+        tdSql.query('describe `1234`')
+        tdSql.checkRows(9)
+
+        tdSql.query('describe `int`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `double`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `from`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `!@#$.%^&*()`')
+        tdSql.checkRows(8)
+
+        tdSql.query('describe `stable`')
+        tdSql.checkRows(8)
+
+        tdSql.query('select * from `3456`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `and`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `for`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `!@#$.%^`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `none`')
+        tdSql.checkRows(1)
+
+        tdSql.query('select * from `create`')
+        tdSql.checkRows(1)
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/script/fullGeneralSuite.sim

@@ -228,4 +228,6 @@ run general/db/show_create_table.sim
 run general/parser/like.sim
 run general/parser/regex.sim
 run general/parser/tbname_escape.sim
+run general/parser/columnName_escape.sim
+run general/parser/tagName_escape.sim
 run general/parser/interp_blocks.sim

tests/script/general/parser/columnName_escape.sim

+system sh/stop_dnodes.sh
+
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/exec.sh -n dnode1 -s start
+
+sleep 100
+sql connect
+print ======================== dnode1 start
+
+sql create database colesc;
+
+sql use colesc;
+
+print ======================= test create table/stable
+
+
+sql create table tb0 (ts timestamp, `123` int, `123 456` int, `123.abc` int)
+sql create table tb1 (ts timestamp, `!%^&*()` int)
+sql create table tb2 (ts timestamp, `int` int, `bool` int, `double` int, `INTO` int, `COLUMN` int)
+
+sql create table stb0 (ts timestamp, `123` int, `123 456` int, `123.abc` int) tags (t1 int)
+sql create table stb1 (ts timestamp, `!%^&*()` int) tags (t2 int)
+sql create table stb2 (ts timestamp, `int` int, `bool` int, `double` int, `INTO` int, `COLUMN` int) tags (t3 int)
+
+sql create table ctb0 using stb0 tags (1)
+sql create table ctb1 using stb1 tags (1)
+sql create table ctb2 using stb2 tags (1)
+
+##check table
+sql describe tb0;
+if $rows != 4 then
+  return -1
+endi
+if $data10 != @123@ then
+  return -1
+endi
+if $data20 != @123 456@ then
+  return -1
+endi
+if $data30 != @123.abc@ then
+  return -1
+endi
+
+sql describe tb1;
+if $rows != 2 then
+  return -1
+endi
+if $data10 != @!%^&*()@ then
+  return -1
+endi
+
+sql describe tb2;
+if $rows != 6 then
+  return -1
+endi
+if $data10 != @int@ then
+  return -1
+endi
+if $data20 != @bool@ then
+  return -1
+endi
+if $data30 != @double@ then
+  return -1
+endi
+if $data40 != @INTO@ then
+  return -1
+endi
+if $data50 != @COLUMN@ then
+  return -1
+endi
+##check stable
+sql describe stb0;
+if $rows != 5 then
+  return -1
+endi
+if $data10 != @123@ then
+  return -1
+endi
+if $data20 != @123 456@ then
+  return -1
+endi
+if $data30 != @123.abc@ then
+  return -1
+endi
+
+sql describe stb1;
+if $rows != 3 then
+  return -1
+endi
+if $data10 != @!%^&*()@ then
+  return -1
+endi
+
+sql describe stb2;
+if $rows != 7 then
+  return -1
+endi
+if $data10 != @int@ then
+  return -1
+endi
+if $data20 != @bool@ then
+  return -1
+endi
+if $data30 != @double@ then
+  return -1
+endi
+if $data40 != @INTO@ then
+  return -1
+endi
+if $data50 != @COLUMN@ then
+  return -1
+endi
+
+
+print ======================= test Alter columns for table/stable
+
+##Add column
+sql_error alter table tb0 add column `123` int
+sql_error alter table tb0 add column `123 456` int
+sql_error alter table tb0 add column `123.abc` int
+
+sql_error alter table ctb0 add column `1234`
+
+sql alter table tb0 add column `!%^&*()` int
+sql alter table tb0 add column `int` int
+sql alter table tb0 add column `bool` int
+sql alter table tb0 add column `double` int
+sql alter table tb0 add column `INTO` nchar(10)
+sql alter table tb0 add column `COLUMN` binary(10)
+
+sql alter table stb0 add column `!%^&*()` int
+sql alter table stb0 add column `int` int
+sql alter table stb0 add column `bool` int
+sql alter table stb0 add column `double` int
+sql alter table stb0 add column `INTO` nchar(10)
+sql alter table stb0 add column `COLUMN` binary(10)
+
+
+##check table
+sql describe tb0;
+if $rows != 10 then
+  return -1
+endi
+if $data40 != @!%^&*()@ then
+  return -1
+endi
+if $data50 != @int@ then
+  return -1
+endi
+if $data60 != @bool@ then
+  return -1
+endi
+if $data70 != @double@ then
+  return -1
+endi
+if $data80 != @INTO@ then
+  return -1
+endi
+if $data90 != @COLUMN@ then
+  return -1
+endi
+
+#check stable
+sql describe stb0;
+if $rows != 11 then
+  return -1
+endi
+if $data40 != @!%^&*()@ then
+  return -1
+endi
+if $data50 != @int@ then
+  return -1
+endi
+if $data60 != @bool@ then
+  return -1
+endi
+if $data70 != @double@ then
+  return -1
+endi
+if $data80 != @INTO@ then
+  return -1
+endi
+if $data90 != @COLUMN@ then
+  return -1
+endi
+
+##Drop column
+
+sql_error alter table ctb0 drop column `123`
+sql_error alter table ctb0 drop column `123 456`
+sql_error alter table ctb0 drop column `123.abc`
+
+sql alter table tb0 drop column `!%^&*()`
+sql alter table tb0 drop column `int`
+sql alter table tb0 drop column `bool`
+sql alter table tb0 drop column `double`
+sql alter table tb0 drop column `INTO`
+sql alter table tb0 drop column `COLUMN`
+
+sql alter table stb0 drop column `!%^&*()`
+sql alter table stb0 drop column `int`
+sql alter table stb0 drop column `bool`
+sql alter table stb0 drop column `double`
+sql alter table stb0 drop column `INTO`
+sql alter table stb0 drop column `COLUMN`
+
+##check table
+sql describe tb0;
+if $rows != 4 then
+  return -1
+endi
+if $data10 != @123@ then
+  return -1
+endi
+if $data20 != @123 456@ then
+  return -1
+endi
+if $data30 != @123.abc@ then
+  return -1
+endi
+
+##check stable
+sql describe stb0;
+if $rows != 5 then
+  return -1
+endi
+if $data10 != @123@ then
+  return -1
+endi
+if $data20 != @123 456@ then
+  return -1
+endi
+if $data30 != @123.abc@ then
+  return -1
+endi
+
+##Modify column for binary/nchar length
+
+sql alter table tb0 add column `INTO` nchar(10)
+sql alter table tb0 add column `COLUMN` binary(10)
+
+sql alter table stb0 add column `INTO` nchar(10)
+sql alter table stb0 add column `COLUMN` binary(10)
+
+sql alter table tb0 modify column `INTO` nchar(15)
+sql alter table tb0 modify column `COLUMN` binary(15)
+
+sql alter table stb0 modify column `INTO` nchar(15)
+sql alter table stb0 modify column `COLUMN` binary(15)
+
+sql describe tb0;
+if $rows != 6 then
+  return -1
+endi
+if $data42 != @15@ then
+  return -1
+endi
+if $data52 != @15@ then
+  return -1
+endi
+
+sql describe stb0;
+if $rows != 7 then
+  return -1
+endi
+if $data42 != @15@ then
+  return -1
+endi
+if $data52 != @15@ then
+  return -1
+endi
+
+print ======================= test insert columns for table/stable
+
+sql insert into tb0 (ts, `123`, `123 456`, `123.abc`) values (now, 1, 1, 1)
+sql insert into tb1 (ts, `!%^&*()`) values (now, 1)
+sql insert into tb2 (ts, `int`, `bool`, `double`, `INTO`, `COLUMN`) values (now, 1, 1, 1, 1, 1)
+
+sql insert into ctb0 (ts, `123`, `123 456`, `123.abc`) values (now, 1, 1, 1)
+sql insert into ctb1 (ts, `!%^&*()`) values (now, 1)
+sql insert into ctb2 (ts, `int`, `bool`, `double`, `INTO`, `COLUMN`) values (now, 1, 1, 1, 1, 1)
+
+sql select * from tb0;
+if $rows != 1 then
+  return -1
+endi
+
+sql select * from tb1;
+if $rows != 1 then
+  return -1
+endi
+
+sql select * from tb2;
+if $rows != 1 then
+  return -1
+endi
+
+sql select * from ctb0;
+if $rows != 1 then
+  return -1
+endi
+
+sql select * from ctb1;
+if $rows != 1 then
+  return -1
+endi
+
+sql select * from ctb2;
+if $rows != 1 then
+  return -1
+endi
+
+print ======================= test select columns for table/stable
+
+sql select `123`,`123 456`,`123.abc` from tb0;
+
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != 1 then
+  return -1
+endi
+
+if $data02 != 1 then
+  return -1
+endi
+
+sql select `!%^&*()` from tb1;
+
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+sql select `int`,`bool`,`double`,`INTO`,`COLUMN` from tb2;
+
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != 1 then
+  return -1
+endi
+
+if $data02 != 1 then
+  return -1
+endi
+
+if $data03 != 1 then
+  return -1
+endi
+
+if $data04 != 1 then
+  return -1
+endi
+
+
+sql select `123`,`123 456`,`123.abc` from stb0;
+
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != 1 then
+  return -1
+endi
+
+if $data02 != 1 then
+  return -1
+endi
+
+sql select `!%^&*()` from stb1;
+
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+sql select `int`,`bool`,`double`,`INTO`,`COLUMN` from stb2;
+
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != 1 then
+  return -1
+endi
+
+if $data02 != 1 then
+  return -1
+endi
+
+if $data03 != 1 then
+  return -1
+endi
+
+if $data04 != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/general/parser/tagName_escape.sim

+system sh/stop_dnodes.sh
+
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/exec.sh -n dnode1 -s start
+
+sleep 100
+sql connect
+print ======================== dnode1 start
+
+sql create database tagesc;
+
+sql use tagesc;
+
+print ======================= test create table/stable
+
+sql create stable stb0 (ts timestamp, c0 int) tags (`123` int, `123 456` int, `123.abc` int)
+sql create stable stb1 (ts timestamp, c1 int) tags (`!%^&*()` int)
+sql create stable stb2 (ts timestamp, c2 int) tags (`int` int, `bool` int, `double` int, `INTO` int, `COLUMN` int)
+
+sql create table ctb0 using stb0 (`123`, `123 456`, `123.abc`) tags (1, 1, 1)
+sql create table ctb1 using stb1 (`!%^&*()`) tags (1)
+sql create table ctb2 using stb2 (`int`, `bool`, `double`, `INTO`, `COLUMN`) tags (1, 1, 1, 1, 1)
+
+##check table
+sql describe ctb0;
+if $rows != 5 then
+  return -1
+endi
+if $data20 != @123@ then
+  return -1
+endi
+if $data30 != @123 456@ then
+  return -1
+endi
+if $data40 != @123.abc@ then
+  return -1
+endi
+
+sql describe ctb1;
+if $rows != 3 then
+  return -1
+endi
+if $data20 != @!%^&*()@ then
+  return -1
+endi
+
+sql describe ctb2;
+if $rows != 7 then
+  return -1
+endi
+if $data20 != @int@ then
+  return -1
+endi
+if $data30 != @bool@ then
+  return -1
+endi
+if $data40 != @double@ then
+  return -1
+endi
+if $data50 != @INTO@ then
+  return -1
+endi
+if $data60 != @COLUMN@ then
+  return -1
+endi
+
+print ======================= test Alter tags for stable
+
+##ADD TAG
+sql_error alter stable stb0 add tag `123` int
+sql_error alter stable stb0 add tag `123 456` int
+sql_error alter stable stb0 add tag `123.abc` int
+
+sql alter stable stb0 add tag `!%^&*()` int
+sql alter stable stb0 add tag `int` int
+sql alter stable stb0 add tag `bool` int
+sql alter stable stb0 add tag `double` int
+sql alter stable stb0 add tag `INTO` int
+sql alter stable stb0 add tag `COLUMN` int
+
+
+sql describe stb0;
+if $rows != 11 then
+  return -1
+endi
+if $data50 != @!%^&*()@ then
+  return -1
+endi
+if $data60 != @int@ then
+  return -1
+endi
+if $data70 != @bool@ then
+  return -1
+endi
+if $data80 != @double@ then
+  return -1
+endi
+if $data90 != @INTO@ then
+  return -1
+endi
+
+
+##DROP TAG
+sql alter stable stb0 drop tag `!%^&*()`
+sql alter stable stb0 drop tag `int`
+sql alter stable stb0 drop tag `bool`
+sql alter stable stb0 drop tag `double`
+sql alter stable stb0 drop tag `INTO`
+sql alter stable stb0 drop tag `COLUMN`
+
+
+sql describe stb0;
+if $rows != 5 then
+  return -1
+endi
+if $data20 != @123@ then
+  return -1
+endi
+if $data30 != @123 456@ then
+  return -1
+endi
+if $data40 != @123.abc@ then
+  return -1
+endi
+
+
+##CHANGE TAG
+
+sql alter stable stb0 change tag `123` `321`
+sql alter stable stb0 change tag `123 456` `456 123`
+#sql alter stable stb0 change tag `123.abc` `abc.123`
+#change tag has bug when using dot in tagname
+
+sql describe stb0;
+if $rows != 5 then
+  return -1
+endi
+if $data20 != @321@ then
+  return -1
+endi
+if $data30 != @456 123@ then
+  return -1
+endi
+
+##SET TAG
+
+sql insert into ctb0 values (now, 1)
+sql insert into ctb1 values (now, 1)
+sql insert into ctb2 values (now, 1)
+
+sql alter table ctb0 set tag `321`=2
+sql alter table ctb0 set tag `456 123`=2
+#sql alter table ctb0 set tag `abc.123`=2
+#change tag has bug when using dot in tagname
+
+print ======================= test insert specific tags automatically create table
+
+sql alter table ctb0 set tag `321`=2
+sql alter table ctb0 set tag `321`=2
+sql insert into ctb0_0 using stb0 (`321`, `456 123`, `123.abc`) tags (1, 1, 1) values (now + 10s, 5)
+sql insert into ctb1_0 using stb1 (`!%^&*()`) tags (1) values (now + 10s, 5)
+sql insert into ctb2_0 using stb2 (`int`, `bool`, `double`, `INTO`, `COLUMN`) tags (1, 1, 1, 1, 1) values (now + 10s, 5)
+sql insert into ctb2_1 using stb2 (`int`, `bool`, `INTO`, `COLUMN`) tags (1, 1, 1, 1) values (now + 10s, 5)
+
+sql select * from stb0;
+if $rows != 2 then
+  return -1
+endi
+
+sql select * from stb1;
+if $rows != 2 then
+  return -1
+endi
+
+sql select * from stb2;
+if $rows != 3 then
+  return -1
+endi
+
+if $data24 != NULL then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT