Merge pull request #8273 from taosdata/fix/TD-10639

support line protocol string with space and comma inside double quote

src/client/src/tscParseLineProtocol.c

@@ -1871,7 +1871,7 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash
 
   //key field cannot start with digit
   if (isdigit(*cur)) {
-    tscError("SML:0x%"PRIx64" Tag key cannnot start with digit", info->id);
+    tscError("SML:0x%"PRIx64" Tag key cannot start with digit", info->id);
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
   while (*cur != '\0') {
@@ -1885,6 +1885,8 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash
     }
     //Escape special character
     if (*cur == '\\') {
+      //TODO: escape will work after column & tag
+      //support spcial characters
       escapeSpecialCharacter(2, &cur);
     }
     key[len] = *cur;
@@ -1911,13 +1913,42 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash
 static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index,
                           bool *is_last_kv, SSmlLinesInfo* info, bool isTag) {
   const char *start, *cur;
+  int32_t ret = TSDB_CODE_SUCCESS;
   char *value = NULL;
   uint16_t len = 0;
+  bool searchQuote = false;
   start = cur = *index;
 
+  //if field value is string
+  if (!isTag) {
+    if (*cur == '"') {
+      searchQuote = true;
+      cur += 1;
+      len += 1;
+    } else if (*cur == 'L' && *(cur + 1) == '"') {
+      searchQuote = true;
+      cur += 2;
+      len += 2;
+    }
+  }
+
   while (1) {
     // unescaped ',' or ' ' or '\0' identifies a value
-    if ((*cur == ',' || *cur == ' ' || *cur == '\0') && *(cur - 1) != '\\') {
+    if (((*cur == ',' || *cur == ' ' ) && *(cur - 1) != '\\') || *cur == '\0') {
+      if (searchQuote == true) {
+        //first quote ignored while searching
+        if (*(cur - 1) == '"' && len != 1 && len != 2) {
+          *is_last_kv = (*cur == ' ' || *cur == '\0') ? true : false;
+          break;
+        } else if (*cur == '\0') {
+          ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
+          goto error;
+        } else {
+          cur++;
+          len++;
+          continue;
+        }
+      }
       //unescaped ' ' or '\0' indicates end of value
       *is_last_kv = (*cur == ' ' || *cur == '\0') ? true : false;
       if (*cur == ' ' && *(cur + 1) == ' ') {
@@ -1929,7 +1960,7 @@ static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index,
     }
     //Escape special character
     if (*cur == '\\') {
-      escapeSpecialCharacter(2, &cur);
+      escapeSpecialCharacter(isTag ? 2 : 3, &cur);
     }
     cur++;
     len++;
@@ -1946,16 +1977,20 @@ static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index,
   if (!convertSmlValueType(pKV, value, len, info, isTag)) {
     tscError("SML:0x%"PRIx64" Failed to convert sml value string(%s) to any type",
             info->id, value);
-    //free previous alocated key field
-    free(pKV->key);
-    pKV->key = NULL;
     free(value);
-    return TSDB_CODE_TSC_INVALID_VALUE;
+    ret = TSDB_CODE_TSC_INVALID_VALUE;
+    goto error;
   }
   free(value);
 
   *index = (*cur == '\0') ? cur : cur + 1;
-  return TSDB_CODE_SUCCESS;
+  return ret;
+
+error:
+  //free previous alocated key field
+  free(pKV->key);
+  pKV->key = NULL;
+  return ret;
 }
 
 static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index,

src/client/src/tscParseOpenTSDB.c

@@ -138,21 +138,41 @@ static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const ch
   const char *start, *cur;
   int32_t ret = TSDB_CODE_SUCCESS;
   int len = 0;
+  bool searchQuote = false;
   char key[] = OTD_METRIC_VALUE_COLUMN_NAME;
   char *value = NULL;
 
   start = cur = *index;
 
+  //if metric value is string
+  if (*cur == '"') {
+    searchQuote = true;
+    cur += 1;
+    len += 1;
+  } else if (*cur == 'L' && *(cur + 1) == '"') {
+    searchQuote = true;
+    cur += 2;
+    len += 2;
+  }
+
   while(*cur != '\0') {
     if (*cur == ' ') {
-      if (*cur == ' ') {
-        if (*(cur + 1) != ' ') {
-          break;
+      if (searchQuote == true) {
+        if (*(cur - 1) == '"' && len != 1 && len != 2) {
+          searchQuote = false;
         } else {
           cur++;
+          len++;
           continue;
         }
       }
+
+      if (*(cur + 1) != ' ') {
+        break;
+      } else {
+        cur++;
+        continue;
+      }
     }
     cur++;
     len++;

tests/pytest/insert/insertTelnetLines.py

@@ -29,7 +29,6 @@ class TDTestCase:
         tdSql.execute("create database if not exists test precision 'us'")
         tdSql.execute('use test')
 
-
         ### metric ###
         print("============= step1 : test metric  ================")
         lines0 = [
@@ -215,7 +214,7 @@ class TDTestCase:
 
         #binary
         lines2_7 = [
-                        "stb2_7 1626006833610ms \"binary_val.!@#$%^&*\" host=\"host0\"",
+                        "stb2_7 1626006833610ms \"  binary_val  .!@#$%^&*  \" host=\"host0\"",
                         "stb2_7 1626006833620ms \"binary_val.:;,./?|+-=\" host=\"host0\"",
                         "stb2_7 1626006833630ms \"binary_val.()[]{}<>\" host=\"host0\""
                      ]
@@ -232,7 +231,7 @@ class TDTestCase:
 
         #nchar
         lines2_8 = [
-                        "stb2_8 1626006833610ms L\"nchar_val数值一\" host=\"host0\"",
+                        "stb2_8 1626006833610ms L\"  nchar_val  数值一  \" host=\"host0\"",
                         "stb2_8 1626006833620ms L\"nchar_val数值二\" host=\"host0\""
                      ]
 

tests/pytest/insert/line_insert.py

@@ -31,9 +31,9 @@ class TDTestCase:
 
         tdSql.execute('create stable ste(ts timestamp, f int) tags(t1 bigint)')
 
-        lines = [   "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000",
+        lines = [   "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"\"\"a    pa,\"s   si,t \"\"\",c2=false,c4=4f64 1626006833639000000",
                     "st,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000",
-                    "ste,t2=5f64,t3=L\"ste\" c1=true,c2=4i64,c3=\"iam\" 1626056811823316532",
+                    "ste,t2=5f64,t3=L\"ste\" c1=true,c2=4i64,c3=\" i,\"a \"m,\"\"\" 1626056811823316532",
                     "stf,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000",
                     "st,t1=4i64,t2=5f64,t3=\"t4\" c1=3i64,c3=L\"passitagain\",c2=true,c4=5f64 1626006833642000000",
                     "ste,t2=5f64,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532",