Merge pull request #19335 from taosdata/refact/submit_req_marks

fix:Refact/submit req marks

source/client/src/clientSmlJson.c

@@ -409,7 +409,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){
     (*start)++;
   }
 
-  if(unlikely(index != OTD_JSON_FIELDS_NUM)) {
+  if(unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) {
     uError("elements != %d", OTD_JSON_FIELDS_NUM)
     return -1;
   }
@@ -1161,6 +1161,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) {
   int32_t payloadNum = 1 << 15;
   int32_t ret = TSDB_CODE_SUCCESS;
 
+  uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload);
   int cnt = 0;
   char *dataPointStart = payload;
   while (1) {

source/client/src/clientSmlTelnet.c

@@ -23,8 +23,8 @@
 int32_t is_same_child_table_telnet(const void *a, const void *b){
   SSmlLineInfo *t1 = (SSmlLineInfo *)a;
   SSmlLineInfo *t2 = (SSmlLineInfo *)b;
-  uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen,
-         t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags);
+//  uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen,
+//         t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags);
   if(t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL
       || t1->tags == NULL || t2->tags == NULL)
     return 1;

source/libs/parser/src/parInsertSml.c

@@ -189,14 +189,23 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
   SSchema* pColSchema = schema + index;
   SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
   SSmlKv*  kv = (SSmlKv*)data;
+  if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0){
+    ret = TSDB_CODE_SML_INVALID_DATA;
+    goto end;
+  }
   if (kv->type == TSDB_DATA_TYPE_NCHAR) {
     int32_t len = 0;
-    char*   pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE);
+    int64_t size = pColSchema->bytes - VARSTR_HEADER_SIZE;
+    if(size <= 0){
+      ret = TSDB_CODE_SML_INVALID_DATA;
+      goto end;
+    }
+    char*   pUcs4 = taosMemoryCalloc(1, size);
     if (NULL == pUcs4) {
       ret = TSDB_CODE_OUT_OF_MEMORY;
       goto end;
     }
-    if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) {
+    if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) {
       if (errno == E2BIG) {
         ret = TSDB_CODE_PAR_VALUE_TOO_LONG;
         goto end;

tests/system-test/1-insert/opentsdb_json_taosc_insert.py

@@ -1432,9 +1432,9 @@ class TDTestCase:
         self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None)
         query_sql = 'select * from `rFa$sta`'
         query_res = tdSql.query(query_sql, True)
-        tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, 'rFas$ta_1', 'ncharTagValue', 2147483647, 9223372036854775807, 22.123456789, 'binaryTagValue', 32767, 11.12345027923584, False, 127)])
+        tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, False, 127, 32767, 2147483647, 9223372036854775807, 11.12345027923584, 22.123456789, 'binaryTagValue', 'ncharTagValue', 'rFas$ta_1')])
         col_tag_res = tdSql.getColNameList(query_sql)
-        tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'id', 't!@#$%^&*()_+[];:<>?,9', 't$3', 't%4', 't&6', 't*7', 't@2', 't^5', 'Tt!0', 'tT@1'])
+        tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't@2', 't$3', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9', 'id'])
         tdSql.execute('drop table `rFa$sta`')
 
     def pointTransCheckCase(self, value_type="obj"):
@@ -1758,7 +1758,7 @@ class TDTestCase:
         self.batchErrorInsertCheckCase()
         self.chineseCheckCase()
         # self.spellCheckCase()
-        # self.tbnameTagsColsNameCheckCase()
+        self.tbnameTagsColsNameCheckCase()
         # # MultiThreads
         # self.sStbStbDdataInsertMultiThreadCheckCase()
         # self.sStbStbDdataAtInsertMultiThreadCheckCase()

tests/system-test/2-query/sml.py

@@ -71,23 +71,26 @@ class TDTestCase:
         tdSql.checkData(0, 2, "web01")
 
         tdSql.query(f"select distinct tbname from {dbname}.`sys.cpu.nice`")
-        tdSql.checkRows(2)
+        tdSql.checkRows(3)
 
         tdSql.query(f"select * from {dbname}.`sys.cpu.nice` order by _ts")
-        tdSql.checkRows(2)
-        tdSql.checkData(0, 1, 9.000000000)
-        tdSql.checkData(0, 2, "web02")
+        tdSql.checkRows(4)
+        tdSql.checkData(0, 1, 13.000000000)
+        tdSql.checkData(0, 2, "web01")
         tdSql.checkData(0, 3, None)
         tdSql.checkData(0, 4, "lga")
 
-        tdSql.checkData(1, 1, 18.000000000)
-        tdSql.checkData(1, 2, "web01")
-        tdSql.checkData(1, 3, "t1")
+        tdSql.checkData(1, 1, 9.000000000)
+        tdSql.checkData(1, 2, "web02")
+        tdSql.checkData(3, 3, "t1")
         tdSql.checkData(0, 4, "lga")
 
         tdSql.query(f"select * from {dbname}.macylr")
         tdSql.checkRows(2)
 
+        tdSql.query(f"select * from {dbname}.qelhxo")
+        tdSql.checkRows(5)
+
         tdSql.query(f"desc {dbname}.macylr")
         tdSql.checkRows(25)
         return

utils/test/c/sml_test.c

@@ -114,13 +114,53 @@ int smlProcess_json1_Test() {
   taos_free_result(pRes);
 
   const char *sql[] = {
-      "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]"
+      "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]"
   };
-  pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL,
+
+  char *sql1[1] = {0};
+  for(int i = 0; i < 1; i++){
+    sql1[i] = taosMemoryCalloc(1, 1024);
+    strncpy(sql1[i], sql[i], 1023);
+  }
+
+  pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL,
                                 TSDB_SML_TIMESTAMP_NANO_SECONDS);
-  printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
   int code = taos_errno(pRes);
+  if(code != 0){
+    printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+  }else{
+    printf("%s result:success\n", __FUNCTION__);
+  }
+  taos_free_result(pRes);
+
+  for(int i = 0; i < 1; i++){
+    taosMemoryFree(sql1[i]);
+  }
+
+  const char *sql2[] = {
+      "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]",
+  };
+
+  char *sql3[1] = {0};
+  for(int i = 0; i < 1; i++){
+    sql3[i] = taosMemoryCalloc(1, 1024);
+    strncpy(sql3[i], sql2[i], 1023);
+  }
+
+  pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL,
+                                TSDB_SML_TIMESTAMP_NANO_SECONDS);
+  code = taos_errno(pRes);
+  if(code != 0){
+    printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+  }else{
+    printf("%s result:success\n", __FUNCTION__);
+  }
   taos_free_result(pRes);
+
+  for(int i = 0; i < 1; i++){
+    taosMemoryFree(sql3[i]);
+  }
+
   taos_close(taos);
 
   return code;
@@ -138,13 +178,26 @@ int smlProcess_json2_Test() {
   const char *sql[] = {
       "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\",\"type\":\"binary\"},\"id\":\"d1001\"}}"
   };
-  pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL,
+  char *sql1[1] = {0};
+  for(int i = 0; i < 1; i++){
+    sql1[i] = taosMemoryCalloc(1, 1024);
+    strncpy(sql1[i], sql[i], 1023);
+  }
+
+  pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL,
                                 TSDB_SML_TIMESTAMP_NANO_SECONDS);
-  printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
   int code = taos_errno(pRes);
+  if(code != 0){
+    printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+  }else{
+    printf("%s result:success\n", __FUNCTION__);
+  }
   taos_free_result(pRes);
   taos_close(taos);
 
+  for(int i = 0; i < 1; i++){
+    taosMemoryFree(sql1[i]);
+  }
   return code;
 }
 
@@ -158,15 +211,29 @@ int smlProcess_json3_Test() {
   taos_free_result(pRes);
 
   const char *sql[] = {
-      "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]"
+//      "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]"
+        "{\"metric\": \"dcxnmr\", \"timestamp\": {\"value\": 1626006833639000000, \"type\": \"ns\"}, \"value\": {\"value\": false, \"type\": \"bool\"}, \"tags\": {\"t0\": {\"value\": false, \"type\": \"bool\"}, \"t1\": {\"value\": 127, \"type\": \"tinyint\"}, \"t2\": {\"value\": 32767, \"type\": \"smallint\"}, \"t3\": {\"value\": 2147483647, \"type\": \"int\"}, \"t4\": {\"value\": 9223372036854775807, \"type\": \"bigint\"}, \"t5\": {\"value\": 11.12345027923584, \"type\": \"float\"}, \"t6\": {\"value\": 22.123456789, \"type\": \"double\"}, \"t7\": {\"value\": \"binaryTagValue\", \"type\": \"binary\"}, \"t8\": {\"value\": \"abc{aaa\", \"type\": \"nchar\"}}}"
   };
-  pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL,
+  char *sql1[1] = {0};
+  for(int i = 0; i < 1; i++){
+    sql1[i] = taosMemoryCalloc(1, 1024);
+    strncpy(sql1[i], sql[i], 1023);
+  }
+
+  pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL,
                                 TSDB_SML_TIMESTAMP_NANO_SECONDS);
-  printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
   int code = taos_errno(pRes);
+  if(code != 0){
+    printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
+  }else{
+    printf("%s result:success\n", __FUNCTION__);
+  }
   taos_free_result(pRes);
   taos_close(taos);
 
+  for(int i = 0; i < 1; i++){
+    taosMemoryFree(sql1[i]);
+  }
   return code;
 }
 
@@ -235,7 +302,7 @@ int sml_16384_Test() {
   if(code) return code;
 
   const char *sql1[] = {
-      "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000",
+      "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833631000000",
   };
   pRes = taos_schemaless_insert(taos, (char **)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0);
   printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes));
@@ -799,7 +866,7 @@ int sml_19221_Test() {
   taos_free_result(pRes);
 
   const char *sql[] = {
-      "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000",
+      "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833632000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833633000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833634000000",
   };
 
   pRes = taos_query(taos, "use sml_db");
@@ -938,9 +1005,9 @@ int main(int argc, char *argv[]) {
   ret = smlProcess_json1_Test();
   ASSERT(!ret);
   ret = smlProcess_json2_Test();
-  ASSERT(ret);
+  ASSERT(!ret);
   ret = smlProcess_json3_Test();
-  ASSERT(ret);
+  ASSERT(!ret);
   ret = sml_TD15662_Test();
   ASSERT(!ret);
   ret = sml_TD15742_Test();