Merge pull request #19295 from taosdata/refact/submit_req_marks

opti:json parse logic for schemaless

source/client/inc/clientSml.h

@@ -69,6 +69,7 @@ extern "C" {
 #define VALUE     "_value"
 #define VALUE_LEN 6
 
+#define OTD_JSON_FIELDS_NUM     4
 #define MAX_RETRY_TIMES 5
 typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
 
@@ -177,9 +178,10 @@ typedef struct {
   int32_t      lineNum;
   SSmlMsgBuf   msgBuf;
 
-//  cJSON       *root;  // for parse json
-  int8_t             offset[4];
+  cJSON       *root;  // for parse json
+  int8_t             offset[OTD_JSON_FIELDS_NUM];
   SSmlLineInfo      *lines; // element is SSmlLineInfo
+  bool               parseJsonByLib;
 
   //
   SArray      *preLineTagKV;
@@ -206,9 +208,9 @@ typedef int32_t (*_equal_fn_sml)(const void *, const void *);
 
 SSmlHandle   *smlBuildSmlInfo(TAOS *taos);
 void          smlDestroyInfo(SSmlHandle *info);
-void          smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
-void          smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
-SArray       *smlJsonParseTags(char *start, char *end);
+int           smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
+int           smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
+//SArray       *smlJsonParseTags(char *start, char *end);
 bool          smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
 void*         nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
 int           nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
@@ -226,6 +228,7 @@ int32_t           is_same_child_table_telnet(const void *a, const void *b);
 int64_t           smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len);
 int32_t           smlClearForRerun(SSmlHandle *info);
 int32_t           smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
+uint8_t           smlGetTimestampLen(int64_t num);
 
 int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);

source/client/src/clientSml.c

@@ -1008,12 +1008,16 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols
   return TSDB_CODE_SUCCESS;
 }
 
-static void smlDestroyTableInfo(SSmlTableInfo *tag) {
+static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
   for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) {
     SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i);
     taosHashCleanup(kvHash);
   }
 
+  if(info->parseJsonByLib){
+    SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
+    if(key != NULL) taosMemoryFree(key->tags);
+  }
   taosMemoryFree(tag->key);
   taosArrayDestroy(tag->cols);
   taosArrayDestroy(tag->tags);
@@ -1028,7 +1032,7 @@ void smlDestroyInfo(SSmlHandle *info) {
   NodeList *tmp = info->childTables;
   while (tmp) {
     if (tmp->data.used) {
-      smlDestroyTableInfo((SSmlTableInfo *)tmp->data.value);
+      smlDestroyTableInfo(info, (SSmlTableInfo *)tmp->data.value);
     }
     NodeList *t = tmp->next;
     taosMemoryFree(tmp);
@@ -1055,6 +1059,9 @@ void smlDestroyInfo(SSmlHandle *info) {
   if (!info->dataFormat) {
     for (int i = 0; i < info->lineNum; i++) {
       taosArrayDestroy(info->lines[i].colArray);
+      if(info->parseJsonByLib){
+        taosMemoryFree(info->lines[i].tags);
+      }
     }
     taosMemoryFree(info->lines);
   }
@@ -1251,7 +1258,7 @@ int32_t smlClearForRerun(SSmlHandle *info) {
   NodeList *pList = info->childTables;
   while (pList) {
     if (pList->data.used) {
-      smlDestroyTableInfo((SSmlTableInfo *)pList->data.value);
+      smlDestroyTableInfo(info, (SSmlTableInfo *)pList->data.value);
       pList->data.used = false;
     }
     pList = pList->next;
@@ -1267,11 +1274,13 @@ int32_t smlClearForRerun(SSmlHandle *info) {
     pList = pList->next;
   }
 
-  if (unlikely(info->lines != NULL)) {
-    uError("SML:0x%" PRIx64 " info->lines != NULL", info->id);
-    return TSDB_CODE_SML_INVALID_DATA;
+  if (!info->dataFormat){
+    if (unlikely(info->lines != NULL)) {
+      uError("SML:0x%" PRIx64 " info->lines != NULL", info->id);
+      return TSDB_CODE_SML_INVALID_DATA;
+    }
+    info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo));
   }
-  info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo));
 
   memset(&info->preLine, 0, sizeof(SSmlLineInfo));
   info->currSTableMeta = NULL;

source/client/test/smlTest.cpp

@@ -411,28 +411,28 @@ TEST(testCase, smlParseCols_Test) {
   smlDestroyInfo(info);
 }
 
-//TEST(testCase, smlGetTimestampLen_Test) {
-//  uint8_t len = smlGetTimestampLen(0);
-//  ASSERT_EQ(len, 1);
-//
-//  len = smlGetTimestampLen(1);
-//  ASSERT_EQ(len, 1);
-//
-//  len = smlGetTimestampLen(10);
-//  ASSERT_EQ(len, 2);
-//
-//  len = smlGetTimestampLen(390);
-//  ASSERT_EQ(len, 3);
-//
-//  len = smlGetTimestampLen(-1);
-//  ASSERT_EQ(len, 1);
-//
-//  len = smlGetTimestampLen(-10);
-//  ASSERT_EQ(len, 2);
-//
-//  len = smlGetTimestampLen(-390);
-//  ASSERT_EQ(len, 3);
-//}
+TEST(testCase, smlGetTimestampLen_Test) {
+  uint8_t len = smlGetTimestampLen(0);
+  ASSERT_EQ(len, 1);
+
+  len = smlGetTimestampLen(1);
+  ASSERT_EQ(len, 1);
+
+  len = smlGetTimestampLen(10);
+  ASSERT_EQ(len, 2);
+
+  len = smlGetTimestampLen(390);
+  ASSERT_EQ(len, 3);
+
+  len = smlGetTimestampLen(-1);
+  ASSERT_EQ(len, 1);
+
+  len = smlGetTimestampLen(-10);
+  ASSERT_EQ(len, 2);
+
+  len = smlGetTimestampLen(-390);
+  ASSERT_EQ(len, 3);
+}
 
 TEST(testCase, smlParseNumber_Test) {
   SSmlKv     kv = {0};

tests/system-test/1-insert/opentsdb_json_taosc_insert.py

@@ -1719,7 +1719,6 @@ class TDTestCase:
             print(err.errno)
 
     def runAll(self):
-        """
         for value_type in ["obj", "default"]:
             self.initCheckCase(value_type)
             self.symbolsCheckCase(value_type)
@@ -1759,7 +1758,7 @@ class TDTestCase:
         self.batchErrorInsertCheckCase()
         self.chineseCheckCase()
         # self.spellCheckCase()
-        self.tbnameTagsColsNameCheckCase()
+        # self.tbnameTagsColsNameCheckCase()
         # # MultiThreads
         # self.sStbStbDdataInsertMultiThreadCheckCase()
         # self.sStbStbDdataAtInsertMultiThreadCheckCase()
@@ -1772,7 +1771,7 @@ class TDTestCase:
         # self.sStbStbDdataDtsMtInsertMultiThreadCheckCase()
         # self.sStbDtbDdataDtsMtInsertMultiThreadCheckCase()
         # self.lengthIcreaseCrashCheckCase()
-        """
+
     def run(self):
         print("running {}".format(__file__))
         self.createDb()

tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py

@@ -243,7 +243,7 @@ class TDTestCase:
             if t_add_tag is not None:
                 sql_seq = f'{stb_name} {ts} {value} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8} t9={t8}'
         if id_change_tag is not None:
-            sql_seq = f'{stb_name} {ts} {value} t0={t0} {id}={tb_name} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}'
+            sql_seq = f'{stb_name} {ts} {value} {id}={tb_name} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}'
         if id_double_tag is not None:
             sql_seq = f'{stb_name} {ts} {value} {id}=\"{tb_name}_1\" t0={t0} t1={t1} {id}=\"{tb_name}_2\" t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}'
         if t_add_tag is not None:
@@ -1126,9 +1126,9 @@ class TDTestCase:
             self._conn.schemaless_insert([input_sql], TDSmlProtocolType.TELNET.value, None)
             query_sql = 'select * from `rFa$sta`'
             query_res = tdSql.query(query_sql, True)
-            tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, '2147483647i32', 'L"ncharTagValue"', '32767i16', '9223372036854775807i64', '22.123456789f64', '"ddzhiksj"', '11.12345f32', 'true', '127Ii8')])
+            tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, 'true', '127Ii8', '32767i16', '2147483647i32', '9223372036854775807i64', '11.12345f32', '22.123456789f64', '"ddzhiksj"', 'L"ncharTagValue"')])
             col_tag_res = tdSql.getColNameList(query_sql)
-            tdSql.checkEqual(col_tag_res, ['_ts', '_value', '"t$3"', 't!@#$%^&*()_+[];:<>?,9', 't#2', 't%4', 't&6', 't*7', 't^5', 'Tt!0', 'tT@1'])
+            tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't#2', '"t$3"', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9'])
             tdSql.execute('drop table `rFa$sta`')
 
     def tcpKeywordsCheckCase(self, protocol="telnet-tcp"):
@@ -1207,7 +1207,6 @@ class TDTestCase:
         tdLog.info(f'{sys._getframe().f_code.co_name}() function is running')
         tdCom.cleanTb(dbname="test")
         input_sql = self.genSqlList()[0]
-        print(input_sql)
         self.multiThreadRun(self.genMultiThreadSeq(input_sql))
         tdSql.query(f"show tables;")
         tdSql.checkRows(5)
@@ -1416,8 +1415,8 @@ class TDTestCase:
         self.symbolsCheckCase()
         self.tsCheckCase()
         self.openTstbTelnetTsCheckCase()
-        #self.idSeqCheckCase()
-        #self.idLetterCheckCase()
+        # self.idSeqCheckCase()
+        self.idLetterCheckCase()
         self.noIdCheckCase()
         self.maxColTagCheckCase()
         self.stbTbNameCheckCase()
@@ -1450,7 +1449,7 @@ class TDTestCase:
         self.spellCheckCase()
         self.pointTransCheckCase()
         self.defaultTypeCheckCase()
-        #self.tbnameTagsColsNameCheckCase()
+        self.tbnameTagsColsNameCheckCase()
         # # # MultiThreads
         # self.stbInsertMultiThreadCheckCase()
         # self.sStbStbDdataInsertMultiThreadCheckCase()