Merge pull request #12735 from taosdata/feature/TD-14761

fix:invalid data type in schemaless

source/client/src/clientSml.c

@@ -65,12 +65,14 @@ for (int i = 1; i < keyLen; ++i) {      \
 #define OTD_TIMESTAMP_COLUMN_NAME "ts"
 #define OTD_METRIC_VALUE_COLUMN_NAME "value"
 
-#define TS        "_ts"
-#define TS_LEN    3
-#define TAG       "_tagNone"
-#define TAG_LEN   8
-#define VALUE     "value"
-#define VALUE_LEN 5
+#define TS              "_ts"
+#define TS_LEN          3
+#define TAG             "_tag"
+#define TAG_LEN         4
+#define TAG_VALUE       "NULL"
+#define TAG_VALUE_LEN   4
+#define VALUE           "value"
+#define VALUE_LEN       5
 
 #define BINARY_ADD_LEN 2        // "binary"   2 means " "
 #define NCHAR_ADD_LEN 3         // L"nchar"   3 means L" "
@@ -598,25 +600,33 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){
     kvVal->type = TSDB_DATA_TYPE_FLOAT;
     kvVal->f = (float)result;
   }else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)){
-    if(result >= (double)INT64_MAX){
-      kvVal->i = INT64_MAX;
-    }else if(result <= (double)INT64_MIN){
-      kvVal->i = INT64_MIN;
-    }else{
-      kvVal->i = result;
+    if(smlDoubleToInt64OverFlow(result)){
+      errno = 0;
+      int64_t tmp = taosStr2Int64(pVal, &endptr, 10);
+      if(errno == ERANGE){
+        smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal);
+        return false;
+      }
+      kvVal->type = TSDB_DATA_TYPE_BIGINT;
+      kvVal->i = tmp;
+      return true;
     }
     kvVal->type = TSDB_DATA_TYPE_BIGINT;
+    kvVal->i = (int64_t)result;
   }else if ((left == 3 && strncasecmp(endptr, "u64", left) == 0)){
-    if(result < 0){
-      smlBuildInvalidDataMsg(msg, "unsigned big int is too large, out of precision", pVal);
-      return false;
-    }
-    if(result >= (double)UINT64_MAX){
-      kvVal->u = UINT64_MAX;
-    }else{
-      kvVal->u = result;
+    if(result >= (double)UINT64_MAX || result < 0){
+      errno = 0;
+      uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);
+      if(errno == ERANGE || result < 0){
+        smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal);
+        return false;
+      }
+      kvVal->type = TSDB_DATA_TYPE_UBIGINT;
+      kvVal->u = tmp;
+      return true;
     }
     kvVal->type = TSDB_DATA_TYPE_UBIGINT;
+    kvVal->u = result;
   }else if (left == 3 && strncasecmp(endptr, "i32", left) == 0){
     if(!IS_VALID_INT(result)){
       smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal);
@@ -1103,8 +1113,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable
   kv->keyLen = VALUE_LEN;
   kv->value = value;
   kv->length = valueLen;
-  if(!smlParseValue(kv, &info->msgBuf) || kv->type == TSDB_DATA_TYPE_BINARY
-      || kv->type == TSDB_DATA_TYPE_NCHAR || kv->type == TSDB_DATA_TYPE_BOOL){
+  if(!smlParseValue(kv, &info->msgBuf)){
     return TSDB_CODE_SML_INVALID_DATA;
   }
 
@@ -1124,8 +1133,8 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, char *c
     if(!kv) return TSDB_CODE_OUT_OF_MEMORY;
     kv->key = TAG;
     kv->keyLen = TAG_LEN;
-    kv->value = TAG;
-    kv->length = TAG_LEN;
+    kv->value = TAG_VALUE;
+    kv->length = TAG_VALUE_LEN;
     kv->type = TSDB_DATA_TYPE_NCHAR;
     if(cols) taosArrayPush(cols, &kv);
     return TSDB_CODE_SUCCESS;
@@ -2264,6 +2273,7 @@ static int32_t smlParseLine(SSmlHandle *info, char* lines[], int numLines){
       uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, *lines);
       return code;
     }
+    return code;
   }
 
   for (int32_t i = 0; i < numLines; ++i) {

source/client/test/smlTest.cpp

@@ -208,6 +208,7 @@ TEST(testCase, smlParseCols_Error_Test) {
     memcpy(sql, data[i], len + 1);
     SArray *cols = taosArrayInit(8, POINTER_BYTES);
     int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf);
+    printf("i:%d\n",i);
     ASSERT_NE(ret, TSDB_CODE_SUCCESS);
     taosHashClear(dumplicateKey);
     taosMemoryFree(sql);
@@ -272,11 +273,11 @@ TEST(testCase, smlParseCols_tag_Test) {
 
   // nchar
   kv = (SSmlKv *)taosArrayGetP(cols, 0);
-  ASSERT_EQ(strncasecmp(kv->key, TAG, strlen(TAG)), 0);
-  ASSERT_EQ(kv->keyLen, strlen(TAG));
+  ASSERT_EQ(strncasecmp(kv->key, TAG, TAG_LEN), 0);
+  ASSERT_EQ(kv->keyLen, TAG_LEN);
   ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
-  ASSERT_EQ(kv->length, strlen(TAG));
-  ASSERT_EQ(strncasecmp(kv->value, TAG, strlen(TAG)), 0);
+  ASSERT_EQ(kv->length, TAG_LEN);
+  ASSERT_EQ(strncasecmp(kv->value, TAG_VALUE, TAG_VALUE_LEN), 0);
   taosMemoryFree(kv);
 
   taosArrayDestroy(cols);
@@ -506,7 +507,7 @@ TEST(testCase, smlProcess_influx_Test) {
     "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,heading=221,grade=0,fuel_consumption=25 1451608403000000000",
     "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451609404000000000",
     "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451619405000000000",
-    "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 145160640600000000",
+    "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606406000000000",
     "readings,name=truck_2,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606407000000000",
     "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609408000000000",
     "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 1451629409000000000",
@@ -745,7 +746,7 @@ TEST(testCase, smlProcess_json1_Test) {
      "        }\n"
      "    }\n"
      "]";
-  int ret = smlProcess(info, (char **)(&sql), -1);
+  int ret = smlProcess(info, (char **)(&sql), 1);
   ASSERT_EQ(ret, 0);
 
   // case 1
@@ -1220,4 +1221,56 @@ TEST(testCase, sml_TD15662_Test) {
   ASSERT_EQ(ts, 1626006833639000000);
 
   taos_free_result(res);
+}
+
+TEST(testCase, sml_TD15735_Test) {
+  TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  ASSERT_NE(taos, nullptr);
+
+  TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db");
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "use sml_db");
+  taos_free_result(pRes);
+
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT);
+  ASSERT_NE(request, nullptr);
+
+  SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
+  ASSERT_NE(info, nullptr);
+
+  const char *sql[1] = {
+      "{'metric': 'pekoiw', 'timestamp': {'value': 1626006833639000000, 'type': 'ns'}, 'value': {'value': False, 'type': 'bool'}, 'tags': {'t0': {'value': True, 'type': 'bool'}, 't1': {'value': 127, 'type': 'tinyint'}, 't2': {'value': 32767, 'type': 'smallint'}, 't3': {'value': 2147483647, 'type': 'int'}, 't4': {'value': 9223372036854775807, 'type': 'bigint'}, 't5': {'value': 11.12345027923584, 'type': 'float'}, 't6': {'value': 22.123456789, 'type': 'double'}, 't7': {'value': 'binaryTagValue', 'type': 'binary'}, 't8': {'value': 'ncharTagValue', 'type': 'nchar'}}}",
+  };
+  int32_t ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
+  ASSERT_NE(ret, 0);
+
+  destroyRequest(request);
+  smlDestroyInfo(info);
+}
+
+TEST(testCase, sml_TD15742_Test) {
+  TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  ASSERT_NE(taos, nullptr);
+
+  TAOS_RES* pRes = taos_query(taos, "create database if not exists TD15742");
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "use TD15742");
+  taos_free_result(pRes);
+
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT);
+  ASSERT_NE(request, nullptr);
+
+  SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
+  ASSERT_NE(info, nullptr);
+
+  const char *sql[] = {
+      "zgzbix 1626006833641 False id=zgzbix_992_38861 t0=t t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\"",
+  };
+  int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0]));
+  ASSERT_EQ(ret, 0);
+
+  destroyRequest(request);
+  smlDestroyInfo(info);
 }
\ No newline at end of file