From 119e7952ded8ee60f0ebee9d0913dfb4d37c0b11 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 5 May 2022 12:08:34 +0800 Subject: [PATCH] refactor:add schemaless test cases --- source/client/src/clientSml.c | 40 +++-- source/client/test/smlTest.cpp | 313 ++++++++++++++++++++++++++++++++- 2 files changed, 329 insertions(+), 24 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8bdf737180..de034957dc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -141,9 +141,9 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const static int smlCompareKv(const void* p1, const void* p2) { SSmlKv* kv1 = (SSmlKv *)p1; SSmlKv* kv2 = (SSmlKv*)p2; - int kvLen1 = (int)strlen(kv1->key); - int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + int32_t kvLen1 = kv1->keyLen; + int32_t kvLen2 = kv2->keyLen; + int32_t res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); if (res != 0) { return res; } else { @@ -154,7 +154,7 @@ static int smlCompareKv(const void* p1, const void* p2) { static void smlBuildChildTableName(SSmlTableInfo *tags) { int32_t size = taosArrayGetSize(tags->tags); ASSERT(size > 0); - qsort(tags->tags, size, POINTER_BYTES, smlCompareKv); + taosArraySort(tags->tags, smlCompareKv); SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); @@ -501,7 +501,7 @@ static bool smlParseTinyInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 2; - if (!strcasecmp(signalPos, "i8")) { + if (!strncasecmp(signalPos, "i8", 2)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -529,7 +529,7 @@ static bool smlParseTinyUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 2; - if (!strcasecmp(signalPos, "u8")) { + if (!strncasecmp(signalPos, "u8", 2)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -554,7 +554,7 @@ static bool smlParseSmallInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (!strcasecmp(signalPos, "i16")) { + if (!strncasecmp(signalPos, "i16", 3)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -582,7 +582,7 @@ static bool smlParseSmallUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u16") == 0) { + if (strncasecmp(signalPos, "u16", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -607,7 +607,7 @@ static bool smlParseInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "i32") == 0) { + if (strncasecmp(signalPos, "i32", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -635,7 +635,7 @@ static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u32") == 0) { + if (strncasecmp(signalPos, "u32", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -656,13 +656,15 @@ static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; - if (len > 3 && strcasecmp(pVal + len - 3, "i64") == 0) { + if (len > 3 && strncasecmp(pVal + len - 3, "i64", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 3){ // 78ri8 *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid big int", endptr); }else if(!IS_VALID_BIGINT(result)){ *isValid = false; + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -696,7 +698,7 @@ static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u64") == 0) { + if (strncasecmp(signalPos, "u64", 3) == 0) { char *endptr = NULL; uint64_t result = strtoull(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -725,7 +727,7 @@ static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return true; } - if (len > 3 && len 3 && len i = true; return true; } - if((len == 5) && !strcasecmp(pVal, "false")) { + if((len == 5) && !strncasecmp(pVal, "false", len)) { kvVal->i = false; return true; } @@ -1085,12 +1087,14 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { case TSDB_TIME_PRECISION_SECONDS: ts *= (1e9); break; - case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_MILLI: ts *= (1e6); break; - case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: ts *= (1e3); break; + case TSDB_TIME_PRECISION_NANO: + break; default: ASSERT(0); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 823432d95d..9337263173 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -28,21 +28,21 @@ #include "../src/clientSml.c" #include "taos.h" -int main(int argc, char** argv) { +int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } -TEST(testCase, sml_Test) { - char msg[256] = {0}; +TEST(testCase, smlParseString_Test) { + char msg[256] = {0}; SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; SSmlLineInfo elements = {0}; - //case 1 + // case 1 char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; - int ret = smlParseString(sql, &elements, &msgBuf); + int ret = smlParseString(sql, &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -71,7 +71,6 @@ TEST(testCase, sml_Test) { ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); - // case 4 tag is null sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memset(&elements, 0, sizeof(SSmlLineInfo)); @@ -131,4 +130,306 @@ TEST(testCase, sml_Test) { ASSERT_NE(ret, 0); } +TEST(testCase, smlParseCols_Error_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + + const char *data[] = { + "c=\"89sd", // binary, nchar + "c=j\"89sd\"", + "c=\"89sd\"k", + "c=u", // bool + "c=truet", + "c=f64", // double + "c=8f64f", + "c=8ef64", + "c=1.7976931348623158e+390f64", + "c=f32", // float + "c=8f32f", + "c=8wef32", + "c=-3.402823466e+39f32", + "c=", // float + "c=8f", + "c=8we", + "c=3.402823466e+39", + "c=i8", // tiny int + "c=-8i8f", + "c=8wei8", + "c=-999i8", + "c=u8", // u tiny int + "c=8fu8", + "c=8weu8", + "c=999u8", + "c=-8u8", + "c=i16", // small int + "c=8fi16u", + "c=8wei16", + "c=-67787i16", + "c=u16", // u small int + "c=8u16f", + "c=8weu16", + "c=-9u16", + "c=67787u16", + "c=i32", // int + "c=8i32f", + "c=8wei32", + "c=2147483649i32", + "c=u32", // u int + "c=8u32f", + "c=8weu32", + "c=-4u32", + "c=42949672958u32", + "c=i64", // big int + "c=8i64i", + "c=8wei64", + "c=-9223372036854775809i64", + "c=i", // big int + "c=8fi", + "c=8wei", + "c=9223372036854775808i", + "c=u64", // u big int + "c=8u64f", + "c=8weu64", + "c=-3.402823466e+39u64", + "c=-339u64", + "c=18446744073709551616u64", + }; + + for(int i = 0; i < sizeof(data)/sizeof(data[0]); i++){ + int32_t len = strlen(data[i]); + int32_t ret = smlParseCols(data[i], len, NULL, false, &msgBuf); + ASSERT_NE(ret, TSDB_CODE_SUCCESS); + } +} + +TEST(testCase, smlParseCols_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + ASSERT_NE(cols, NULL); + + const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + int32_t len = strlen(data); + int32_t ret = smlParseCols(data, len, cols, false, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + int32_t size = taosArrayGetSize(cols); + ASSERT_EQ(size, 19); + + // binary + SSmlKv *kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); + ASSERT_EQ(kv->keyLen, 4; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY; + ASSERT_EQ(kv->length, 16); + ASSERT_EQ(strncasecmp(kv->value, "passit", 6), 0); + taosMemoryFree(kv); + + // nchar + kv = taosArrayGetP(cols, 1); + ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->length, 7); + ASSERT_EQ(strncasecmp(kv->value, "iisd", 4), 0); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 2); + ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1; + ASSERT_EQ(kv->i, false); + taosMemoryFree(kv); + + // double + kv = taosArrayGetP(cols, 3); + ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(kv->length, 8; + ASSERT_EQ(kv->d, 4); + taosMemoryFree(kv); + + // float + kv = taosArrayGetP(cols, 4); + ASSERT_EQ(strncasecmp(kv->key, "cf32_", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->f, 8.32); + taosMemoryFree(kv); + + // float + kv = taosArrayGetP(cols, 5); + ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->f, 8.23); + taosMemoryFree(kv); + + // tiny int + kv = taosArrayGetP(cols, 6); + ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); + ASSERT_EQ(kv->keyLen, 3); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT; + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(i, -34); + taosMemoryFree(kv); + + // unsigned tiny int + kv = taosArrayGetP(cols, 7); + ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); + ASSERT_EQ(kv->keyLen, 3); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->u, 89); + taosMemoryFree(kv); + + // small int + kv = taosArrayGetP(cols, 8); + ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); + ASSERT_EQ(kv->length, 2); + ASSERT_EQ(kv->u, 233); + taosMemoryFree(kv); + + // unsigned smallint + kv = taosArrayGetP(cols, 9); + ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); + ASSERT_EQ(kv->length, 2); + ASSERT_EQ(kv->u, 898); + taosMemoryFree(kv); + + // int + kv = taosArrayGetP(cols, 10); + ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->u, 98289); + taosMemoryFree(kv); + + // unsigned int + kv = taosArrayGetP(cols, 11); + ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->u, 12323); + taosMemoryFree(kv); + + + // bigint + kv = taosArrayGetP(cols, 12); + ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); + ASSERT_EQ(kv->keyLen, 4; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT; + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, -89238); + taosMemoryFree(kv); + + // bigint + kv = taosArrayGetP(cols, 13); + ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); + ASSERT_EQ(kv->keyLen, 2); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, 989); + taosMemoryFree(kv); + + // unsigned bigint + kv = taosArrayGetP(cols, 14); + ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->u, 8989323); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 15); + ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); + ASSERT_EQ(kv->keyLen, 9); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, true); + taosMemoryFree(kv); + + + // bool + kv = taosArrayGetP(cols, 16); + ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); + ASSERT_EQ(kv->keyLen, 6; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL; + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, -89238); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 17); + ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); + ASSERT_EQ(kv->keyLen, 6); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, 989); + taosMemoryFree(kv); + + // unsigned bigint + kv = taosArrayGetP(cols, 18); + ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); + taosMemoryFree(kv); + + taosArrayDestroy(cols); +} + +TEST(testCase, smlParseLine_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, NULL); + + SRequestObj *request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, NULL); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, NULL); + + const char *sql[3] = { + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451606400000000000", + "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000", + "readings,name=truck_2,fleet=North,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000" + }; + for (int i = 0; i < 3; i++) { + smlParseLine(info, sql[i]); + } +} +// TEST(testCase, smlParseTS_Test) { +// char msg[256] = {0}; +// SSmlMsgBuf msgBuf; +// msgBuf.buf = msg; +// msgBuf.len = 256; +// SSmlLineInfo elements = {0}; +// +// SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); +// if(!info){ +// return (TAOS_RES*)request; +// } +// ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); +// if(ret != TSDB_CODE_SUCCESS){ +// uError("SML:0x%"PRIx64" smlParseTS failed", info->id); +// return ret; +// } +// } -- GitLab