From b1561c95c33783be8478eb5b0fe1a8ddb0d5fe4e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Apr 2023 15:31:18 +0800 Subject: [PATCH] fix:add test cases for escape in schemaless --- source/client/src/clientSml.c | 2 + source/client/src/clientSmlLine.c | 2 +- source/client/src/clientTmq.c | 3 - source/client/test/smlTest.cpp | 36 ++++++++-- source/common/src/tglobal.c | 2 +- utils/test/c/sml_test.c | 107 ++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 12 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 855e48b88c..c972575e59 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1175,6 +1175,7 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + terrno = 0; taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; @@ -1250,6 +1251,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + terrno = 0; smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); if (terrno == TSDB_CODE_DUP_KEY) { return terrno; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index f4f4a5b630..da1bba19dc 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -465,7 +465,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin value = tmp; } - SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped}; int32_t ret = smlParseValue(&kv, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 6642e716e6..16a4f55840 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); - if (topic[0] != '`') { - strtolower(topic, src); - } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 76911e229a..1578b8b607 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -50,8 +50,9 @@ TEST(testCase, smlParseInfluxString_Test) { int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); - ASSERT_EQ(elements.measureLen, strlen(",st")); - ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3")); + ASSERT_EQ(elements.measureLen, strlen("\\,st")); + ASSERT_EQ(elements.measureEscaped, true); + ASSERT_EQ(elements.measureTagsLen, strlen("\\,st,t1=3,t2=4,t3=t3")); ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); @@ -204,7 +205,26 @@ TEST(testCase, smlParseCols_Error_Test) { "st,t=1 c=-3.402823466e+39u64 1626006833639000000", "st,t=1 c=-339u64 1626006833639000000", "st,t=1 c=18446744073709551616u64 1626006833639000000", - "st,t=1 c=1=2 1626006833639000000"}; + "st,t=1 c=1=2 1626006833639000000,", + // escape error test + // measure comma,space + "s,t,t=1 c=1 1626006833639000000,", + "s t,t=1 c=1 1626006833639000000,", + //tag key comma,equal,space + "st,t,t=1 c=2 1626006833639000000,", + "st,t=t=1 c=2 1626006833639000000,", + "st,t t=1 c=2 1626006833639000000,", + //tag value comma,equal,space + "st,tt=a,a c=2 1626006833639000000,", + "st,t=t=a a c=2 1626006833639000000,", + "st,t t=a=a c=2 1626006833639000000,", + //field key comma,equal,space + "st,tt=aa c,1=2 1626006833639000000,", + "st,tt=aa c=1=2 1626006833639000000,", + "st,tt=aa c 1=2 1626006833639000000,", + //field value double quote,slash + "st,tt=aa c=\"a\"a\" 1626006833639000000,", + }; SSmlHandle *info = smlBuildSmlInfo(NULL); info->protocol = TSDB_SML_LINE_PROTOCOL; @@ -256,16 +276,18 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); - ASSERT_EQ(kv->length, 17); - ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); + ASSERT_EQ(kv->length, 18); + ASSERT_EQ(kv->keyEscaped, true); + ASSERT_EQ(kv->valueEscaped, false); + ASSERT_EQ(strncasecmp(kv->value, "pass\\,it ", 9), 0); // nchar kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 8); - ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); + ASSERT_EQ(kv->length, 9); + ASSERT_EQ(strncasecmp(kv->value, "ii\\=sd", 5), 0); // bool kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index da4a912238..aa35b298e6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -198,7 +198,7 @@ int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; int32_t tsStreamCheckpointTickInterval = 1; int32_t tsTtlUnit = 86400; -int32_t tsTtlPushInterval = 86400; +int32_t tsTtlPushInterval = 3600; int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 755ab55625..515da64081 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -895,6 +895,61 @@ int smlProcess_18784_Test() { return code; } +int sml_escape_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use db_escape"); + taos_free_result(pRes); + + const char *sql[] = { + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_used=176059i,total=1076048383523889174i 1661943960000000000", + "d\\,i=\\ s\\k\",dev\"i\\,\\=\\ ce=s\"i\\,\\=\\ dc inode\"i\\,\\=\\ s_free=\"\\\"id,= ei\\\\\\f\" 1661943960000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 1); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from `d,i= s\\k\"`"); //check stable name + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 5); + printf("fieldNum:%d\n", fieldNum); + + int numFields = taos_num_fields(pRes); + TAOS_FIELD *fields = taos_fetch_fields(pRes); + ASSERT(numFields == 5); + ASSERT(strcmp(fields[4].name, "dev\"i,= ce") == 0); + + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + int64_t used = *(int64_t *)row[1]; + int64_t total = *(int64_t *)row[2]; + int64_t freed = *(int64_t *)row[3]; + + if (rowIndex == 0) { + ASSERT(ts == 1661943960000); + ASSERT(used == 176059); + ASSERT(total == 1076048383523889174); + ASSERT(freed == 66932805); + ASSERT(strcmp(row[4], "s\"i,= dc") == 0); + + } + rowIndex++; + } + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_19221_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -961,6 +1016,55 @@ int sml_ts2164_Test() { return code; } + +int sml_ts3116_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = + taos_query(taos, "DROP DATABASE IF EXISTS ts3116"); + taos_free_result(pRes); + + pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts3116 BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + taos_free_result(pRes); + + char *sql = { + "meters,location=la,groupid=ca current=11.8,voltage=221", + }; + + pRes = taos_query(taos, "use ts3116"); + taos_free_result(pRes); + int32_t totalRows = 0; + char *tmp = (char *)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql, strlen(sql)); + totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taosMemoryFree(tmp); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + + char *sql1 = { + "meters,location=la,groupid=ca\\=3 current=11.8,voltage=221\nmeters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + }; + + pRes = taos_query(taos, "use ts3116"); + taos_free_result(pRes); + + tmp = (char *)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql1, strlen(sql1)); + totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(tmp), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + taosMemoryFree(tmp); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_td22898_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1195,6 +1299,9 @@ int main(int argc, char *argv[]) { } int ret = 0; + ret = sml_escape_Test(); + ret = sml_ts3116_Test(); + ASSERT(!ret); ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file ASSERT(!ret); // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ -- GitLab