From 4fa52dc6d63ae9f5f72b62f1e7a83a9307c0fc19 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 1 May 2022 22:08:18 +0800 Subject: [PATCH] refactor:fix schemaless error & add unit test cases --- cmake/cmake.define | 2 +- include/libs/parser/parser.h | 2 +- source/client/src/clientMain.c | 5 - source/client/src/clientSml.c | 37 +++-- source/client/test/CMakeLists.txt | 21 ++- source/client/test/smlTest.cpp | 134 ++++++++++++++++++ .../libs/scalar/test/scalar/scalarTests.cpp | 2 +- 7 files changed, 177 insertions(+), 26 deletions(-) create mode 100644 source/client/test/smlTest.cpp diff --git a/cmake/cmake.define b/cmake/cmake.define index c985ba1cc7..69fc3cd0eb 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -56,7 +56,7 @@ ELSE () MESSAGE(STATUS "Will compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fpermissive -fPIC -gdwarf-2 -g3") ENDIF () MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index f608e38226..d8a4eac4e5 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -98,7 +98,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void* smlInitHandle(SQuery *pQuery); void smlDestroyHandle(void *pHandle); -int32_t smlBindData(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 27efcee76e..54ac42743c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -758,8 +758,3 @@ int taos_stmt_close(TAOS_STMT *stmt) { return stmtClose(stmt); } -TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { - // TODO - return NULL; -} - diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index bde9aeed49..8bdf737180 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,4 +1,3 @@ -#include #include #include #include @@ -15,6 +14,7 @@ #include "ttypes.h" #include "tcommon.h" #include "catalog.h" +#include "clientInt.h" //================================================================================================= #define SPACE ' ' @@ -971,7 +971,7 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu if(!elements->cols) { smlBuildInvalidDataMsg(msg, "invalid columns", elements->cols); return TSDB_CODE_SML_INVALID_DATA; - } + } bool isInQuote = false; while (*sql != '\0') { @@ -983,12 +983,18 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu } sql++; } + if(isInQuote){ + smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); + return TSDB_CODE_SML_INVALID_DATA; + } elements->colsLen = sql - elements->cols; // parse ts,ts can be empty while (*sql != '\0') { - if(*sql != SPACE) { + if(*sql != SPACE && elements->timestamp == NULL) { elements->timestamp = sql; + } + if(*sql == SPACE && elements->timestamp != NULL){ break; } sql++; @@ -1321,7 +1327,7 @@ static SSmlTableInfo* smlBuildTableInfo(bool format){ goto cleanup; } - tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (tag->columnsHash == NULL) { uError("SML:smlParseLine failed to allocate memory"); goto cleanup; @@ -1365,7 +1371,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col if(dataFormat){ taosArrayPush(oneTable->colsFormat, &cols); }else{ - SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if(!kvHash){ uError("SML:smlDealCols failed to allocate memory"); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1381,6 +1387,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col } taosArrayPush(oneTable->cols, &kvHash); } + return TSDB_CODE_SUCCESS; } static SSmlSTableMeta* smlBuildSTableMeta(){ @@ -1388,13 +1395,13 @@ static SSmlSTableMeta* smlBuildSTableMeta(){ if(!meta){ return NULL; } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (meta->tagHash == NULL) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; } - meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (meta->fieldHash == NULL) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; @@ -1528,6 +1535,7 @@ static void smlDestroyInfo(SSmlHandle* info){ } static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ + int32_t code = TSDB_CODE_SUCCESS; SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle)); if (NULL == info) { return NULL; @@ -1550,7 +1558,7 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; info->taos = taos; - int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if(code != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); goto cleanup; @@ -1564,9 +1572,9 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->exec = smlInitHandle(info->pQuery); - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if(NULL == info->exec || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash){ @@ -1653,7 +1661,6 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines); cleanup: - smlDestroyInfo(info); return code; } @@ -1681,12 +1688,12 @@ cleanup: TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision, bool dataFormat) { SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); if(!request){ - goto end; + return NULL; } SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); if(!info){ - goto end; + return (TAOS_RES*)request; } switch (protocol) { @@ -1703,7 +1710,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr default: break; } + smlDestroyInfo(info); end: return (TAOS_RES*)request; } + diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 9df6b260f6..3fc91d07b6 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -17,14 +17,27 @@ TARGET_LINK_LIBRARIES( PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom ) +ADD_EXECUTABLE(smlTest smlTest.cpp) +TARGET_LINK_LIBRARIES( + smlTest + PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom +) + TARGET_INCLUDE_DIRECTORIES( clientTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/client/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc" + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) TARGET_INCLUDE_DIRECTORIES( tmqTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/client/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc" + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) + +TARGET_INCLUDE_DIRECTORIES( + smlTest + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp new file mode 100644 index 0000000000..823432d95d --- /dev/null +++ b/source/client/test/smlTest.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "../src/clientSml.c" +#include "taos.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, sml_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + SSmlLineInfo elements = {0}; + + //case 1 + char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; + int ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st,t1=3,t2=4,t3=t3")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); + ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 2 false + sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_NE(ret, 0); + + // case 3 false + sql = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); + ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + + + // case 4 tag is null + sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); + ASSERT_EQ(elements.tagsLen, 0); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 2 + elements.colsLen + 1); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 5 tag is null + sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + sql++; + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen); + ASSERT_EQ(elements.tagsLen, 0); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 3 + elements.colsLen + 2); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 6 + sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + + // case 7 + sql = " st , "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + sql++; + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); + ASSERT_EQ(elements.colsLen, strlen(",")); + + // case 8 false + sql = ", st , "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_NE(ret, 0); +} + + diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 206b2ac542..c03e3408d4 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -127,7 +127,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t ronwNum, void *value) { +void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; -- GitLab