refactor:fix schemaless error & add unit test cases

cmake/cmake.define

@@ -56,7 +56,7 @@ ELSE ()
 	MESSAGE(STATUS "Will compile with Address Sanitizer!")
     ELSE ()
         SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fpermissive -fPIC -gdwarf-2 -g3")
     ENDIF ()
 
     MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")

include/libs/parser/parser.h

@@ -98,7 +98,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
 
 void*   smlInitHandle(SQuery *pQuery);
 void    smlDestroyHandle(void *pHandle);
-int32_t smlBindData(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen);
+int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
 
 #ifdef __cplusplus

source/client/src/clientMain.c

@@ -758,8 +758,3 @@ int taos_stmt_close(TAOS_STMT *stmt) {
   return stmtClose(stmt);
 }
 
-TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
-  // TODO
-  return NULL;
-}
-

source/client/src/clientSml.c

-#include <clientInt.h>
 #include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -15,6 +14,7 @@
 #include "ttypes.h"
 #include "tcommon.h"
 #include "catalog.h"
+#include "clientInt.h"
 //=================================================================================================
 
 #define SPACE ' '
@@ -971,7 +971,7 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu
   if(!elements->cols) {
     smlBuildInvalidDataMsg(msg, "invalid columns", elements->cols);
     return TSDB_CODE_SML_INVALID_DATA;
-    }
+  }
 
   bool isInQuote = false;
   while (*sql != '\0') {
@@ -983,12 +983,18 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu
     }
     sql++;
   }
+  if(isInQuote){
+    smlBuildInvalidDataMsg(msg, "only one quote", elements->cols);
+    return TSDB_CODE_SML_INVALID_DATA;
+  }
   elements->colsLen = sql - elements->cols;
 
   // parse ts,ts can be empty
   while (*sql != '\0') {
-    if(*sql != SPACE) {
+    if(*sql != SPACE && elements->timestamp == NULL) {
       elements->timestamp = sql;
+    }
+    if(*sql == SPACE && elements->timestamp != NULL){
       break;
     }
     sql++;
@@ -1321,7 +1327,7 @@ static SSmlTableInfo* smlBuildTableInfo(bool format){
       goto cleanup;
     }
 
-    tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
+    tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
     if (tag->columnsHash == NULL) {
       uError("SML:smlParseLine failed to allocate memory");
       goto cleanup;
@@ -1365,7 +1371,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col
   if(dataFormat){
     taosArrayPush(oneTable->colsFormat, &cols);
   }else{
-    SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
+    SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
     if(!kvHash){
       uError("SML:smlDealCols failed to allocate memory");
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -1381,6 +1387,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col
     }
     taosArrayPush(oneTable->cols, &kvHash);
   }
+  return TSDB_CODE_SUCCESS;
 }
 
 static SSmlSTableMeta* smlBuildSTableMeta(){
@@ -1388,13 +1395,13 @@ static SSmlSTableMeta* smlBuildSTableMeta(){
   if(!meta){
     return NULL;
   }
-  meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
+  meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
   if (meta->tagHash == NULL) {
     uError("SML:smlBuildSTableMeta failed to allocate memory");
     goto cleanup;
   }
 
-  meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
+  meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
   if (meta->fieldHash == NULL) {
     uError("SML:smlBuildSTableMeta failed to allocate memory");
     goto cleanup;
@@ -1528,6 +1535,7 @@ static void smlDestroyInfo(SSmlHandle* info){
 }
 
 static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){
+  int32_t code = TSDB_CODE_SUCCESS;
   SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle));
   if (NULL == info) {
     return NULL;
@@ -1550,7 +1558,7 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol
   ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV;
 
   info->taos        = taos;
-  int32_t code      = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
+  code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
   if(code != TSDB_CODE_SUCCESS){
     uError("SML:0x%"PRIx64" get catalog error %d", info->id, code);
     goto cleanup;
@@ -1564,9 +1572,9 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol
   info->msgBuf.len  = ERROR_MSG_BUF_DEFAULT_SIZE;
 
   info->exec        = smlInitHandle(info->pQuery);
-  info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
-  info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
-  info->pVgHash     = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+  info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  info->pVgHash     = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
 
   if(NULL == info->exec || NULL == info->childTables
       || NULL == info->superTables || NULL == info->pVgHash){
@@ -1653,7 +1661,6 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) {
   uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines);
 
 cleanup:
-  smlDestroyInfo(info);
   return code;
 }
 
@@ -1681,12 +1688,12 @@ cleanup:
 TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision, bool dataFormat) {
   SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT);
   if(!request){
-    goto end;
+    return NULL;
   }
 
   SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat);
   if(!info){
-    goto end;
+    return (TAOS_RES*)request;
   }
 
   switch (protocol) {
@@ -1703,7 +1710,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
     default:
       break;
   }
+  smlDestroyInfo(info);
 
 end:
   return (TAOS_RES*)request;
 }
+

source/client/test/CMakeLists.txt

@@ -17,14 +17,27 @@ TARGET_LINK_LIBRARIES(
         PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom
 )
 
+ADD_EXECUTABLE(smlTest smlTest.cpp)
+TARGET_LINK_LIBRARIES(
+        smlTest
+        PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom
+)
+
 TARGET_INCLUDE_DIRECTORIES(
         clientTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/client/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc"
+        PUBLIC "${TD_SOURCE_DIR}/include/client/"
+        PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
 )
 
 TARGET_INCLUDE_DIRECTORIES(
         tmqTest
-        PUBLIC "${TD_SOURCE_DIR}/include/libs/client/"
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc"
+        PUBLIC "${TD_SOURCE_DIR}/include/client/"
+        PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
 )
+
+TARGET_INCLUDE_DIRECTORIES(
+        smlTest
+        PUBLIC "${TD_SOURCE_DIR}/include/client/"
+        PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
+)
+

source/client/test/smlTest.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+#include <taoserror.h>
+#include <tglobal.h>
+#include <iostream>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#include "../src/clientSml.c"
+#include "taos.h"
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+TEST(testCase, sml_Test) {
+  char msg[256] = {0};
+  SSmlMsgBuf msgBuf;
+  msgBuf.buf = msg;
+  msgBuf.len = 256;
+  SSmlLineInfo elements = {0};
+
+  //case 1
+  char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000    ,32,c=3";
+  int ret = smlParseString(sql, &elements, &msgBuf);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(elements.measure, sql);
+  ASSERT_EQ(elements.measureLen, strlen("st"));
+  ASSERT_EQ(elements.measureTagsLen, strlen("st,t1=3,t2=4,t3=t3"));
+
+  ASSERT_EQ(elements.tags, sql + elements.measureLen + 1);
+  ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3"));
+
+  ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1);
+  ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64"));
+
+  ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
+  ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
+
+  // case 2  false
+  sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
+  memset(&elements, 0, sizeof(SSmlLineInfo));
+  ret = smlParseString(sql, &elements, &msgBuf);
+  ASSERT_NE(ret, 0);
+
+  // case 3  false
+  sql = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
+  memset(&elements, 0, sizeof(SSmlLineInfo));
+  ret = smlParseString(sql, &elements, &msgBuf);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2);
+  ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3"));
+
+
+  // case 4  tag is null
+  sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000";
+  memset(&elements, 0, sizeof(SSmlLineInfo));
+  ret = smlParseString(sql, &elements, &msgBuf);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(elements.measure, sql);
+  ASSERT_EQ(elements.measureLen, strlen("st"));
+  ASSERT_EQ(elements.measureTagsLen, strlen("st"));
+
+  ASSERT_EQ(elements.tags, sql + elements.measureLen + 1);
+  ASSERT_EQ(elements.tagsLen, 0);
+
+  ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2);
+  ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64"));
+
+  ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 2 + elements.colsLen + 1);
+  ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
+
+  // case 5 tag is null
+  sql = " st   c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64  1626006833639000000 ";
+  memset(&elements, 0, sizeof(SSmlLineInfo));
+  ret = smlParseString(sql, &elements, &msgBuf);
+  sql++;
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(elements.measure, sql);
+  ASSERT_EQ(elements.measureLen, strlen("st"));
+  ASSERT_EQ(elements.measureTagsLen, strlen("st"));
+
+  ASSERT_EQ(elements.tags, sql + elements.measureLen);
+  ASSERT_EQ(elements.tagsLen, 0);
+
+  ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3);
+  ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64"));
+
+  ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 3 + elements.colsLen + 2);
+  ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
+
+  // case 6
+  sql = " st   c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64   ";
+  memset(&elements, 0, sizeof(SSmlLineInfo));
+  ret = smlParseString(sql, &elements, &msgBuf);
+  ASSERT_EQ(ret, 0);
+
+  // case 7
+  sql = " st   ,   ";
+  memset(&elements, 0, sizeof(SSmlLineInfo));
+  ret = smlParseString(sql, &elements, &msgBuf);
+  sql++;
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3);
+  ASSERT_EQ(elements.colsLen, strlen(","));
+
+  // case 8 false
+  sql = ", st   ,   ";
+  memset(&elements, 0, sizeof(SSmlLineInfo));
+  ret = smlParseString(sql, &elements, &msgBuf);
+  ASSERT_NE(ret, 0);
+}
+
+

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -127,7 +127,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
   *pNode = (SNode *)vnode;
 }
 
-void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t ronwNum, void *value) {
+void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
   SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
   SColumnNode *rnode = (SColumnNode *)node;
   rnode->node.resType.type = dataType;