refactor:add schemaless function

include/client/taos.h

@@ -199,7 +199,7 @@ DLL_EXPORT void         taos_close_stream(TAOS_STREAM *tstr);
 #endif
 
 DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
-DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
+DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision, bool dataFormat);
 
 /* --------------------------TMQ INTERFACE------------------------------- */
 

include/libs/parser/parser.h

@@ -93,12 +93,12 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf,
 int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields);
 int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields);
 int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
-void destroyBoundColumnInfo(void* pBoundInfo);
+void    destroyBoundColumnInfo(void* pBoundInfo);
 int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen);
 
-void* tscSmlInitHandle(SQuery *pQuery);
-void tscSmlDestroyHandle(void *pHandle);
-int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen);
+void*   smlInitHandle(SQuery *pQuery);
+void    smlDestroyHandle(void *pHandle);
+int32_t smlBindData(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
 
 #ifdef __cplusplus

source/client/inc/clientSml.h

-/*
- * Copyright (c) 2021 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_CLIENTSML_H
-#define TDENGINE_CLIENTSML_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "thash.h"
-#include "clientInt.h"
-#include "catalog.h"
-
-typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
-
-typedef struct {
-  const char* measure;
-  const char* tags;
-  const char* cols;
-  const char* timestamp;
-
-  int32_t measureLen;
-  int32_t measureTagsLen;
-  int32_t tagsLen;
-  int32_t colsLen;
-  int32_t timestampLen;
-} TAOS_PARSE_ELEMENTS;
-
-typedef struct {
-  const char     *sTableName;   // super table name
-  uint8_t        sTableNameLen;
-  char           childTableName[TSDB_TABLE_NAME_LEN];
-  uint64_t       uid;
-
-  SArray         *tags;
-  SArray         *cols;        // elements are SHashObj<key, SSmlKv*> for find by key quickly
-
-  SArray         colsColumn;   // elements are cols key string
-} TAOS_SML_DATA_POINT_TAGS;
-
-typedef struct SSmlSTableMeta {
-//  char     *sTableName;   // super table name
-//  uint8_t  sTableNameLen;
-  uint8_t    precision;     // the number of precision
-  SHashObj   *tagHash;
-  SHashObj   *fieldHash;
-} SSmlSTableMeta;
-
-typedef struct SMsgBuf {
-  int32_t   len;
-  char      *buf;
-} SMsgBuf;
-
-typedef struct {
-  uint64_t          id;
-
-  SMLProtocolType   protocol;
-  int32_t           tsType;
-
-  SHashObj          *childTables;
-  SHashObj          *superTables;
-
-  SHashObj          *metaHashObj;
-  SHashObj          *pVgHash;
-
-  void              *exec;
-
-  STscObj           *taos;
-  SCatalog          *pCatalog;
-  SRequestObj       *pRequest;
-  SQuery            *pQuery;
-
-  int32_t           affectedRows;
-  SMsgBuf           msgBuf;
-} SSmlLinesInfo;
-
-int smlInsert(TAOS* taos, SSmlLinesInfo* info);
-
-bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info);
-bool isValidInteger(char *str);
-bool isValidFloat(char *str);
-
-int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info);
-
-bool convertSmlValueType(SSmlKv *pVal, char *value,
-                         uint16_t len, SSmlLinesInfo* info, bool isTag);
-int32_t convertSmlTimeStamp(SSmlKv *pVal, char *value,
-                            uint16_t len, SSmlLinesInfo* info);
-
-
-int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol,
-                      SMLTimeStampType tsType);
-int sml_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol,
-                             SMLTimeStampType tsType, int* affectedRows);
-int sml_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol,
-                             SMLTimeStampType tsType, int* affectedRows);
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_CLIENTSML_H

source/libs/parser/src/parInsert.c

@@ -1514,6 +1514,7 @@ int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields
   return TSDB_CODE_SUCCESS;
 }
 
+// schemaless logic start
 
 typedef struct SmlExecHandle {
   SHashObj*    pBlockHash;
@@ -1523,7 +1524,7 @@ typedef struct SmlExecHandle {
   SVCreateTbReq createTblReq;   // each table
 
   SQuery* pQuery;
-} SmlExecHandle;
+} SSmlExecHandle;
 
 static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSchema* pSchema) {
   col_id_t nCols = pColList->numOfCols;
@@ -1620,14 +1621,15 @@ static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) {
+int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format,
+                    STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) {
   SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
   int32_t rowNum = taosArrayGetSize(cols);
   if(rowNum <= 0) {
     return buildInvalidOperationMsg(&pBuf, "cols size <= 0");
   }
 
-  SmlExecHandle *smlHandle = (SmlExecHandle *)handle;
+  SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle;
   SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
   setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta));
   int ret = smlBoundColumns(tags, &smlHandle->tags, pTagsSchema);
@@ -1651,7 +1653,21 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
 
   SSchema* pSchema = getTableColumnSchema(pTableMeta);
 
-  ret = smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema);
+
+  if(format){
+    ret = smlBoundColumns(taosArrayGetP(colsFormat, 0), &pDataBlock->boundColumnInfo, pSchema);
+  }else{
+    SArray *columns = taosArrayInit(16, POINTER_BYTES);
+    void **p1 = taosHashIterate(colsHash, NULL);
+    while (p1) {
+      SSmlKv* kv = *p1;
+      taosArrayPush(columns, &kv);
+      p1 = taosHashIterate(colsHash, p1);
+    }
+    ret = smlBoundColumns(columns, &pDataBlock->boundColumnInfo, pSchema);
+    taosArrayDestroy(columns);
+  }
+
   if(ret != TSDB_CODE_SUCCESS){
     buildInvalidOperationMsg(&pBuf, "bound cols error");
     return ret;
@@ -1671,7 +1687,12 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
   for (int32_t r = 0; r < rowNum; ++r) {
     STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size);  // skip the SSubmitBlk header
     tdSRowResetBuf(pBuilder, row);
-    SArray *rowData = taosArrayGetP(cols, r);
+    void *rowData = NULL;
+    if(format){
+      rowData = taosArrayGetP(colsFormat, r);
+    }else{
+      rowData = taosArrayGetP(cols, r);
+    }
 
     // 1. set the parsed value from sql string
     for (int c = 0; c < spd->numOfBound; ++c) {
@@ -1680,7 +1701,18 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
       param.schema = pColSchema;
       getSTSRowAppendInfo(pBuilder->rowType, spd, c, &param.toffset, &param.colIdx);
 
-      SSmlKv *kv = taosArrayGetP(rowData, c);
+      SSmlKv *kv = NULL;
+      if(format){
+        kv = taosArrayGetP(rowData, c);
+        if (!kv){
+          char msg[64] = {0};
+          sprintf(msg, "cols num not the same like before:%d", r);
+          return buildInvalidOperationMsg(&pBuf, msg);
+        }
+      }else{
+        void **p =taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
+        kv = *p;
+      }
 
       if (kv->valueLen == 0) {
         MemRowAppend(&pBuf, NULL, 0, &param);
@@ -1720,23 +1752,25 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta
   return TSDB_CODE_SUCCESS;
 }
 
-void* tscSmlInitHandle(SQuery *pQuery){
-  SmlExecHandle *handle = taosMemoryCalloc(sizeof(SmlExecHandle));
+void* smlInitHandle(SQuery *pQuery){
+  SSmlExecHandle *handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle));
+  if(!handle) return NULL;
   handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
   handle->pQuery = pQuery;
 
   return handle;
 }
 
-void tscSmlDestroyHandle(void *pHandle){
+void smlDestroyHandle(void *pHandle){
   if(!pHandle) return;
-  SmlExecHandle *handle = (SmlExecHandle *)pHandle;
-  taosHashCleanup(handle->pBlockHash);
+  SSmlExecHandle *handle = (SSmlExecHandle *)pHandle;
+  destroyBlockHashmap(handle->pBlockHash);
   taosMemoryFree(handle);
 }
 
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) {
-  SmlExecHandle *smlHandle = (SmlExecHandle *)handle;
+  SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle;
   return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash);
 }
+// schemaless logic end