Merge pull request #9042 from taosdata/feature/3.0_wxy

Feature/3.0 the main process code of the client insert

.gitignore

@@ -12,6 +12,7 @@ debug/
 release/
 target/
 debs/
+deps/
 rpms/
 mac/
 *.pyc

include/common/tdataformat.h

@@ -544,7 +544,7 @@ void   tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
 void   tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
-static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value) {
+static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
     SColIdx* pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);

include/common/ttime.h

@@ -44,10 +44,10 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);
 int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision);
 int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision);
 
-int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
+int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
 int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision);
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
+int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);

include/common/tvariant.h

@@ -40,7 +40,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool*
 
 bool taosVariantIsValid(SVariant *pVar);
 
-void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type);
+void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type);
 
 void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type);
 

include/libs/parser/parser.h

@@ -131,6 +131,18 @@ struct SInsertStmtInfo;
  */
 bool qIsInsertSql(const char* pStr, size_t length);
 
+typedef struct SParseContext {
+  const char* pSql;    // sql string
+  size_t sqlLen;       // length of the sql string
+  int64_t id;          // operator id, generated by uuid generator
+  const char* pDbname;
+  const SEpSet* pEpSet; 
+  int8_t schemaAttached; // denote if submit block is built with table schema or not
+
+  char* pMsg;           // extended error message if exists to help avoid the problem in sql statement.
+  int32_t msgLen;      // max length of the msg
+} SParseContext;
+
 /**
  * Parse the sql statement and then return the SQueryStmtInfo as the result of bounded AST.
  * @param pSql     sql string
@@ -141,16 +153,35 @@ bool qIsInsertSql(const char* pStr, size_t length);
  */
 int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen);
 
+typedef enum {
+  PAYLOAD_TYPE_KV = 0,
+  PAYLOAD_TYPE_RAW = 1,
+} EPayloadType;
+
+typedef struct SVgDataBlocks {
+  int64_t     vgId;         // virtual group id
+  int32_t     numOfTables;  // number of tables in current submit block
+  uint32_t    size;
+  char       *pData;
+} SVgDataBlocks;
+
+typedef struct SInsertStmtInfo {
+  SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
+  int8_t      schemaAttache;        // denote if submit block is built with table schema or not
+  uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+  uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
+  const char* sql;                 // current sql statement position
+} SInsertStmtInfo;
+
 /**
  * Parse the insert sql statement.
  * @param pStr            sql string
  * @param length          length of the sql string
- * @param pInsertParam    data in binary format to submit to vnode directly.
  * @param id              operator id, generated by uuid generator.
  * @param msg             extended error message if exists to help avoid the problem in sql statement.
- * @return
+ * @return                data in binary format to submit to vnode directly.
  */
-int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen);
+ int32_t qParseInsertSql(SParseContext* pContext, struct SInsertStmtInfo** pInfo);
 
 /**
  * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that

include/os/osString.h

@@ -38,11 +38,11 @@ extern "C" {
     (dst)[(size)-1] = 0;            \
   } while (0)
 
-int64_t taosStr2int64(char *str);
+int64_t taosStr2int64(const char *str);
 
 // USE_LIBICONV
 int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
-bool    taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
+bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
 int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
 bool    taosValidateEncodec(const char *encodec);
 char *  taosCharsetReplace(char *charsetstr);

include/util/tutil.h

@@ -29,7 +29,7 @@ int32_t strdequote(char *src);
 int32_t strndequote(char *dst, const char *z, int32_t len);
 int32_t strRmquote(char *z, int32_t len);
 size_t  strtrim(char *src);
-char   *strnchr(char *haystack, char needle, int32_t len, bool skipquote);
+char   *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
 char  **strsplit(char *src, const char *delim, int32_t *num);
 char   *strtolower(char *dst, const char *src);
 char   *strntolower(char *dst, const char *src, int32_t n);

source/common/src/ttime.c

@@ -82,18 +82,18 @@ void deltaToUtcInitOnce() {
 }
 
 static int64_t parseFraction(char* str, char** end, int32_t timePrec);
-static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec, char delim);
+static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim);
 static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec);
 static int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec);
 static char* forwardToTimeStringEnd(char* str);
-static bool checkTzPresent(char *str, int32_t len);
+static bool checkTzPresent(const char *str, int32_t len);
 
 static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = {
   parseLocaltime,
   parseLocaltimeDst
 };
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) {
+int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) {
   /* parse datatime string in with tz */
   if (strnchr(timestr, 'T', len, false) != NULL) {
     return parseTimeWithTz(timestr, time, timePrec, 'T');
@@ -104,7 +104,7 @@ int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePre
   }
 }
 
-bool checkTzPresent(char *str, int32_t len) {
+bool checkTzPresent(const char *str, int32_t len) {
   char *seg = forwardToTimeStringEnd(str);
   int32_t seg_len = len - (int32_t)(seg - str);
 
@@ -237,7 +237,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
  * 2013-04-12T15:52:01+0800
  * 2013-04-12T15:52:01.123+0800
  */
-int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec, char delim) {
+int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) {
 
   int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 :
                              (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000);
@@ -432,7 +432,7 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time
  * d - Days (24 hours)
  * w - Weeks (7 days)
  */
-int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) {
+int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) {
   errno = 0;
   char* endPtr = NULL;
 

source/common/src/tvariant.c

@@ -75,7 +75,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool*
   return 0;
 }
 
-void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
+void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) {
   int32_t ret = 0;
   memset(pVar, 0, sizeof(SVariant));
 

source/libs/parser/inc/dataBlockMgt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DATABLOCKMGT_H
+#define TDENGINE_DATABLOCKMGT_H
+
+#include "catalog.h"
+#include "os.h"
+#include "ttypes.h"
+#include "tname.h"
+
+#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED)
+
+typedef enum EOrderStatus {
+  ORDER_STATUS_UNKNOWN = 0,
+  ORDER_STATUS_ORDERED = 1,
+  ORDER_STATUS_DISORDERED = 2,
+} EOrderStatus;
+
+typedef enum EValStat {
+  VAL_STAT_HAS = 0x0,    // 0 means has val
+  VAL_STAT_NONE = 0x01,  // 1 means no val
+} EValStat;
+
+typedef enum ERowCompareStat {
+  ROW_COMPARE_NO_NEED = 0,
+  ROW_COMPARE_NEED = 1,
+} ERowCompareStat;
+
+typedef struct SBoundColumn {
+  int32_t offset;   // all column offset value
+  int32_t toffset;  // first part offset for SDataRow TODO: get offset from STSchema on future
+  uint8_t valStat;  // EValStat. denote if current column bound or not(0 means has val, 1 means no val)
+} SBoundColumn;
+
+typedef struct {
+  uint16_t schemaColIdx;
+  uint16_t boundIdx;
+  uint16_t finalIdx;
+} SBoundIdxInfo;
+
+typedef struct SParsedDataColInfo {
+  int16_t        numOfCols;
+  int16_t        numOfBound;
+  uint16_t       flen;        // TODO: get from STSchema
+  uint16_t       allNullLen;  // TODO: get from STSchema
+  uint16_t       extendedVarLen;
+  int32_t        *boundedColumns;  // bound column idx according to schema
+  SBoundColumn   *cols;
+  SBoundIdxInfo  *colIdxInfo;
+  int8_t         orderStatus;  // bound columns
+} SParsedDataColInfo;
+
+typedef struct SMemRowInfo {
+  int32_t dataLen;  // len of SDataRow
+  int32_t kvLen;    // len of SKVRow
+} SMemRowInfo;
+
+typedef struct {
+  uint8_t      memRowType;   // default is 0, that is SDataRow 
+  uint8_t      compareStat;  // 0 no need, 1 need compare
+  TDRowTLenT   kvRowInitLen;
+  SMemRowInfo *rowInfo;
+} SMemRowBuilder;
+
+typedef struct SParamInfo {
+  int32_t  idx;
+  uint8_t  type;
+  uint8_t  timePrec;
+  int16_t  bytes;
+  uint32_t offset;
+} SParamInfo;
+
+typedef struct STableDataBlocks {
+  SName       tableName;
+  int8_t      tsSource;     // where does the UNIX timestamp come from, server or client
+  bool        ordered;      // if current rows are ordered or not
+  int64_t     vgId;         // virtual group id
+  int64_t     prevTS;       // previous timestamp, recorded to decide if the records array is ts ascending
+  int32_t     numOfTables;  // number of tables in current submit block
+  int32_t     rowSize;      // row size for current table
+  uint32_t    nAllocSize;
+  uint32_t    headerSize;   // header for table info (uid, tid, submit metadata)
+  uint32_t    size;
+  STableMeta *pTableMeta;   // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache
+  char       *pData;
+  bool        cloned;
+  STagData    tagData; 
+  
+  SParsedDataColInfo boundColumnInfo;
+
+  // for parameter ('?') binding
+  uint32_t       numOfAllocedParams;
+  uint32_t       numOfParams;
+  SParamInfo *   params;
+  SMemRowBuilder rowBuilder;
+} STableDataBlocks;
+
+static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) {
+  memRowSetType(row, memRowType);
+  if (isDataRowT(memRowType)) {
+    dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion);
+    dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen));
+  } else {
+    ASSERT(nBoundCols > 0);
+    memRowSetKvVersion(row, pBlock->pTableMeta->sversion);
+    kvRowSetNCols(memRowKvBody(row), nBoundCols);
+    kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
+  }
+}
+
+static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
+  ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize);
+  return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen;
+}
+
+// Applicable to consume by one row
+static FORCE_INLINE void appendMemRowColValEx(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
+                                                 int8_t colType, int32_t toffset, int32_t *dataLen, int32_t *kvLen,
+                                                 uint8_t compareStat) {
+  tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset);
+  if (compareStat == ROW_COMPARE_NEED) {
+    tdGetColAppendDeltaLen(value, colType, dataLen, kvLen);
+  }
+}
+
+static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd,
+                                                int32_t idx, int32_t *toffset) {
+  int32_t schemaIdx = 0;
+  if (IS_DATA_COL_ORDERED(spd)) {
+    schemaIdx = spd->boundedColumns[idx];
+    if (isDataRowT(memRowType)) {
+      *toffset = (spd->cols + schemaIdx)->toffset;  // the offset of firstPart
+    } else {
+      *toffset = idx * sizeof(SColIdx);  // the offset of SColIdx
+    }
+  } else {
+    ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx);
+    schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx;
+    if (isDataRowT(memRowType)) {
+      *toffset = (spd->cols + schemaIdx)->toffset;
+    } else {
+      *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx);
+    }
+  }
+}
+
+static FORCE_INLINE void convertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) {
+  if (isDataRow(row)) {
+    if (kvLen < (dataLen * KVRatioConvert)) {
+      memRowSetConvert(row);
+    }
+  } else if (kvLen > dataLen) {
+    memRowSetConvert(row);
+  }
+}
+
+static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
+  pBlocks->tid = pTableMeta->suid;
+  pBlocks->uid = pTableMeta->uid;
+  pBlocks->sversion = pTableMeta->sversion;
+
+  if (pBlocks->numOfRows + numOfRows >= INT16_MAX) {
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  } else {
+    pBlocks->numOfRows += numOfRows;
+    return TSDB_CODE_SUCCESS;
+  }
+}
+
+int32_t schemaIdxCompar(const void *lhs, const void *rhs);
+int32_t boundIdxCompar(const void *lhs, const void *rhs);
+void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols);
+void destroyBoundColumnInfo(SParsedDataColInfo* pColList);
+int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen);
+int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
+int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,
+    SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList);
+int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap);
+
+#endif  // TDENGINE_DATABLOCKMGT_H

source/libs/parser/inc/insertParser.h

@@ -16,4 +16,8 @@
 #ifndef TDENGINE_INSERTPARSER_H
 #define TDENGINE_INSERTPARSER_H
 
+#include "parser.h"
+
+int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo);
+
 #endif  // TDENGINE_INSERTPARSER_H

source/libs/parser/inc/parserInt.h

@@ -26,14 +26,6 @@ extern "C" {
 
 struct SSqlNode;
 
-typedef struct SInsertStmtInfo {
-  SHashObj *pTableBlockHashList;     // data block for each table
-  SArray   *pDataBlocks;             // SArray<STableDataBlocks*>. Merged submit block for each vgroup
-  int8_t    schemaAttached;          // denote if submit block is built with table schema or not
-  uint8_t   payloadType;             // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
-  uint32_t  insertType;              // insert data from [file|sql statement| bound statement]
-  char     *sql;                     // current sql statement position
-} SInsertStmtInfo;
 
 typedef struct SInternalField {
   TAOS_FIELD      field;

source/libs/parser/inc/parserUtil.h

@@ -46,7 +46,7 @@ SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index);
 
 int32_t parserValidateIdToken(SToken* pToken);
 int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
-int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo,  const char* sourceStr);
+int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo,  const char* sourceStr);
 
 STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo);
 
@@ -61,6 +61,8 @@ void cleanupColumnCond(SArray** pCond);
 uint32_t convertRelationalOperator(SToken *pToken);
 int32_t    getExprFunctionId(SExprInfo *pExprInfo);
 
+STableMeta* tableMetaDup(const STableMeta* pTableMeta);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/parser/inc/ttoken.h

@@ -44,7 +44,7 @@ typedef struct SToken {
  * @param tokenType
  * @return
  */
-uint32_t tGetToken(char *z, uint32_t *tokenType);
+uint32_t tGetToken(const char *z, uint32_t *tokenType);
 
 /**
  * enhanced tokenizer for sql string.
@@ -54,7 +54,7 @@ uint32_t tGetToken(char *z, uint32_t *tokenType);
  * @param isPrevOptr
  * @return
  */
-SToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr);
+SToken tStrGetToken(const char *str, int32_t *i, bool isPrevOptr);
 
 /**
  * check if it is a keyword or not

source/libs/parser/src/astGenerator.c

@@ -191,7 +191,7 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
   pExpr->type = SQL_NODE_EXPR;
 
   if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) {
-    char* endPos   = pRight->exprToken.z + pRight->exprToken.n;
+    const char* endPos   = pRight->exprToken.z + pRight->exprToken.n;
     pExpr->exprToken.z = pLeft->exprToken.z;
     pExpr->exprToken.n = (uint32_t)(endPos - pExpr->exprToken.z);
     pExpr->exprToken.type = pLeft->exprToken.type;

source/libs/parser/src/parser.c

@@ -18,6 +18,7 @@
 #include "parserUtil.h"
 #include "ttoken.h"
 #include "function.h"
+#include "insertParser.h"
 
 bool qIsInsertSql(const char* pStr, size_t length) {
   int32_t index = 0;
@@ -46,8 +47,8 @@ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo**
   return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen);
 }
 
-int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen) {
-  return 0;
+int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
+  return parseInsertSql(pContext, pInfo);
 }
 
 int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) {
@@ -173,7 +174,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet
       assert(t != NULL);
 
       if (t->n >= TSDB_FUNC_NAME_LEN) {
-        return buildSyntaxErrMsg(msg, msgBufLen, "too long function name", t->z);
+        return buildSyntaxErrMsg(&msgBuf, "too long function name", t->z);
       }
 
       // Let's assume that it is an UDF/UDAF, if it is not a built-in function.

source/libs/parser/src/parserUtil.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "parserUtil.h"
+
 #include "taosmsg.h"
 #include "parser.h"
 #include "taoserror.h"
@@ -18,7 +34,6 @@ typedef struct STableFilterCond {
 
 static STableMetaInfo* addTableMetaInfo(SQueryStmtInfo* pQueryInfo, SName* name, STableMeta* pTableMeta,
                                         SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables);
-STableMeta* tableMetaDup(STableMeta* pTableMeta);
 
 int32_t parserValidateIdToken(SToken* pToken) {
   if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) {
@@ -87,7 +102,7 @@ int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) {
   return TSDB_CODE_TSC_INVALID_OPERATION;
 }
 
-int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo,  const char* sourceStr) {
+int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo,  const char* sourceStr) {
   const char* msgFormat1 = "syntax error near \'%s\'";
   const char* msgFormat2 = "syntax error near \'%s\' (%s)";
   const char* msgFormat3 = "%s";
@@ -95,7 +110,7 @@ int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalIn
   const char* prefix = "syntax error";
   if (sourceStr == NULL) {
     assert(additionalInfo != NULL);
-    snprintf(dst, dstBufLen, msgFormat1, additionalInfo);
+    snprintf(pBuf->buf, pBuf->len, msgFormat1, additionalInfo);
     return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
   }
 
@@ -103,10 +118,10 @@ int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalIn
   strncpy(buf, sourceStr, tListLen(buf) - 1);
 
   if (additionalInfo != NULL) {
-    snprintf(dst, dstBufLen, msgFormat2, buf, additionalInfo);
+    snprintf(pBuf->buf, pBuf->len, msgFormat2, buf, additionalInfo);
   } else {
     const char* msgFormat = (0 == strncmp(sourceStr, prefix, strlen(prefix))) ? msgFormat3 : msgFormat1;
-    snprintf(dst, dstBufLen, msgFormat, buf);
+    snprintf(pBuf->buf, pBuf->len, msgFormat, buf);
   }
 
   return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
@@ -1490,7 +1505,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) {
   return pTableMeta;
 }
 
-uint32_t getTableMetaSize(STableMeta* pTableMeta) {
+uint32_t getTableMetaSize(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
 
   int32_t totalCols = 0;
@@ -1505,7 +1520,7 @@ uint32_t getTableMetaMaxSize() {
   return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema);
 }
 
-STableMeta* tableMetaDup(STableMeta* pTableMeta) {
+STableMeta* tableMetaDup(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
   size_t size = getTableMetaSize(pTableMeta);
 

source/libs/parser/src/ttokenizer.c

@@ -284,7 +284,7 @@ static int32_t tKeywordCode(const char* z, int n) {
  * Return the length of the token that begins at z[0].
  * Store the token type in *type before returning.
  */
-uint32_t tGetToken(char* z, uint32_t* tokenId) {
+uint32_t tGetToken(const char* z, uint32_t* tokenId) {
   uint32_t i;
   switch (*z) {
     case ' ':
@@ -595,7 +595,7 @@ SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken) {
   return ntoken;
 }
 
-SToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
+SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) {
   SToken t0 = {0};
 
   // here we reach the end of sql string, null-terminated string
@@ -689,13 +689,12 @@ void taosCleanupKeywordsTable() {
 }
 
 SToken taosTokenDup(SToken* pToken, char* buf, int32_t len) {
-  assert(pToken != NULL && buf != NULL);
+  assert(pToken != NULL && buf != NULL && len > pToken->n);
+  
+  strncpy(buf, pToken->z, pToken->n);
+  buf[pToken->n] = 0;
+
   SToken token = *pToken;
   token.z = buf;
-
-  assert(len > token.n);
-  strncpy(token.z, pToken->z, pToken->n);
-  token.z[token.n] = 0;
-
   return token;
 }

source/os/src/osString.c

@@ -37,7 +37,7 @@ char *taosCharsetReplace(char *charsetstr) {
   return strdup(charsetstr);
 }
 
-int64_t taosStr2int64(char *str) {
+int64_t taosStr2int64(const char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
 }
@@ -107,7 +107,7 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
   return len;
 }
 
-bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
+bool taosMbsToUcs4(const char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, int32_t *len) {
   memset(ucs4, 0, ucs4_max_len);
   mbstate_t state = {0};
   int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);

source/util/src/tutil.c

@@ -166,7 +166,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) {
   return split;
 }
 
-char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) {
+char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) {
   for (int32_t i = 0; i < len; ++i) {
 
     // skip the needle in quote, jump to the end of quoted string
@@ -179,7 +179,7 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) {
     }
 
     if (haystack[i] == needle) {
-      return &haystack[i];
+      return (char *)&haystack[i];
     }
   }