Merge branch 'develop' into hotfix/TD-5863

documentation20/cn/12.taos-sql/docs.md

@@ -435,6 +435,17 @@ INSERT INTO
     INSERT INTO d1001 FILE '/tmp/csvfile.csv';
     ```
 
+- **插入来自文件的数据记录，并自动建表**  
+    从 2.1.5.0 版本开始，支持在插入来自 CSV 文件的数据时，以超级表为模板来自动创建不存在的数据表。例如：  
+    ```mysql
+    INSERT INTO d21001 USING meters TAGS ('Beijing.Chaoyang', 2) FILE '/tmp/csvfile.csv';
+    ```
+    也可以在一条语句中向多个表以自动建表的方式插入记录。例如：  
+    ```mysql
+    INSERT INTO d21001 USING meters TAGS ('Beijing.Chaoyang', 2) FILE '/tmp/csvfile_21001.csv'
+                d21002 USING meters (groupId) TAGS (2) FILE '/tmp/csvfile_21002.csv';
+    ```
+
 **历史记录写入**：可使用IMPORT或者INSERT命令，IMPORT的语法，功能与INSERT完全一样。
 
 **说明：**针对 insert 类型的 SQL 语句，我们采用的流式解析策略，在发现后面的错误之前，前面正确的部分 SQL 仍会执行。下面的 SQL 中，INSERT 语句是无效的，但是 d1001 仍会被创建。
@@ -1215,6 +1226,37 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
     Query OK, 1 row(s) in set (0.001042s)
     ```
 
+- **INTERP**
+    ```mysql
+    SELECT INTERP(field_name) FROM { tb_name | stb_name } WHERE ts='timestamp' [FILL ({ VALUE | PREV | NULL | LINEAR})];
+    ```
+    功能说明：返回表/超级表的指定时间截面、指定字段的记录。
+
+    返回结果数据类型：同应用的字段。
+
+    应用字段：所有字段。
+
+    适用于：**表、超级表**。
+
+    说明：（从 2.0.15.0 版本开始新增此函数）INTERP 必须指定时间断面，如果该时间断面不存在直接对应的数据，那么会根据 FILL 参数的设定进行插值。其中，条件语句里面可以附带更多的筛选条件，例如标签、tbname。
+
+    限制：INTERP 目前不支持 FILL(NEXT)。
+
+    示例：
+    ```mysql
+    taos> select interp(*) from meters where ts='2017-7-14 10:42:00.005' fill(prev);
+           interp(ts)        | interp(f1)  | interp(f2)  | interp(f3)  |
+    ====================================================================
+     2017-07-14 10:42:00.005 |           5 |           9 |           6 |
+    Query OK, 1 row(s) in set (0.002912s)
+    
+    taos> select interp(*) from meters where tbname in ('t1') and ts='2017-7-14 10:42:00.005' fill(prev);
+           interp(ts)        | interp(f1)  | interp(f2)  | interp(f3)  |
+    ====================================================================
+     2017-07-14 10:42:00.005 |           5 |           6 |           7 |
+    Query OK, 1 row(s) in set (0.002005s)
+    ```
+
 ### 计算函数
 
 - **DIFF**

src/client/inc/tscUtil.h

@@ -61,6 +61,7 @@ typedef struct SJoinSupporter {
   uint64_t        uid;            // query table uid
   SArray*         colList;        // previous query information, no need to use this attribute, and the corresponding attribution
   SArray*         exprList;
+  SArray*         colCond;
   SFieldInfo      fieldsInfo;
   STagCond        tagCond;
   SGroupbyExpr    groupInfo;       // group by info
@@ -244,8 +245,9 @@ SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid);
 void   tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw);
 
 int32_t tscTagCondCopy(STagCond* dest, const STagCond* src);
+int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx);
 void tscTagCondRelease(STagCond* pCond);
-
+void tscColCondRelease(SArray** pCond);
 void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo);
 
 bool tscShouldBeFreed(SSqlObj* pSql);
@@ -340,7 +342,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild);
 uint32_t tscGetTableMetaSize(STableMeta* pTableMeta);
 CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta);
 uint32_t tscGetTableMetaMaxSize();
-int32_t tscCreateTableMetaFromSTableMeta(STableMeta** pChild, const char* name, size_t *tableMetaCapacity);
+int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta **ppStable);
 STableMeta* tscTableMetaDup(STableMeta* pTableMeta);
 SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo);
 
@@ -355,6 +357,7 @@ char* strdup_throw(const char* str);
 
 bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src);
 SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg);
+STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx);
 
 void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id);
 

src/client/src/tscPrepare.c

@@ -299,7 +299,7 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) {
   SSchema *schema = (SSchema*)pBlock->pTableMeta->schema;
 
   for (int32_t i = 0; i < spd->numOfCols; ++i) {
-    if (!spd->cols[i].hasVal) {  // current column do not have any value to insert, set it to null
+    if (spd->cols[i].valStat == VAL_STAT_NONE) {  // current column do not have any value to insert, set it to null
       for (int32_t n = 0; n < rowNum; ++n) {
         char *ptr = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * n + offset;
 

src/client/src/tscServer.c

@@ -684,7 +684,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
   SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
 
   int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
-  int32_t srcColFilterSize = tscGetColFilterSerializeLen(pQueryInfo);
+  int32_t srcColFilterSize = 0;
   int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo);
 
   size_t  numOfExprs = tscNumOfExprs(pQueryInfo);
@@ -695,6 +695,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
 
   int32_t tableSerialize = 0;
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;  
   if (pTableMetaInfo->pVgroupTables != NULL) {
     size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
 
@@ -707,8 +708,15 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
     tableSerialize = totalTables * sizeof(STableIdInfo);
   }
 
-  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize +
-         exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
+  if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) {
+    STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0);
+    if (pCond != NULL && pCond->cond != NULL) {
+      srcColFilterSize = pCond->len;
+    }
+  }
+
+  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + exprSize + tsBufSize +
+         tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
 }
 
 static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg,
@@ -966,10 +974,21 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pQueryMsg->tableCols[i].colId = htons(pCol->colId);
     pQueryMsg->tableCols[i].bytes = htons(pCol->bytes);
     pQueryMsg->tableCols[i].type  = htons(pCol->type);
-    pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters);
+    //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters);
+    pQueryMsg->tableCols[i].flist.numOfFilters = 0;
 
     // append the filter information after the basic column information
-    serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg);
+    //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg);
+  }
+
+  if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) {
+    STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0);
+    if (pCond != NULL && pCond->cond != NULL) {
+      pQueryMsg->colCondLen = htons(pCond->len);
+      memcpy(pMsg, pCond->cond, pCond->len);
+
+      pMsg += pCond->len;
+    }
   }
 
   for (int32_t i = 0; i < query.numOfOutput; ++i) {
@@ -1044,7 +1063,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
     SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid);
     if (pCond != NULL && pCond->cond != NULL) {
-      pQueryMsg->tagCondLen = htonl(pCond->len);
+      pQueryMsg->tagCondLen = htons(pCond->len);
       memcpy(pMsg, pCond->cond, pCond->len);
 
       pMsg += pCond->len;
@@ -2853,18 +2872,19 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool
   tNameExtractFullName(&pTableMetaInfo->name, name);
 
   size_t len = strlen(name);
-  if (pTableMetaInfo->tableMetaCapacity != 0) {
-    if (pTableMetaInfo->pTableMeta != NULL) {
+   // just make runtime happy
+  if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) {
     memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity);
   } 
-  } 
   taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity);
   
   STableMeta* pMeta   = pTableMetaInfo->pTableMeta;
+  STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf);
   if (pMeta && pMeta->id.uid > 0) {
     // in case of child table, here only get the
     if (pMeta->tableType == TSDB_CHILD_TABLE) {
-      int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity);
+      int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta));
+      pSql->pBuf   = (void *)(pSTMeta); 
       if (code != TSDB_CODE_SUCCESS) {
         return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate);
       }

src/client/src/tscSubquery.c

@@ -796,6 +796,7 @@ static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
   STimeWindow window = pQueryInfo->window;
   tscInitQueryInfo(pQueryInfo);
 
+  pQueryInfo->colCond = pSupporter->colCond;
   pQueryInfo->window = window;
   TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
   TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
@@ -1883,6 +1884,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
     if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag
       SColumnIndex colIndex = {0};
 
+      pSupporter->colCond = pNewQueryInfo->colCond;
+      pNewQueryInfo->colCond = NULL;
+
       STagCond* pTagCond = &pSupporter->tagCond;
       assert(pTagCond->joinInfo.hasJoin);
 
@@ -2319,6 +2323,11 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) {
     goto _error;
   }
 
+  if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, 0) != 0) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto _error;
+  }
+
   pNewQueryInfo->window   = pQueryInfo->window;
   pNewQueryInfo->interval = pQueryInfo->interval;
   pNewQueryInfo->sessionWindow = pQueryInfo->sessionWindow;

src/common/inc/tdataformat.h

@@ -186,6 +186,7 @@ typedef void *SDataRow;
 #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
 
 #define dataRowLen(r) (*(TDRowLenT *)(r))  // 0~65535
+#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r))
 #define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
 #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
 #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
@@ -201,14 +202,18 @@ void     tdFreeDataRow(SDataRow row);
 void     tdInitDataRow(SDataRow row, STSchema *pSchema);
 SDataRow tdDataRowDup(SDataRow row);
 
+
 // offset here not include dataRow header length
-static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
+static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
+                                           int32_t offset) {
   ASSERT(value != NULL);
   int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
 
   if (IS_VAR_DATA_TYPE(type)) {
     *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
+    if (isCopyVarData) {
       memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
+    }
     dataRowLen(row) += varDataTLen(value);
   } else {
     if (offset == 0) {
@@ -223,6 +228,12 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t
   return 0;
 }
 
+
+// offset here not include dataRow header length
+static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
+  return tdAppendDataColVal(row, value, true, type, offset);
+}
+
 // NOTE: offset here including the header size
 static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
   if (IS_VAR_DATA_TYPE(type)) {
@@ -328,11 +339,10 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 int tdAllocMemForCol(SDataCol *pCol, int maxPoints);
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints);
-void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
+int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
 void dataColSetOffset(SDataCol *pCol, int nEle);
 
 bool isNEleNull(SDataCol *pCol, int nEle);
-void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
 
 // Get the data pointer from a column-wised data
 static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) {
@@ -357,10 +367,8 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
 }
 
 typedef struct {
-  int maxRowSize;
   int      maxCols;    // max number of columns
   int      maxPoints;  // max number of points
-
   int      numOfRows;
   int      numOfCols;  // Total number of cols
   int      sversion;   // TODO: set sversion
@@ -407,7 +415,7 @@ static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) {
   }
 }
 
-SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
+SDataCols *tdNewDataCols(int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
 int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
@@ -475,9 +483,10 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
 }
 
 // offset here not include kvRow header length
-static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t *offset) {
+static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
+                                         int32_t offset) {
   ASSERT(value != NULL);
-  int32_t  toffset = *offset + TD_KV_ROW_HEAD_SIZE;
+  int32_t  toffset = offset + TD_KV_ROW_HEAD_SIZE;
   SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset);
   char *   ptr = (char *)POINTER_SHIFT(row, kvRowLen(row));
 
@@ -485,10 +494,12 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t
   pColIdx->offset = kvRowLen(row);  // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE
 
   if (IS_VAR_DATA_TYPE(type)) {
+    if (isCopyValData) {
       memcpy(ptr, value, varDataTLen(value));
+    }
     kvRowLen(row) += varDataTLen(value);
   } else {
-    if (*offset == 0) {
+    if (offset == 0) {
       ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
       TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
       memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]);
@@ -497,7 +508,6 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t
     }
     kvRowLen(row) += TYPE_BYTES[type];
   }
-  *offset += sizeof(SColIdx);
 
   return 0;
 }
@@ -592,12 +602,24 @@ typedef void *SMemRow;
 #define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE)
 #define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE)
 
-#define SMEM_ROW_DATA 0U  // SDataRow
-#define SMEM_ROW_KV 1U    // SKVRow
+#define SMEM_ROW_DATA 0x0U      // SDataRow
+#define SMEM_ROW_KV 0x01U       // SKVRow
+#define SMEM_ROW_CONVERT 0x80U  // SMemRow convert flag
+
+#define KVRatioKV (0.2f)  // all bool
+#define KVRatioPredict (0.4f)
+#define KVRatioData (0.75f)  // all bigint
+#define KVRatioConvert (0.9f)
 
-#define memRowType(r) (*(uint8_t *)(r))
+#define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
+
+#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t))  // set the total byte in case of dirty memory
+#define memRowSetConvert(r) ((*(uint8_t *)(r)) = (((*(uint8_t *)(r)) & 0x7F) | SMEM_ROW_CONVERT))  // highest bit
+#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
 #define isDataRow(r) (SMEM_ROW_DATA == memRowType(r))
+#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
 #define isKvRow(r) (SMEM_ROW_KV == memRowType(r))
+#define isNeedConvertRow(r) (((*(uint8_t *)(r)) & 0x80) == SMEM_ROW_CONVERT)
 
 #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)  // section after flag
 #define memRowKvBody(r) \
@@ -614,6 +636,14 @@ typedef void *SMemRow;
 #define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
 #define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r))  // using uint32_t/int32_t to store the TLen
 
+static FORCE_INLINE char *memRowEnd(SMemRow row) {
+  if (isDataRow(row)) {
+    return (char *)dataRowEnd(memRowDataBody(row));
+  } else {
+    return (char *)kvRowEnd(memRowKvBody(row));
+  }
+}
+
 #define memRowDataVersion(r) dataRowVersion(memRowDataBody(r))
 #define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
 #define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r))  // schema version
@@ -631,7 +661,6 @@ typedef void *SMemRow;
     }                                       \
   } while (0)
 
-#define memRowSetType(r, t) (memRowType(r) = (t))
 #define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l))
 #define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v))
 #define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r))
@@ -664,12 +693,12 @@ static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_
   }
 }
 
-static FORCE_INLINE int tdAppendMemColVal(SMemRow row, const void *value, int16_t colId, int8_t type, int32_t offset,
-                                          int32_t *kvOffset) {
+static FORCE_INLINE int tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
+                                             int8_t type, int32_t offset) {
   if (isDataRow(row)) {
-    tdAppendColVal(memRowDataBody(row), value, type, offset);
+    tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset);
   } else {
-    tdAppendKvColVal(memRowKvBody(row), value, colId, type, kvOffset);
+    tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset);
   }
   return 0;
 }
@@ -691,6 +720,30 @@ static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value
   return len;
 }
 
+/**
+ * 1. calculate the delta of AllNullLen for SDataRow.
+ * 2. calculate the real len for SKVRow.
+ */
+static FORCE_INLINE void tdGetColAppendDeltaLen(const void *value, int8_t colType, int32_t *dataLen, int32_t *kvLen) {
+  switch (colType) {
+    case TSDB_DATA_TYPE_BINARY: {
+      int32_t varLen = varDataLen(value);
+      *dataLen += (varLen - CHAR_BYTES);
+      *kvLen += (varLen + sizeof(SColIdx));
+      break;
+    }
+    case TSDB_DATA_TYPE_NCHAR: {
+      int32_t varLen = varDataLen(value);
+      *dataLen += (varLen - TSDB_NCHAR_SIZE);
+      *kvLen += (varLen + sizeof(SColIdx));
+      break;
+    }
+    default: {
+      *kvLen += (TYPE_BYTES[colType] + sizeof(SColIdx));
+      break;
+    }
+  }
+}
 
 typedef struct {
   int16_t colId;
@@ -706,7 +759,7 @@ static FORCE_INLINE void setSColInfo(SColInfo* colInfo, int16_t colId, uint8_t c
 
 SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
 
-
+#if 0
 // ----------------- Raw payload structure for row:
 /* |<------------ Head ------------->|<----------- body of column data tuple ------------------->|
  * |                                 |<----------------- flen ------------->|<--- value part --->|
@@ -752,6 +805,8 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch
 
 static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); }
 
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/common/inc/tvariant.h

@@ -53,6 +53,8 @@ int32_t tVariantToString(tVariant *pVar, char *dst);
 
 int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix);
 
+int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo);
+
 int32_t tVariantTypeSetType(tVariant *pVariant, char type);
 
 #ifdef __cplusplus

src/common/src/tdataformat.c

@@ -19,10 +19,10 @@
 #include "wchar.h"
 #include "tarray.h"
 
+static void dataColSetNEleNull(SDataCol *pCol, int nEle);
 static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
                                int limit2, int tRows, bool forceSetNull);
 
-//TODO: change caller to use return val
 int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   int spaceNeeded = pCol->bytes * maxPoints;
   if(IS_VAR_DATA_TYPE(pCol->type)) {
@@ -31,7 +31,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   if(pCol->spaceSize < spaceNeeded) {
     void* ptr = realloc(pCol->pData, spaceNeeded);
     if(ptr == NULL) {
-      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)pCol->spaceSize,
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded,
              strerror(errno));
       return -1;
     } else {
@@ -239,20 +239,19 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
   pDataCol->len = 0;
 }
 // value from timestamp should be TKEY here instead of TSKEY
-void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
+int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
   ASSERT(pCol != NULL && value != NULL);
 
   if (isAllRowsNull(pCol)) {
     if (isNull(value, pCol->type)) {
       // all null value yet, just return
-      return;
+      return 0;
     }
 
+    if(tdAllocMemForCol(pCol, maxPoints) < 0) return -1;
     if (numOfRows > 0) {
       // Find the first not null value, fill all previouse values as NULL
-      dataColSetNEleNull(pCol, numOfRows, maxPoints);
-    } else {
-      tdAllocMemForCol(pCol, maxPoints);
+      dataColSetNEleNull(pCol, numOfRows);
     }
   }
 
@@ -268,12 +267,21 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP
     memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
     pCol->len += pCol->bytes;
   }
+  return 0;
+}
+
+static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) {
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
+  } else {
+    return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
+  }
 }
 
 bool isNEleNull(SDataCol *pCol, int nEle) {
   if(isAllRowsNull(pCol)) return true;
   for (int i = 0; i < nEle; i++) {
-    if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
+    if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false;
   }
   return true;
 }
@@ -290,9 +298,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
   }
 }
 
-void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) {
-  tdAllocMemForCol(pCol, maxPoints);
-
+static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->len = 0;
     for (int i = 0; i < nEle; i++) {
@@ -318,7 +324,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle) {
   }
 }
 
-SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
+SDataCols *tdNewDataCols(int maxCols, int maxRows) {
   SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols));
   if (pCols == NULL) {
     uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno));
@@ -326,6 +332,9 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
   }
 
   pCols->maxPoints = maxRows;
+  pCols->maxCols = maxCols;
+  pCols->numOfRows = 0;
+  pCols->numOfCols = 0;
 
   if (maxCols > 0) {
     pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
@@ -342,13 +351,8 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
       pCols->cols[i].pData = NULL;
       pCols->cols[i].dataOff = NULL;
     }
-
-    pCols->maxCols = maxCols;
   }
 
-  pCols->maxRowSize = maxRowSize;
-
-
   return pCols;
 }
 
@@ -357,8 +361,9 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
   int oldMaxCols = pCols->maxCols;
   if (schemaNCols(pSchema) > oldMaxCols) {
     pCols->maxCols = schemaNCols(pSchema);
-    pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
-    if (pCols->cols == NULL) return -1;
+    void* ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
+    if (ptr == NULL) return -1;
+    pCols->cols = ptr;
     for(i = oldMaxCols; i < pCols->maxCols; i++) {
       pCols->cols[i].pData = NULL;
       pCols->cols[i].dataOff = NULL;
@@ -366,10 +371,6 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
     }
   }
 
-  if (schemaTLen(pSchema) > pCols->maxRowSize) {
-    pCols->maxRowSize = schemaTLen(pSchema);
-  }
-
   tdResetDataCols(pCols);
   pCols->numOfCols = schemaNCols(pSchema);
 
@@ -398,7 +399,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
 }
 
 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
-  SDataCols *pRet = tdNewDataCols(pDataCols->maxRowSize, pDataCols->maxCols, pDataCols->maxPoints);
+  SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
   if (pRet == NULL) return NULL;
 
   pRet->numOfCols = pDataCols->numOfCols;
@@ -413,7 +414,10 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 
     if (keepData) {
       if (pDataCols->cols[i].len > 0) {
-        tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints);
+        if(tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) {
+          tdFreeDataCols(pRet);
+          return NULL;
+        }
         pRet->cols[i].len = pDataCols->cols[i].len;
         memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
         if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
@@ -584,9 +588,12 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
       if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
         for (int i = 0; i < src2->numOfCols; i++) {
           ASSERT(target->cols[i].type == src2->cols[i].type);
-          if (src2->cols[i].len > 0 && (forceSetNull || (!forceSetNull && !isNull(src2->cols[i].pData, src2->cols[i].type)))) {
+          if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) {
             dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
                              target->maxPoints);
+          } else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
+            dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
+                             target->maxPoints);
           }
         }
         target->numOfRows++;
@@ -844,7 +851,8 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch
     int16_t k;
     for (k = 0; k < nKvNCols; ++k) {
       SColInfo *pColInfo = taosArrayGet(stashRow, k);
-      tdAppendKvColVal(kvRow, pColInfo->colVal, pColInfo->colId, pColInfo->colType, &toffset);
+      tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset);
+      toffset += sizeof(SColIdx);
     }
     ASSERT(kvLen == memRowTLen(tRow));
   }

src/common/src/texpr.c

@@ -118,7 +118,7 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) {
   } else if (pNode->nodeType == TSQL_NODE_VALUE) {
     tVariantDestroy(pNode->pVal);
   } else if (pNode->nodeType == TSQL_NODE_COL) {
-    free(pNode->pSchema);
+    tfree(pNode->pSchema);
   }
 
   free(pNode);
@@ -435,7 +435,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
     expr->_node.optr = TSDB_RELATION_IN;
     tVariant* pVal = exception_calloc(1, sizeof(tVariant));
     right->pVal = pVal;
-    pVal->nType = TSDB_DATA_TYPE_ARRAY;
+    pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY;
     pVal->arr = taosArrayInit(2, POINTER_BYTES);
 
     const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN;
@@ -502,6 +502,183 @@ void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) {
   *q = (void *)pObj;
 }
 
+void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) {
+  SBufferReader br = tbufInitReader(buf, len, false); 
+  uint32_t sType  = tbufReadUint32(&br);     
+  SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false);
+  
+  taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType)); 
+  
+  int dummy = -1;
+  tVariant tmpVar = {0};  
+  size_t  t = 0;
+  int32_t sz = tbufReadInt32(&br);
+  void *pvar = NULL;  
+  int64_t val = 0;
+  int32_t bufLen = 0;
+  if (IS_NUMERIC_TYPE(sType)) {
+    bufLen = 60;  // The maximum length of string that a number is converted to.
+  } else {
+    bufLen = 128;
+  }
+
+  char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE);
+    
+  for (int32_t i = 0; i < sz; i++) {
+    switch (sType) {
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_TINYINT: {
+      *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); 
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_SMALLINT: {
+      *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); 
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_INT: {
+      *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); 
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_TIMESTAMP:
+    case TSDB_DATA_TYPE_UBIGINT:
+    case TSDB_DATA_TYPE_BIGINT: {
+      *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); 
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      *(double *)&val = tbufReadDouble(&br);
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      *(float *)&val = (float)tbufReadDouble(&br);
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_BINARY: {
+      pvar = (char *)tbufReadBinary(&br, &t);
+      break;
+    }
+    case TSDB_DATA_TYPE_NCHAR: {
+      pvar = (char *)tbufReadBinary(&br, &t);      
+      break;
+    }
+    default:
+      taosHashCleanup(pObj);
+      *q = NULL;
+      return;
+    }
+    
+    tVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType);
+
+    if (bufLen < t) {
+      tmp = realloc(tmp, t * TSDB_NCHAR_SIZE);
+      bufLen = (int32_t)t;
+    }
+
+    switch (tType) {
+      case TSDB_DATA_TYPE_BOOL:
+      case TSDB_DATA_TYPE_UTINYINT:
+      case TSDB_DATA_TYPE_TINYINT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_USMALLINT:
+      case TSDB_DATA_TYPE_SMALLINT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_UINT:
+      case TSDB_DATA_TYPE_INT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_TIMESTAMP:
+      case TSDB_DATA_TYPE_UBIGINT:
+      case TSDB_DATA_TYPE_BIGINT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_DOUBLE: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_FLOAT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_BINARY: {
+        if (tVariantDump(&tmpVar, tmp, tType, true)) {
+          goto err_ret;
+        }
+        t = varDataLen(tmp);
+        pvar = varDataVal(tmp);
+        break;
+      }
+      case TSDB_DATA_TYPE_NCHAR: {
+        if (tVariantDump(&tmpVar, tmp, tType, true)) {
+          goto err_ret;
+        }
+        t = varDataLen(tmp);
+        pvar = varDataVal(tmp);        
+        break;
+      }
+      default:
+        goto err_ret;
+    }
+    
+    taosHashPut(pObj, (char *)pvar, t,  &dummy, sizeof(dummy));
+    tVariantDestroy(&tmpVar);
+    memset(&tmpVar, 0, sizeof(tmpVar));
+  } 
+
+  *q = (void *)pObj;
+  pObj = NULL;
+  
+err_ret:  
+  tVariantDestroy(&tmpVar);
+  taosHashCleanup(pObj);
+  tfree(tmp);
+}
+
+
 tExprNode* exprdup(tExprNode* pNode) {
   if (pNode == NULL) {
     return NULL;

src/common/src/tname.c

@@ -61,7 +61,7 @@ bool tscValidateTableNameLength(size_t len) {
 
 // TODO refactor
 SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters) {
-  if (numOfFilters == 0) {
+  if (numOfFilters == 0 || src == NULL) {
     assert(src == NULL);
     return NULL;
   }

src/common/src/ttypes.c

@@ -372,21 +372,21 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i
 }
 
 tDataTypeDescriptor tDataTypes[15] = {
-  {TSDB_DATA_TYPE_NULL,      6,1,        "NOTYPE",   NULL,     NULL,      NULL},
-  {TSDB_DATA_TYPE_BOOL,      4,  CHAR_BYTES,   "BOOL",               tsCompressBool,      tsDecompressBool,      getStatics_bool},
-  {TSDB_DATA_TYPE_TINYINT,   7,  CHAR_BYTES,   "TINYINT",            tsCompressTinyint,   tsDecompressTinyint,   getStatics_i8},
-  {TSDB_DATA_TYPE_SMALLINT,  8,  SHORT_BYTES,  "SMALLINT",           tsCompressSmallint,  tsDecompressSmallint,  getStatics_i16},
-  {TSDB_DATA_TYPE_INT,       3,  INT_BYTES,    "INT",                tsCompressInt,       tsDecompressInt,       getStatics_i32},
-  {TSDB_DATA_TYPE_BIGINT,    6,  LONG_BYTES,   "BIGINT",             tsCompressBigint,    tsDecompressBigint,    getStatics_i64},
-  {TSDB_DATA_TYPE_FLOAT,     5,  FLOAT_BYTES,  "FLOAT",              tsCompressFloat,     tsDecompressFloat,     getStatics_f},
-  {TSDB_DATA_TYPE_DOUBLE,    6,  DOUBLE_BYTES, "DOUBLE",             tsCompressDouble,    tsDecompressDouble,    getStatics_d},
-  {TSDB_DATA_TYPE_BINARY,    6,  0,      "BINARY",             tsCompressString,    tsDecompressString,    getStatics_bin},
-  {TSDB_DATA_TYPE_TIMESTAMP, 9,  LONG_BYTES,   "TIMESTAMP",          tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
-  {TSDB_DATA_TYPE_NCHAR,     5,  8,      "NCHAR",              tsCompressString,    tsDecompressString,    getStatics_nchr},
-  {TSDB_DATA_TYPE_UTINYINT,  16, CHAR_BYTES,   "TINYINT UNSIGNED",   tsCompressTinyint,   tsDecompressTinyint,   getStatics_u8},
-  {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES,  "SMALLINT UNSIGNED",  tsCompressSmallint,  tsDecompressSmallint,  getStatics_u16},
-  {TSDB_DATA_TYPE_UINT,      12, INT_BYTES,    "INT UNSIGNED",       tsCompressInt,       tsDecompressInt,       getStatics_u32},
-  {TSDB_DATA_TYPE_UBIGINT,   15, LONG_BYTES,   "BIGINT UNSIGNED",    tsCompressBigint,    tsDecompressBigint,    getStatics_u64},
+  {TSDB_DATA_TYPE_NULL,      6,  1,            "NOTYPE",             0,          0,              NULL,                NULL,                  NULL},
+  {TSDB_DATA_TYPE_BOOL,      4,  CHAR_BYTES,   "BOOL",               false,      true,           tsCompressBool,      tsDecompressBool,      getStatics_bool},
+  {TSDB_DATA_TYPE_TINYINT,   7,  CHAR_BYTES,   "TINYINT",            INT8_MIN,   INT8_MAX,       tsCompressTinyint,   tsDecompressTinyint,   getStatics_i8},
+  {TSDB_DATA_TYPE_SMALLINT,  8,  SHORT_BYTES,  "SMALLINT",           INT16_MIN,  INT16_MAX,      tsCompressSmallint,  tsDecompressSmallint,  getStatics_i16},
+  {TSDB_DATA_TYPE_INT,       3,  INT_BYTES,    "INT",                INT32_MIN,  INT32_MAX,      tsCompressInt,       tsDecompressInt,       getStatics_i32},
+  {TSDB_DATA_TYPE_BIGINT,    6,  LONG_BYTES,   "BIGINT",             INT64_MIN,  INT64_MAX,      tsCompressBigint,    tsDecompressBigint,    getStatics_i64},
+  {TSDB_DATA_TYPE_FLOAT,     5,  FLOAT_BYTES,  "FLOAT",              0,          0,              tsCompressFloat,     tsDecompressFloat,     getStatics_f},
+  {TSDB_DATA_TYPE_DOUBLE,    6,  DOUBLE_BYTES, "DOUBLE",             0,          0,              tsCompressDouble,    tsDecompressDouble,    getStatics_d},
+  {TSDB_DATA_TYPE_BINARY,    6,  0,            "BINARY",             0,          0,              tsCompressString,    tsDecompressString,    getStatics_bin},
+  {TSDB_DATA_TYPE_TIMESTAMP, 9,  LONG_BYTES,   "TIMESTAMP",          INT64_MIN,  INT64_MAX,      tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
+  {TSDB_DATA_TYPE_NCHAR,     5,  8,            "NCHAR",              0,          0,              tsCompressString,    tsDecompressString,    getStatics_nchr},
+  {TSDB_DATA_TYPE_UTINYINT,  16, CHAR_BYTES,   "TINYINT UNSIGNED",   0,          UINT8_MAX,      tsCompressTinyint,   tsDecompressTinyint,   getStatics_u8},
+  {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES,  "SMALLINT UNSIGNED",  0,          UINT16_MAX,     tsCompressSmallint,  tsDecompressSmallint,  getStatics_u16},
+  {TSDB_DATA_TYPE_UINT,      12, INT_BYTES,    "INT UNSIGNED",       0,          UINT32_MAX,     tsCompressInt,       tsDecompressInt,       getStatics_u32},
+  {TSDB_DATA_TYPE_UBIGINT,   15, LONG_BYTES,   "BIGINT UNSIGNED",    0,          UINT64_MAX,     tsCompressBigint,    tsDecompressBigint,    getStatics_u64},
 };
 
 char tTokenTypeSwitcher[13] = {
@@ -405,6 +405,32 @@ char tTokenTypeSwitcher[13] = {
     TSDB_DATA_TYPE_NCHAR,   // TK_NCHAR
 };
 
+float floatMin = -FLT_MAX, floatMax = FLT_MAX;
+double doubleMin = -DBL_MAX, doubleMax = DBL_MAX;
+
+FORCE_INLINE void* getDataMin(int32_t type) {
+  switch (type) {
+    case TSDB_DATA_TYPE_FLOAT:
+      return &floatMin;
+    case TSDB_DATA_TYPE_DOUBLE:
+      return &doubleMin;
+    default:
+      return &tDataTypes[type].minValue;
+  }
+}
+
+FORCE_INLINE void* getDataMax(int32_t type) {
+  switch (type) {
+    case TSDB_DATA_TYPE_FLOAT:
+      return &floatMax;
+    case TSDB_DATA_TYPE_DOUBLE:
+      return &doubleMax;
+    default:
+      return &tDataTypes[type].maxValue;
+  }
+}
+
+
 bool isValidDataType(int32_t type) {
   return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT;
 }
@@ -566,6 +592,53 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
   }
 }
 
+void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
+  if (optr == TSDB_BINARY_OP_ADD) {
+    switch (type) {
+      case TSDB_DATA_TYPE_TINYINT:
+        *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_UTINYINT:
+        *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2);        
+        break;
+      case TSDB_DATA_TYPE_SMALLINT:
+        *((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_USMALLINT:
+        *((uint16_t *)dst) = GET_UINT16_VAL(s1) + GET_UINT16_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_INT:
+        *((int32_t *)dst) = GET_INT32_VAL(s1) + GET_INT32_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_UINT:
+        *((uint32_t *)dst) = GET_UINT32_VAL(s1) + GET_UINT32_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_BIGINT:
+        *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_UBIGINT:
+        *((uint64_t *)dst) = GET_UINT64_VAL(s1) + GET_UINT64_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_TIMESTAMP:
+        *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_FLOAT:
+        SET_FLOAT_VAL(dst, GET_FLOAT_VAL(s1) + GET_FLOAT_VAL(s2));
+        break;
+      case TSDB_DATA_TYPE_DOUBLE:
+        SET_DOUBLE_VAL(dst, GET_DOUBLE_VAL(s1) + GET_DOUBLE_VAL(s2));
+        break;
+      default: {
+        assert(0);
+        break;
+      }
+    }
+  } else {
+    assert(0);
+  }
+}
+
+
 void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) {
   switch (type) {
     case TSDB_DATA_TYPE_INT:

src/common/src/tvariant.c

@@ -23,6 +23,13 @@
 #include "tutil.h"
 #include "tvariant.h"
 
+#define SET_EXT_INFO(converted, res, minv, maxv, exti) do {                                                       \
+                                                        if (converted == NULL || exti == NULL || *converted == false) { break; }  \
+                                                        if ((res) < (minv)) { *exti = -1; break; }                                  \
+                                                        if ((res) > (maxv)) { *exti = 1; break; }                                   \
+                                                        assert(0);                                                                  \
+                                                       } while (0)
+
 void tVariantCreate(tVariant *pVar, SStrToken *token) {
   int32_t ret = 0;
   int32_t type = token->type;
@@ -184,7 +191,7 @@ void tVariantDestroy(tVariant *pVar) {
   }
 
   // NOTE: this is only for string array
-  if (pVar->nType == TSDB_DATA_TYPE_ARRAY) {
+  if (pVar->nType == TSDB_DATA_TYPE_POINTER_ARRAY) {
     size_t num = taosArrayGetSize(pVar->arr);
     for(size_t i = 0; i < num; i++) {
       void* p = taosArrayGetP(pVar->arr, i);
@@ -192,6 +199,9 @@ void tVariantDestroy(tVariant *pVar) {
     }
     taosArrayDestroy(pVar->arr);
     pVar->arr = NULL;
+  } else if (pVar->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
+    taosArrayDestroy(pVar->arr);
+    pVar->arr = NULL;
   }
 }
 
@@ -220,7 +230,7 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
 
   if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
     pDst->i64 = pSrc->i64;
-  } else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) {  // this is only for string array
+  } else if (pSrc->nType == TSDB_DATA_TYPE_POINTER_ARRAY) {  // this is only for string array
     size_t num = taosArrayGetSize(pSrc->arr);
     pDst->arr = taosArrayInit(num, sizeof(char*));
     for(size_t i = 0; i < num; i++) {
@@ -228,9 +238,18 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
       char* n = strdup(p);
       taosArrayPush(pDst->arr, &n);
     }
+  } else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
+      size_t num = taosArrayGetSize(pSrc->arr);
+      pDst->arr = taosArrayInit(num, sizeof(int64_t));
+      pDst->nLen = pSrc->nLen;
+      assert(pSrc->nLen == num);
+      for(size_t i = 0; i < num; i++) {
+        int64_t *p = taosArrayGet(pSrc->arr, i);
+        taosArrayPush(pDst->arr, p);
+      }
   }
 
-  if (pDst->nType != TSDB_DATA_TYPE_ARRAY) {
+  if (pDst->nType != TSDB_DATA_TYPE_POINTER_ARRAY && pDst->nType != TSDB_DATA_TYPE_VALUE_ARRAY) {
     pDst->nLen = tDataTypes[pDst->nType].bytes;
   }
 }
@@ -450,7 +469,7 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val
   return 0;
 }
 
-static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr) {
+static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr, bool *converted) {
   if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
     setNull((char *)result, type, tDataTypes[type].bytes);
     return 0;
@@ -540,6 +559,10 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
     }
   }
 
+  if (converted) {
+    *converted = true;
+  }
+
   bool code = false;
 
   uint64_t ui = 0;
@@ -602,6 +625,18 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) {
  * to column type defined in schema
  */
 int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) {
+  return tVariantDumpEx(pVariant, payload, type, includeLengthPrefix, NULL, NULL);
+}
+
+/*
+ * transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType
+ * to column type defined in schema
+ */
+int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo) {
+  if (converted) {
+    *converted = false;
+  }
+  
   if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) {
     return -1;
   }
@@ -620,7 +655,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
     
     case TSDB_DATA_TYPE_TINYINT: {
-      if (convertToInteger(pVariant, &result, type, true, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, INT8_MIN + 1, INT8_MAX, extInfo);
         return -1;
       }
       *((int8_t *)payload) = (int8_t) result;
@@ -628,7 +664,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
 
     case TSDB_DATA_TYPE_UTINYINT: {
-      if (convertToInteger(pVariant, &result, type, false, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {        
+        SET_EXT_INFO(converted, result, 0, UINT8_MAX - 1, extInfo);
         return -1;
       }
       *((uint8_t *)payload) = (uint8_t) result;
@@ -636,7 +673,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
     
     case TSDB_DATA_TYPE_SMALLINT: {
-      if (convertToInteger(pVariant, &result, type, true, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, INT16_MIN + 1, INT16_MAX, extInfo);
         return -1;
       }
       *((int16_t *)payload) = (int16_t)result;
@@ -644,7 +682,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
 
     case TSDB_DATA_TYPE_USMALLINT: {
-      if (convertToInteger(pVariant, &result, type, false, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, 0, UINT16_MAX - 1, extInfo);
         return -1;
       }
       *((uint16_t *)payload) = (uint16_t)result;
@@ -652,7 +691,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
     
     case TSDB_DATA_TYPE_INT: {
-      if (convertToInteger(pVariant, &result, type, true, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, INT32_MIN + 1, INT32_MAX, extInfo);
         return -1;
       }
       *((int32_t *)payload) = (int32_t)result;
@@ -660,7 +700,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
 
     case TSDB_DATA_TYPE_UINT: {
-      if (convertToInteger(pVariant, &result, type, false, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, 0, UINT32_MAX - 1, extInfo);
         return -1;
       }
       *((uint32_t *)payload) = (uint32_t)result;
@@ -668,7 +709,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
     
     case TSDB_DATA_TYPE_BIGINT: {
-      if (convertToInteger(pVariant, &result, type, true, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
+        SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo);
         return -1;
       }
       *((int64_t *)payload) = (int64_t)result;
@@ -676,7 +718,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
 
     case TSDB_DATA_TYPE_UBIGINT: {
-      if (convertToInteger(pVariant, &result, type, false, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
+        SET_EXT_INFO(converted, (uint64_t)result, 0, UINT64_MAX - 1, extInfo);
         return -1;
       }
       *((uint64_t *)payload) = (uint64_t)result;
@@ -696,11 +739,37 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
             return -1;
           }
 
+          if (converted) {
+            *converted = true;
+          }
+          
+          if (value > FLT_MAX || value < -FLT_MAX) {
+            SET_EXT_INFO(converted, value, -FLT_MAX, FLT_MAX, extInfo);
+            return -1;
+          }
           SET_FLOAT_VAL(payload, value);
         }
       } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
+        if (converted) {
+          *converted = true;
+        }
+        
+        if (pVariant->i64 > FLT_MAX || pVariant->i64 < -FLT_MAX) {          
+          SET_EXT_INFO(converted, pVariant->i64, -FLT_MAX, FLT_MAX, extInfo);
+          return -1;
+        }
+
         SET_FLOAT_VAL(payload, pVariant->i64);
       } else if (IS_FLOAT_TYPE(pVariant->nType)) {
+        if (converted) {
+          *converted = true;
+        }
+        
+        if (pVariant->dKey > FLT_MAX || pVariant->dKey < -FLT_MAX) {          
+          SET_EXT_INFO(converted, pVariant->dKey, -FLT_MAX, FLT_MAX, extInfo);
+          return -1;
+        }
+      
         SET_FLOAT_VAL(payload, pVariant->dKey);
       } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
         *((uint32_t *)payload) = TSDB_DATA_FLOAT_NULL;
@@ -824,6 +893,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
   return 0;
 }
 
+
 /*
  * In variant, bool/smallint/tinyint/int/bigint share the same attribution of
  * structure, also ignore the convert the type required
@@ -848,7 +918,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) {
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_SMALLINT: {
-      convertToInteger(pVariant, &(pVariant->i64), type, true, true);
+      convertToInteger(pVariant, &(pVariant->i64), type, true, true, NULL);
       pVariant->nType = TSDB_DATA_TYPE_BIGINT;
       break;
     }

src/connector/jdbc/pom.xml

@@ -113,7 +113,6 @@
                     </includes>
                     <excludes>
                         <exclude>**/AppMemoryLeakTest.java</exclude>
-                        <exclude>**/AuthenticationTest.java</exclude>
                         <exclude>**/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java</exclude>
                         <exclude>**/DatetimeBefore1970Test.java</exclude>
                         <exclude>**/FailOverTest.java</exclude>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -14,6 +14,8 @@
  *****************************************************************************/
 package com.taosdata.jdbc;
 
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.sql.*;
 import java.util.*;
 import java.util.logging.Logger;
@@ -127,6 +129,11 @@ public class TSDBDriver extends AbstractDriver {
             return null;
         }
 
+        if (!props.containsKey(TSDBDriver.PROPERTY_KEY_USER))
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED);
+        if (!props.containsKey(TSDBDriver.PROPERTY_KEY_PASSWORD))
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED);
+
         try {
             TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE),
                     (String) props.get(PROPERTY_KEY_CHARSET), (String) props.get(PROPERTY_KEY_TIME_ZONE));

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java

@@ -33,6 +33,8 @@ public class TSDBError {
         TSDBErrorMap.put(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE, "numeric value out of range");
         TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE, "unknown taos type in tdengine");
         TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PRECISION, "unknown timestamp precision");
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, "user is required");
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, "password is required");
 
         TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN, "unknown error");
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java

@@ -29,6 +29,9 @@ public class TSDBErrorNumbers {
     public static final int ERROR_UNKNOWN_TIMESTAMP_PRECISION = 0x2316;     // unknown timestamp precision
     public static final int ERROR_RESTFul_Client_Protocol_Exception = 0x2317;
     public static final int ERROR_RESTFul_Client_IOException = 0x2318;
+    public static final int ERROR_USER_IS_REQUIRED = 0x2319;        // user is required
+    public static final int ERROR_PASSWORD_IS_REQUIRED = 0x231a;    // password is required
+
 
     public static final int ERROR_UNKNOWN = 0x2350;    //unknown error
 
@@ -67,6 +70,8 @@ public class TSDBErrorNumbers {
         errorNumbers.add(ERROR_UNKNOWN_TAOS_TYPE);
         errorNumbers.add(ERROR_UNKNOWN_TIMESTAMP_PRECISION);
         errorNumbers.add(ERROR_RESTFul_Client_IOException);
+        errorNumbers.add(ERROR_USER_IS_REQUIRED);
+        errorNumbers.add(ERROR_PASSWORD_IS_REQUIRED);
 
         errorNumbers.add(ERROR_RESTFul_Client_Protocol_Exception);
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -36,7 +36,6 @@ public class TSDBJNIConnector {
 
     static {
         System.loadLibrary("taos");
-        System.out.println("java.library.path:" + System.getProperty("java.library.path"));
     }
 
     public boolean isClosed() {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java

@@ -7,6 +7,7 @@ import com.taosdata.jdbc.utils.HttpClientPoolUtil;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.sql.*;
 import java.util.Properties;
 import java.util.logging.Logger;
@@ -40,8 +41,13 @@ public class RestfulDriver extends AbstractDriver {
 
         String loginUrl = "http://" + host + ":" + port + "/rest/login/" + props.getProperty(TSDBDriver.PROPERTY_KEY_USER) + "/" + props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD) + "";
         try {
-            String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), "UTF-8");
-            String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), "UTF-8");
+            if (!props.containsKey(TSDBDriver.PROPERTY_KEY_USER))
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED);
+            if (!props.containsKey(TSDBDriver.PROPERTY_KEY_PASSWORD))
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED);
+
+            String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), StandardCharsets.UTF_8.displayName());
+            String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), StandardCharsets.UTF_8.displayName());
             loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + user + "/" + password + "";
         } catch (UnsupportedEncodingException e) {
             e.printStackTrace();

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java

@@ -7,6 +7,7 @@ import com.taosdata.jdbc.AbstractStatement;
 import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.TSDBError;
 import com.taosdata.jdbc.TSDBErrorNumbers;
+import com.taosdata.jdbc.enums.TimestampFormat;
 import com.taosdata.jdbc.utils.HttpClientPoolUtil;
 import com.taosdata.jdbc.utils.SqlSyntaxValidator;
 
@@ -45,9 +46,7 @@ public class RestfulStatement extends AbstractStatement {
         if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql))
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql);
 
-        final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
-
-        return executeOneUpdate(url, sql);
+        return executeOneUpdate(sql);
     }
 
     @Override
@@ -62,34 +61,25 @@ public class RestfulStatement extends AbstractStatement {
     public boolean execute(String sql) throws SQLException {
         if (isClosed())
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
-        if (!SqlSyntaxValidator.isValidForExecute(sql))
-            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE, "not a valid sql for execute: " + sql);
 
         //如果执行了use操作应该将当前Statement的catalog设置为新的database
         boolean result = true;
-        String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
-        if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("TIMESTAMP")) {
-            url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt";
-        }
-        if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("UTC")) {
-            url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc";
-        }
 
         if (SqlSyntaxValidator.isUseSql(sql)) {
-            HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
+            HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
             this.database = sql.trim().replace("use", "").trim();
             this.conn.setCatalog(this.database);
             result = false;
         } else if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) {
             executeOneQuery(sql);
         } else if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) {
-            executeOneUpdate(url, sql);
+            executeOneUpdate(sql);
             result = false;
         } else {
             if (SqlSyntaxValidator.isValidForExecuteQuery(sql)) {
-                executeQuery(sql);
+                executeOneQuery(sql);
             } else {
-                executeUpdate(sql);
+                executeOneUpdate(sql);
                 result = false;
             }
         }
@@ -97,19 +87,25 @@ public class RestfulStatement extends AbstractStatement {
         return result;
     }
 
-    private ResultSet executeOneQuery(String sql) throws SQLException {
-        if (!SqlSyntaxValidator.isValidForExecuteQuery(sql))
-            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql);
-
-        // row data
-        String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
-        String timestampFormat = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT);
-        if ("TIMESTAMP".equalsIgnoreCase(timestampFormat))
+    private String getUrl() throws SQLException {
+        TimestampFormat timestampFormat = TimestampFormat.valueOf(conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).trim().toUpperCase());
+        String url;
+        switch (timestampFormat) {
+            case TIMESTAMP:
                 url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt";
-        if ("UTC".equalsIgnoreCase(timestampFormat))
+                break;
+            case UTC:
                 url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc";
+                break;
+            default:
+                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
+        }
+        return url;
+    }
 
-        String result = HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
+    private ResultSet executeOneQuery(String sql) throws SQLException {
+        // row data
+        String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
         JSONObject resultJson = JSON.parseObject(result);
         if (resultJson.getString("status").equals("error")) {
             throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc"));
@@ -119,11 +115,8 @@ public class RestfulStatement extends AbstractStatement {
         return resultSet;
     }
 
-    private int executeOneUpdate(String url, String sql) throws SQLException {
-        if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql))
-            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql);
-
-        String result = HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
+    private int executeOneUpdate(String sql) throws SQLException {
+        String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
         JSONObject jsonObject = JSON.parseObject(result);
         if (jsonObject.getString("status").equals("error")) {
             throw TSDBError.createSQLException(jsonObject.getInteger("code"), jsonObject.getString("desc"));
@@ -134,7 +127,7 @@ public class RestfulStatement extends AbstractStatement {
     }
 
     private int getAffectedRows(JSONObject jsonObject) throws SQLException {
-        // create ... SQLs should return 0 , and Restful result is this:
+        // create ... SQLs should return 0 , and Restful result like this:
         // {"status": "succ", "head": ["affected_rows"], "data": [[0]], "rows": 1}
         JSONArray head = jsonObject.getJSONArray("head");
         if (head.size() != 1 || !"affected_rows".equals(head.getString(0)))

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java

@@ -16,8 +16,7 @@ package com.taosdata.jdbc.utils;
 
 public class SqlSyntaxValidator {
 
-    private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe", "reset"};
-    private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set"};
+    private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set", "reset"};
     private static final String[] querySQL = {"select", "show", "describe"};
 
     private static final String[] databaseUnspecifiedShow = {"databases", "dnodes", "mnodes", "variables"};
@@ -38,14 +37,6 @@ public class SqlSyntaxValidator {
         return false;
     }
 
-    public static boolean isValidForExecute(String sql) {
-        for (String prefix : SQL) {
-            if (sql.trim().toLowerCase().startsWith(prefix))
-                return true;
-        }
-        return false;
-    }
-
     public static boolean isDatabaseUnspecifiedQuery(String sql) {
         for (String databaseObj : databaseUnspecifiedShow) {
             if (sql.trim().toLowerCase().matches("show\\s+" + databaseObj + ".*"))
@@ -63,9 +54,5 @@ public class SqlSyntaxValidator {
         return sql.trim().toLowerCase().startsWith("use");
     }
 
-    public static boolean isSelectSql(String sql) {
-        return sql.trim().toLowerCase().startsWith("select");
-    }
-
 
 }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java

@@ -69,6 +69,8 @@ public class SubscribeTest {
     @Before
     public void createDatabase() throws SQLException {
         Properties properties = new Properties();
+        properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+        properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java

 package com.taosdata.jdbc.cases;
 
+import com.taosdata.jdbc.TSDBErrorNumbers;
+import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.sql.*;
@@ -12,6 +15,47 @@ public class AuthenticationTest {
     private static final String password = "taos?data";
     private Connection conn;
 
+    @Test
+    public void connectWithoutUserByJni() {
+        try {
+            DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?");
+        } catch (SQLException e) {
+            Assert.assertEquals(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, e.getErrorCode());
+            Assert.assertEquals("ERROR (2319): user is required", e.getMessage());
+        }
+    }
+
+    @Test
+    public void connectWithoutUserByRestful() {
+        try {
+            DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?");
+        } catch (SQLException e) {
+            Assert.assertEquals(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, e.getErrorCode());
+            Assert.assertEquals("ERROR (2319): user is required", e.getMessage());
+        }
+    }
+
+    @Test
+    public void connectWithoutPasswordByJni() {
+        try {
+            DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root");
+        } catch (SQLException e) {
+            Assert.assertEquals(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, e.getErrorCode());
+            Assert.assertEquals("ERROR (231a): password is required", e.getMessage());
+        }
+    }
+
+    @Test
+    public void connectWithoutPasswordByRestful() {
+        try {
+            DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root");
+        } catch (SQLException e) {
+            Assert.assertEquals(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, e.getErrorCode());
+            Assert.assertEquals("ERROR (231a): password is required", e.getMessage());
+        }
+    }
+
+    @Ignore
     @Test
     public void test() {
         // change password

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java

@@ -29,6 +29,8 @@ public class BatchInsertTest {
     public void before() {
         try {
             Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java

@@ -21,6 +21,8 @@ public class ImportTest {
     public static void before() {
         try {
             Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java

@@ -270,6 +270,41 @@ public class InsertSpecialCharacterJniTest {
         }
     }
 
+    @Ignore
+    @Test
+    public void testSingleQuotaEscape() throws SQLException {
+        final long now = System.currentTimeMillis();
+        final String sql = "insert into t? using ? tags(?) values(?, ?, ?) t? using " + tbname2 + " tags(?) values(?,?,?) ";
+        try (PreparedStatement pstmt = conn.prepareStatement(sql)) {
+            // t1
+            pstmt.setInt(1, 1);
+            pstmt.setString(2, tbname2);
+            pstmt.setString(3, special_character_str_5);
+            pstmt.setTimestamp(4, new Timestamp(now));
+            pstmt.setBytes(5, special_character_str_5.getBytes());
+            // t2
+            pstmt.setInt(7, 2);
+            pstmt.setString(8, special_character_str_5);
+            pstmt.setTimestamp(9, new Timestamp(now));
+            pstmt.setString(11, special_character_str_5);
+
+            int ret = pstmt.executeUpdate();
+            Assert.assertEquals(2, ret);
+        }
+
+        String query = "select * from ?.t? where ? < ? and ts >= ? and f1 is not null";
+        try (PreparedStatement pstmt = conn.prepareStatement(query)) {
+            pstmt.setString(1, dbName);
+            pstmt.setInt(2, 1);
+            pstmt.setString(3, "ts");
+            pstmt.setTimestamp(4, new Timestamp(System.currentTimeMillis()));
+            pstmt.setTimestamp(5, new Timestamp(0));
+
+            ResultSet rs = pstmt.executeQuery();
+            Assert.assertNotNull(rs);
+        }
+    }
+
     @Test
     public void testCase10() throws SQLException {
         final long now = System.currentTimeMillis();
@@ -293,13 +328,12 @@ public class InsertSpecialCharacterJniTest {
             Assert.assertEquals(2, ret);
         }
         //query t1
-        String query = "select * from ?.t? where ts < ? and ts >= ? and ? is not null";
+        String query = "select * from ?.t? where ts < ? and ts >= ? and f1 is not null";
         try (PreparedStatement pstmt = conn.prepareStatement(query)) {
             pstmt.setString(1, dbName);
             pstmt.setInt(2, 1);
             pstmt.setTimestamp(3, new Timestamp(System.currentTimeMillis()));
             pstmt.setTimestamp(4, new Timestamp(0));
-            pstmt.setString(5, "f1");
 
             ResultSet rs = pstmt.executeQuery();
             rs.next();
@@ -311,12 +345,11 @@ public class InsertSpecialCharacterJniTest {
             Assert.assertNull(f2);
         }
         // query t2
-        query = "select * from t? where ts < ? and ts >= ? and ? is not null";
+        query = "select * from t? where ts < ? and ts >= ? and f2 is not null";
         try (PreparedStatement pstmt = conn.prepareStatement(query)) {
             pstmt.setInt(1, 2);
             pstmt.setTimestamp(2, new Timestamp(System.currentTimeMillis()));
             pstmt.setTimestamp(3, new Timestamp(0));
-            pstmt.setString(4, "f2");
 
             ResultSet rs = pstmt.executeQuery();
             rs.next();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java

@@ -293,13 +293,12 @@ public class InsertSpecialCharacterRestfulTest {
             Assert.assertEquals(2, ret);
         }
         //query t1
-        String query = "select * from ?.t? where ts < ? and ts >= ? and ? is not null";
+        String query = "select * from ?.t? where ts < ? and ts >= ? and f1 is not null";
         try (PreparedStatement pstmt = conn.prepareStatement(query)) {
             pstmt.setString(1, dbName);
             pstmt.setInt(2, 1);
             pstmt.setTimestamp(3, new Timestamp(System.currentTimeMillis()));
             pstmt.setTimestamp(4, new Timestamp(0));
-            pstmt.setString(5, "f1");
 
             ResultSet rs = pstmt.executeQuery();
             rs.next();
@@ -311,12 +310,11 @@ public class InsertSpecialCharacterRestfulTest {
             Assert.assertNull(f2);
         }
         // query t2
-        query = "select * from t? where ts < ? and ts >= ? and ? is not null";
+        query = "select * from t? where ts < ? and ts >= ? and f2 is not null";
         try (PreparedStatement pstmt = conn.prepareStatement(query)) {
             pstmt.setInt(1, 2);
             pstmt.setTimestamp(2, new Timestamp(System.currentTimeMillis()));
             pstmt.setTimestamp(3, new Timestamp(0));
-            pstmt.setString(4, "f2");
 
             ResultSet rs = pstmt.executeQuery();
             rs.next();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java

@@ -22,6 +22,8 @@ public class QueryDataTest {
     public void createDatabase() {
         try {
             Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java

 package com.taosdata.jdbc.cases;
 
-import com.taosdata.jdbc.TSDBDriver;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 
-import java.sql.*;
-import java.util.Properties;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 public class ResetQueryCacheTest {
 
-    static Connection connection;
-    static Statement statement;
-    static String host = "127.0.0.1";
-
-    @Before
-    public void init() {
-        try {
-            Properties properties = new Properties();
-            properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
-            properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
-            properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-            connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
-            statement = connection.createStatement();
-        } catch (SQLException e) {
-            return;
-        }
+    @Test
+    public void jni() throws SQLException {
+        // given
+        Connection connection = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:0/?user=root&password=taosdata&timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8");
+        Statement statement = connection.createStatement();
+
+        // when
+        boolean execute = statement.execute("reset query cache");
+
+        // then
+        assertFalse(execute);
+        assertEquals(0, statement.getUpdateCount());
+
+        statement.close();
+        connection.close();
     }
 
     @Test
-    public void testResetQueryCache() throws SQLException {
-        String resetSql = "reset query cache";
-        statement.execute(resetSql);
-    }
+    public void restful() throws SQLException {
+        // given
+        Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://127.0.0.1:6041/?user=root&password=taosdata&timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8");
+        Statement statement = connection.createStatement();
+
+        // when
+        boolean execute = statement.execute("reset query cache");
+
+        // then
+        assertFalse(execute);
+        assertEquals(0, statement.getUpdateCount());
 
-    @After
-    public void close() {
-        try {
-            if (statement != null)
         statement.close();
-            if (connection != null)
         connection.close();
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
     }
 
 }
\ No newline at end of file