Merge branch 'develop' into test/skci_dev

Jenkinsfile

@@ -4,7 +4,9 @@ properties([pipelineTriggers([githubPush()])])
 node {
     git url: 'https://github.com/taosdata/TDengine.git'
 }
+
 def skipbuild=0
+
 def abortPreviousBuilds() {
   def currentJobName = env.JOB_NAME
   def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@@ -146,6 +148,7 @@ pipeline {
             skipbuild='2'     
             skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true)
             println skipbuild
+
           }
           sh'''
           rm -rf ${WORKSPACE}.tes
@@ -160,6 +163,7 @@ pipeline {
               changeRequest()
               expression{
                 return skipbuild.trim() == '2'
+
               }
            }
           }

src/client/inc/tscUtil.h

@@ -61,6 +61,7 @@ typedef struct SJoinSupporter {
   uint64_t        uid;            // query table uid
   SArray*         colList;        // previous query information, no need to use this attribute, and the corresponding attribution
   SArray*         exprList;
+  SArray*         colCond;
   SFieldInfo      fieldsInfo;
   STagCond        tagCond;
   SGroupbyExpr    groupInfo;       // group by info
@@ -244,8 +245,9 @@ SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid);
 void   tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw);
 
 int32_t tscTagCondCopy(STagCond* dest, const STagCond* src);
+int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx);
 void tscTagCondRelease(STagCond* pCond);
-
+void tscColCondRelease(SArray** pCond);
 void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo);
 
 bool tscShouldBeFreed(SSqlObj* pSql);
@@ -340,7 +342,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild);
 uint32_t tscGetTableMetaSize(STableMeta* pTableMeta);
 CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta);
 uint32_t tscGetTableMetaMaxSize();
-int32_t tscCreateTableMetaFromSTableMeta(STableMeta** pChild, const char* name, size_t *tableMetaCapacity);
+int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity, STableMeta **ppStable);
 STableMeta* tscTableMetaDup(STableMeta* pTableMeta);
 SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo);
 
@@ -355,6 +357,7 @@ char* strdup_throw(const char* str);
 
 bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src);
 SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg);
+STblCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx);
 
 void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id);
 

src/client/src/tscPrepare.c

@@ -299,7 +299,7 @@ static int fillColumnsNull(STableDataBlocks* pBlock, int32_t rowNum) {
   SSchema *schema = (SSchema*)pBlock->pTableMeta->schema;
 
   for (int32_t i = 0; i < spd->numOfCols; ++i) {
-    if (!spd->cols[i].hasVal) {  // current column do not have any value to insert, set it to null
+    if (spd->cols[i].valStat == VAL_STAT_NONE) {  // current column do not have any value to insert, set it to null
       for (int32_t n = 0; n < rowNum; ++n) {
         char *ptr = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * n + offset;
 

src/client/src/tscServer.c

@@ -684,7 +684,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
   SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
 
   int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
-  int32_t srcColFilterSize = tscGetColFilterSerializeLen(pQueryInfo);
+  int32_t srcColFilterSize = 0;
   int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo);
 
   size_t  numOfExprs = tscNumOfExprs(pQueryInfo);
@@ -695,6 +695,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
 
   int32_t tableSerialize = 0;
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;  
   if (pTableMetaInfo->pVgroupTables != NULL) {
     size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables);
 
@@ -707,8 +708,15 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
     tableSerialize = totalTables * sizeof(STableIdInfo);
   }
 
-  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize +
-         exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
+  if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) {
+    STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0);
+    if (pCond != NULL && pCond->cond != NULL) {
+      srcColFilterSize = pCond->len;
+    }
+  }
+
+  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + exprSize + tsBufSize +
+         tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
 }
 
 static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg,
@@ -966,10 +974,21 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     pQueryMsg->tableCols[i].colId = htons(pCol->colId);
     pQueryMsg->tableCols[i].bytes = htons(pCol->bytes);
     pQueryMsg->tableCols[i].type  = htons(pCol->type);
-    pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters);
+    //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters);
+    pQueryMsg->tableCols[i].flist.numOfFilters = 0;
 
     // append the filter information after the basic column information
-    serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg);
+    //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg);
+  }
+
+  if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) {
+    STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0);
+    if (pCond != NULL && pCond->cond != NULL) {
+      pQueryMsg->colCondLen = htons(pCond->len);
+      memcpy(pMsg, pCond->cond, pCond->len);
+
+      pMsg += pCond->len;
+    }
   }
 
   for (int32_t i = 0; i < query.numOfOutput; ++i) {
@@ -1044,7 +1063,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
     SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid);
     if (pCond != NULL && pCond->cond != NULL) {
-      pQueryMsg->tagCondLen = htonl(pCond->len);
+      pQueryMsg->tagCondLen = htons(pCond->len);
       memcpy(pMsg, pCond->cond, pCond->len);
 
       pMsg += pCond->len;
@@ -2853,18 +2872,19 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool
   tNameExtractFullName(&pTableMetaInfo->name, name);
 
   size_t len = strlen(name);
-  if (pTableMetaInfo->tableMetaCapacity != 0) {
-    if (pTableMetaInfo->pTableMeta != NULL) {
-      memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity);
-    } 
+   // just make runtime happy
+  if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) {
+    memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity);
   } 
   taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity);
-
-  STableMeta* pMeta = pTableMetaInfo->pTableMeta;
+  
+  STableMeta* pMeta   = pTableMetaInfo->pTableMeta;
+  STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf);
   if (pMeta && pMeta->id.uid > 0) {
     // in case of child table, here only get the
     if (pMeta->tableType == TSDB_CHILD_TABLE) {
-      int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity);
+      int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta));
+      pSql->pBuf   = (void *)(pSTMeta); 
       if (code != TSDB_CODE_SUCCESS) {
         return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate);
       }

src/client/src/tscSubquery.c

@@ -796,6 +796,7 @@ static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj*
   STimeWindow window = pQueryInfo->window;
   tscInitQueryInfo(pQueryInfo);
 
+  pQueryInfo->colCond = pSupporter->colCond;
   pQueryInfo->window = window;
   TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY);
   TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
@@ -1883,6 +1884,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
     if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag
       SColumnIndex colIndex = {0};
 
+      pSupporter->colCond = pNewQueryInfo->colCond;
+      pNewQueryInfo->colCond = NULL;
+
       STagCond* pTagCond = &pSupporter->tagCond;
       assert(pTagCond->joinInfo.hasJoin);
 
@@ -2319,6 +2323,11 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) {
     goto _error;
   }
 
+  if (tscColCondCopy(&pNewQueryInfo->colCond, pQueryInfo->colCond, pTableMetaInfo->pTableMeta->id.uid, 0) != 0) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto _error;
+  }
+
   pNewQueryInfo->window   = pQueryInfo->window;
   pNewQueryInfo->interval = pQueryInfo->interval;
   pNewQueryInfo->sessionWindow = pQueryInfo->sessionWindow;

src/common/inc/tdataformat.h

@@ -186,6 +186,7 @@ typedef void *SDataRow;
 #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
 
 #define dataRowLen(r) (*(TDRowLenT *)(r))  // 0~65535
+#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r))
 #define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
 #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
 #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
@@ -201,14 +202,18 @@ void     tdFreeDataRow(SDataRow row);
 void     tdInitDataRow(SDataRow row, STSchema *pSchema);
 SDataRow tdDataRowDup(SDataRow row);
 
+
 // offset here not include dataRow header length
-static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
+static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type,
+                                           int32_t offset) {
   ASSERT(value != NULL);
   int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
 
   if (IS_VAR_DATA_TYPE(type)) {
     *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
-    memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
+    if (isCopyVarData) {
+      memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
+    }
     dataRowLen(row) += varDataTLen(value);
   } else {
     if (offset == 0) {
@@ -223,6 +228,12 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t t
   return 0;
 }
 
+
+// offset here not include dataRow header length
+static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
+  return tdAppendDataColVal(row, value, true, type, offset);
+}
+
 // NOTE: offset here including the header size
 static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
   if (IS_VAR_DATA_TYPE(type)) {
@@ -328,11 +339,10 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 int tdAllocMemForCol(SDataCol *pCol, int maxPoints);
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints);
-void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
+int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
 void dataColSetOffset(SDataCol *pCol, int nEle);
 
 bool isNEleNull(SDataCol *pCol, int nEle);
-void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
 
 // Get the data pointer from a column-wised data
 static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) {
@@ -357,13 +367,11 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
 }
 
 typedef struct {
-  int maxRowSize;
-  int maxCols;    // max number of columns
-  int maxPoints;  // max number of points
-
-  int       numOfRows;
-  int       numOfCols;  // Total number of cols
-  int       sversion;   // TODO: set sversion
+  int      maxCols;    // max number of columns
+  int      maxPoints;  // max number of points
+  int      numOfRows;
+  int      numOfCols;  // Total number of cols
+  int      sversion;   // TODO: set sversion
   SDataCol *cols;
 } SDataCols;
 
@@ -407,7 +415,7 @@ static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) {
   }
 }
 
-SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
+SDataCols *tdNewDataCols(int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
 int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
@@ -475,9 +483,10 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
 }
 
 // offset here not include kvRow header length
-static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t *offset) {
+static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
+                                         int32_t offset) {
   ASSERT(value != NULL);
-  int32_t  toffset = *offset + TD_KV_ROW_HEAD_SIZE;
+  int32_t  toffset = offset + TD_KV_ROW_HEAD_SIZE;
   SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset);
   char *   ptr = (char *)POINTER_SHIFT(row, kvRowLen(row));
 
@@ -485,10 +494,12 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t
   pColIdx->offset = kvRowLen(row);  // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE
 
   if (IS_VAR_DATA_TYPE(type)) {
-    memcpy(ptr, value, varDataTLen(value));
+    if (isCopyValData) {
+      memcpy(ptr, value, varDataTLen(value));
+    }
     kvRowLen(row) += varDataTLen(value);
   } else {
-    if (*offset == 0) {
+    if (offset == 0) {
       ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
       TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
       memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]);
@@ -497,7 +508,6 @@ static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t
     }
     kvRowLen(row) += TYPE_BYTES[type];
   }
-  *offset += sizeof(SColIdx);
 
   return 0;
 }
@@ -592,12 +602,24 @@ typedef void *SMemRow;
 #define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE)
 #define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE)
 
-#define SMEM_ROW_DATA 0U  // SDataRow
-#define SMEM_ROW_KV 1U    // SKVRow
+#define SMEM_ROW_DATA 0x0U      // SDataRow
+#define SMEM_ROW_KV 0x01U       // SKVRow
+#define SMEM_ROW_CONVERT 0x80U  // SMemRow convert flag
+
+#define KVRatioKV (0.2f)  // all bool
+#define KVRatioPredict (0.4f)
+#define KVRatioData (0.75f)  // all bigint
+#define KVRatioConvert (0.9f)
 
-#define memRowType(r) (*(uint8_t *)(r))
+#define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
+
+#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t))  // set the total byte in case of dirty memory
+#define memRowSetConvert(r) ((*(uint8_t *)(r)) = (((*(uint8_t *)(r)) & 0x7F) | SMEM_ROW_CONVERT))  // highest bit
+#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
 #define isDataRow(r) (SMEM_ROW_DATA == memRowType(r))
+#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
 #define isKvRow(r) (SMEM_ROW_KV == memRowType(r))
+#define isNeedConvertRow(r) (((*(uint8_t *)(r)) & 0x80) == SMEM_ROW_CONVERT)
 
 #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)  // section after flag
 #define memRowKvBody(r) \
@@ -614,6 +636,14 @@ typedef void *SMemRow;
 #define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
 #define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r))  // using uint32_t/int32_t to store the TLen
 
+static FORCE_INLINE char *memRowEnd(SMemRow row) {
+  if (isDataRow(row)) {
+    return (char *)dataRowEnd(memRowDataBody(row));
+  } else {
+    return (char *)kvRowEnd(memRowKvBody(row));
+  }
+}
+
 #define memRowDataVersion(r) dataRowVersion(memRowDataBody(r))
 #define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
 #define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r))  // schema version
@@ -631,7 +661,6 @@ typedef void *SMemRow;
     }                                       \
   } while (0)
 
-#define memRowSetType(r, t) (memRowType(r) = (t))
 #define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l))
 #define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v))
 #define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r))
@@ -664,12 +693,12 @@ static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_
   }
 }
 
-static FORCE_INLINE int tdAppendMemColVal(SMemRow row, const void *value, int16_t colId, int8_t type, int32_t offset,
-                                          int32_t *kvOffset) {
+static FORCE_INLINE int tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
+                                             int8_t type, int32_t offset) {
   if (isDataRow(row)) {
-    tdAppendColVal(memRowDataBody(row), value, type, offset);
+    tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset);
   } else {
-    tdAppendKvColVal(memRowKvBody(row), value, colId, type, kvOffset);
+    tdAppendKvColVal(memRowKvBody(row), value, isCopyVarData, colId, type, offset);
   }
   return 0;
 }
@@ -691,6 +720,30 @@ static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value
   return len;
 }
 
+/**
+ * 1. calculate the delta of AllNullLen for SDataRow.
+ * 2. calculate the real len for SKVRow.
+ */
+static FORCE_INLINE void tdGetColAppendDeltaLen(const void *value, int8_t colType, int32_t *dataLen, int32_t *kvLen) {
+  switch (colType) {
+    case TSDB_DATA_TYPE_BINARY: {
+      int32_t varLen = varDataLen(value);
+      *dataLen += (varLen - CHAR_BYTES);
+      *kvLen += (varLen + sizeof(SColIdx));
+      break;
+    }
+    case TSDB_DATA_TYPE_NCHAR: {
+      int32_t varLen = varDataLen(value);
+      *dataLen += (varLen - TSDB_NCHAR_SIZE);
+      *kvLen += (varLen + sizeof(SColIdx));
+      break;
+    }
+    default: {
+      *kvLen += (TYPE_BYTES[colType] + sizeof(SColIdx));
+      break;
+    }
+  }
+}
 
 typedef struct {
   int16_t colId;
@@ -706,7 +759,7 @@ static FORCE_INLINE void setSColInfo(SColInfo* colInfo, int16_t colId, uint8_t c
 
 SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
 
-
+#if 0
 // ----------------- Raw payload structure for row:
 /* |<------------ Head ------------->|<----------- body of column data tuple ------------------->|
  * |                                 |<----------------- flen ------------->|<--- value part --->|
@@ -752,6 +805,8 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch
 
 static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); }
 
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/common/inc/tvariant.h

@@ -53,6 +53,8 @@ int32_t tVariantToString(tVariant *pVar, char *dst);
 
 int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix);
 
+int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo);
+
 int32_t tVariantTypeSetType(tVariant *pVariant, char type);
 
 #ifdef __cplusplus

src/common/src/tdataformat.c

@@ -19,10 +19,10 @@
 #include "wchar.h"
 #include "tarray.h"
 
+static void dataColSetNEleNull(SDataCol *pCol, int nEle);
 static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
                                int limit2, int tRows, bool forceSetNull);
 
-//TODO: change caller to use return val
 int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   int spaceNeeded = pCol->bytes * maxPoints;
   if(IS_VAR_DATA_TYPE(pCol->type)) {
@@ -31,7 +31,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   if(pCol->spaceSize < spaceNeeded) {
     void* ptr = realloc(pCol->pData, spaceNeeded);
     if(ptr == NULL) {
-      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)pCol->spaceSize,
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded,
              strerror(errno));
       return -1;
     } else {
@@ -239,20 +239,19 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
   pDataCol->len = 0;
 }
 // value from timestamp should be TKEY here instead of TSKEY
-void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
+int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
   ASSERT(pCol != NULL && value != NULL);
 
   if (isAllRowsNull(pCol)) {
     if (isNull(value, pCol->type)) {
       // all null value yet, just return
-      return;
+      return 0;
     }
 
+    if(tdAllocMemForCol(pCol, maxPoints) < 0) return -1;
     if (numOfRows > 0) {
       // Find the first not null value, fill all previouse values as NULL
-      dataColSetNEleNull(pCol, numOfRows, maxPoints);
-    } else {
-      tdAllocMemForCol(pCol, maxPoints);
+      dataColSetNEleNull(pCol, numOfRows);
     }
   }
 
@@ -268,12 +267,21 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP
     memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes);
     pCol->len += pCol->bytes;
   }
+  return 0;
+}
+
+static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) {
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
+  } else {
+    return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
+  }
 }
 
 bool isNEleNull(SDataCol *pCol, int nEle) {
   if(isAllRowsNull(pCol)) return true;
   for (int i = 0; i < nEle; i++) {
-    if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
+    if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false;
   }
   return true;
 }
@@ -290,9 +298,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
   }
 }
 
-void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) {
-  tdAllocMemForCol(pCol, maxPoints);
-
+static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->len = 0;
     for (int i = 0; i < nEle; i++) {
@@ -318,7 +324,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle) {
   }
 }
 
-SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
+SDataCols *tdNewDataCols(int maxCols, int maxRows) {
   SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols));
   if (pCols == NULL) {
     uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno));
@@ -326,6 +332,9 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
   }
 
   pCols->maxPoints = maxRows;
+  pCols->maxCols = maxCols;
+  pCols->numOfRows = 0;
+  pCols->numOfCols = 0;
 
   if (maxCols > 0) {
     pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
@@ -342,13 +351,8 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
       pCols->cols[i].pData = NULL;
       pCols->cols[i].dataOff = NULL;
     }
-
-    pCols->maxCols = maxCols;
   }
 
-  pCols->maxRowSize = maxRowSize;
-
-
   return pCols;
 }
 
@@ -357,8 +361,9 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
   int oldMaxCols = pCols->maxCols;
   if (schemaNCols(pSchema) > oldMaxCols) {
     pCols->maxCols = schemaNCols(pSchema);
-    pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
-    if (pCols->cols == NULL) return -1;
+    void* ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
+    if (ptr == NULL) return -1;
+    pCols->cols = ptr;
     for(i = oldMaxCols; i < pCols->maxCols; i++) {
       pCols->cols[i].pData = NULL;
       pCols->cols[i].dataOff = NULL;
@@ -366,10 +371,6 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
     }
   }
 
-  if (schemaTLen(pSchema) > pCols->maxRowSize) {
-    pCols->maxRowSize = schemaTLen(pSchema);
-  }
-
   tdResetDataCols(pCols);
   pCols->numOfCols = schemaNCols(pSchema);
 
@@ -398,7 +399,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
 }
 
 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
-  SDataCols *pRet = tdNewDataCols(pDataCols->maxRowSize, pDataCols->maxCols, pDataCols->maxPoints);
+  SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
   if (pRet == NULL) return NULL;
 
   pRet->numOfCols = pDataCols->numOfCols;
@@ -413,7 +414,10 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 
     if (keepData) {
       if (pDataCols->cols[i].len > 0) {
-        tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints);
+        if(tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) {
+          tdFreeDataCols(pRet);
+          return NULL;
+        }
         pRet->cols[i].len = pDataCols->cols[i].len;
         memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
         if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
@@ -584,9 +588,12 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
       if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
         for (int i = 0; i < src2->numOfCols; i++) {
           ASSERT(target->cols[i].type == src2->cols[i].type);
-          if (src2->cols[i].len > 0 && (forceSetNull || (!forceSetNull && !isNull(src2->cols[i].pData, src2->cols[i].type)))) {
+          if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) {
             dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows,
                              target->maxPoints);
+          } else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
+            dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows,
+                             target->maxPoints);
           }
         }
         target->numOfRows++;
@@ -844,7 +851,8 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch
     int16_t k;
     for (k = 0; k < nKvNCols; ++k) {
       SColInfo *pColInfo = taosArrayGet(stashRow, k);
-      tdAppendKvColVal(kvRow, pColInfo->colVal, pColInfo->colId, pColInfo->colType, &toffset);
+      tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset);
+      toffset += sizeof(SColIdx);
     }
     ASSERT(kvLen == memRowTLen(tRow));
   }

src/common/src/texpr.c

@@ -118,7 +118,7 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) {
   } else if (pNode->nodeType == TSQL_NODE_VALUE) {
     tVariantDestroy(pNode->pVal);
   } else if (pNode->nodeType == TSQL_NODE_COL) {
-    free(pNode->pSchema);
+    tfree(pNode->pSchema);
   }
 
   free(pNode);
@@ -435,7 +435,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
     expr->_node.optr = TSDB_RELATION_IN;
     tVariant* pVal = exception_calloc(1, sizeof(tVariant));
     right->pVal = pVal;
-    pVal->nType = TSDB_DATA_TYPE_ARRAY;
+    pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY;
     pVal->arr = taosArrayInit(2, POINTER_BYTES);
 
     const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN;
@@ -502,6 +502,183 @@ void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) {
   *q = (void *)pObj;
 }
 
+void convertFilterSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) {
+  SBufferReader br = tbufInitReader(buf, len, false); 
+  uint32_t sType  = tbufReadUint32(&br);     
+  SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false);
+  
+  taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType)); 
+  
+  int dummy = -1;
+  tVariant tmpVar = {0};  
+  size_t  t = 0;
+  int32_t sz = tbufReadInt32(&br);
+  void *pvar = NULL;  
+  int64_t val = 0;
+  int32_t bufLen = 0;
+  if (IS_NUMERIC_TYPE(sType)) {
+    bufLen = 60;  // The maximum length of string that a number is converted to.
+  } else {
+    bufLen = 128;
+  }
+
+  char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE);
+    
+  for (int32_t i = 0; i < sz; i++) {
+    switch (sType) {
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_TINYINT: {
+      *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); 
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_SMALLINT: {
+      *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); 
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_INT: {
+      *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); 
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_TIMESTAMP:
+    case TSDB_DATA_TYPE_UBIGINT:
+    case TSDB_DATA_TYPE_BIGINT: {
+      *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); 
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      *(double *)&val = tbufReadDouble(&br);
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      *(float *)&val = (float)tbufReadDouble(&br);
+      t = sizeof(val);
+      pvar = &val;
+      break;
+    }
+    case TSDB_DATA_TYPE_BINARY: {
+      pvar = (char *)tbufReadBinary(&br, &t);
+      break;
+    }
+    case TSDB_DATA_TYPE_NCHAR: {
+      pvar = (char *)tbufReadBinary(&br, &t);      
+      break;
+    }
+    default:
+      taosHashCleanup(pObj);
+      *q = NULL;
+      return;
+    }
+    
+    tVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType);
+
+    if (bufLen < t) {
+      tmp = realloc(tmp, t * TSDB_NCHAR_SIZE);
+      bufLen = (int32_t)t;
+    }
+
+    switch (tType) {
+      case TSDB_DATA_TYPE_BOOL:
+      case TSDB_DATA_TYPE_UTINYINT:
+      case TSDB_DATA_TYPE_TINYINT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_USMALLINT:
+      case TSDB_DATA_TYPE_SMALLINT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_UINT:
+      case TSDB_DATA_TYPE_INT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_TIMESTAMP:
+      case TSDB_DATA_TYPE_UBIGINT:
+      case TSDB_DATA_TYPE_BIGINT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_DOUBLE: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_FLOAT: {
+        if (tVariantDump(&tmpVar, (char *)&val, tType, false)) {
+          goto err_ret;
+        }
+        pvar = &val;
+        t = sizeof(val);
+        break;
+      }
+      case TSDB_DATA_TYPE_BINARY: {
+        if (tVariantDump(&tmpVar, tmp, tType, true)) {
+          goto err_ret;
+        }
+        t = varDataLen(tmp);
+        pvar = varDataVal(tmp);
+        break;
+      }
+      case TSDB_DATA_TYPE_NCHAR: {
+        if (tVariantDump(&tmpVar, tmp, tType, true)) {
+          goto err_ret;
+        }
+        t = varDataLen(tmp);
+        pvar = varDataVal(tmp);        
+        break;
+      }
+      default:
+        goto err_ret;
+    }
+    
+    taosHashPut(pObj, (char *)pvar, t,  &dummy, sizeof(dummy));
+    tVariantDestroy(&tmpVar);
+    memset(&tmpVar, 0, sizeof(tmpVar));
+  } 
+
+  *q = (void *)pObj;
+  pObj = NULL;
+  
+err_ret:  
+  tVariantDestroy(&tmpVar);
+  taosHashCleanup(pObj);
+  tfree(tmp);
+}
+
+
 tExprNode* exprdup(tExprNode* pNode) {
   if (pNode == NULL) {
     return NULL;

src/common/src/tname.c

@@ -61,7 +61,7 @@ bool tscValidateTableNameLength(size_t len) {
 
 // TODO refactor
 SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters) {
-  if (numOfFilters == 0) {
+  if (numOfFilters == 0 || src == NULL) {
     assert(src == NULL);
     return NULL;
   }

src/common/src/ttypes.c

@@ -372,21 +372,21 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i
 }
 
 tDataTypeDescriptor tDataTypes[15] = {
-  {TSDB_DATA_TYPE_NULL,      6,1,        "NOTYPE",   NULL,     NULL,      NULL},
-  {TSDB_DATA_TYPE_BOOL,      4,  CHAR_BYTES,   "BOOL",               tsCompressBool,      tsDecompressBool,      getStatics_bool},
-  {TSDB_DATA_TYPE_TINYINT,   7,  CHAR_BYTES,   "TINYINT",            tsCompressTinyint,   tsDecompressTinyint,   getStatics_i8},
-  {TSDB_DATA_TYPE_SMALLINT,  8,  SHORT_BYTES,  "SMALLINT",           tsCompressSmallint,  tsDecompressSmallint,  getStatics_i16},
-  {TSDB_DATA_TYPE_INT,       3,  INT_BYTES,    "INT",                tsCompressInt,       tsDecompressInt,       getStatics_i32},
-  {TSDB_DATA_TYPE_BIGINT,    6,  LONG_BYTES,   "BIGINT",             tsCompressBigint,    tsDecompressBigint,    getStatics_i64},
-  {TSDB_DATA_TYPE_FLOAT,     5,  FLOAT_BYTES,  "FLOAT",              tsCompressFloat,     tsDecompressFloat,     getStatics_f},
-  {TSDB_DATA_TYPE_DOUBLE,    6,  DOUBLE_BYTES, "DOUBLE",             tsCompressDouble,    tsDecompressDouble,    getStatics_d},
-  {TSDB_DATA_TYPE_BINARY,    6,  0,      "BINARY",             tsCompressString,    tsDecompressString,    getStatics_bin},
-  {TSDB_DATA_TYPE_TIMESTAMP, 9,  LONG_BYTES,   "TIMESTAMP",          tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
-  {TSDB_DATA_TYPE_NCHAR,     5,  8,      "NCHAR",              tsCompressString,    tsDecompressString,    getStatics_nchr},
-  {TSDB_DATA_TYPE_UTINYINT,  16, CHAR_BYTES,   "TINYINT UNSIGNED",   tsCompressTinyint,   tsDecompressTinyint,   getStatics_u8},
-  {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES,  "SMALLINT UNSIGNED",  tsCompressSmallint,  tsDecompressSmallint,  getStatics_u16},
-  {TSDB_DATA_TYPE_UINT,      12, INT_BYTES,    "INT UNSIGNED",       tsCompressInt,       tsDecompressInt,       getStatics_u32},
-  {TSDB_DATA_TYPE_UBIGINT,   15, LONG_BYTES,   "BIGINT UNSIGNED",    tsCompressBigint,    tsDecompressBigint,    getStatics_u64},
+  {TSDB_DATA_TYPE_NULL,      6,  1,            "NOTYPE",             0,          0,              NULL,                NULL,                  NULL},
+  {TSDB_DATA_TYPE_BOOL,      4,  CHAR_BYTES,   "BOOL",               false,      true,           tsCompressBool,      tsDecompressBool,      getStatics_bool},
+  {TSDB_DATA_TYPE_TINYINT,   7,  CHAR_BYTES,   "TINYINT",            INT8_MIN,   INT8_MAX,       tsCompressTinyint,   tsDecompressTinyint,   getStatics_i8},
+  {TSDB_DATA_TYPE_SMALLINT,  8,  SHORT_BYTES,  "SMALLINT",           INT16_MIN,  INT16_MAX,      tsCompressSmallint,  tsDecompressSmallint,  getStatics_i16},
+  {TSDB_DATA_TYPE_INT,       3,  INT_BYTES,    "INT",                INT32_MIN,  INT32_MAX,      tsCompressInt,       tsDecompressInt,       getStatics_i32},
+  {TSDB_DATA_TYPE_BIGINT,    6,  LONG_BYTES,   "BIGINT",             INT64_MIN,  INT64_MAX,      tsCompressBigint,    tsDecompressBigint,    getStatics_i64},
+  {TSDB_DATA_TYPE_FLOAT,     5,  FLOAT_BYTES,  "FLOAT",              0,          0,              tsCompressFloat,     tsDecompressFloat,     getStatics_f},
+  {TSDB_DATA_TYPE_DOUBLE,    6,  DOUBLE_BYTES, "DOUBLE",             0,          0,              tsCompressDouble,    tsDecompressDouble,    getStatics_d},
+  {TSDB_DATA_TYPE_BINARY,    6,  0,            "BINARY",             0,          0,              tsCompressString,    tsDecompressString,    getStatics_bin},
+  {TSDB_DATA_TYPE_TIMESTAMP, 9,  LONG_BYTES,   "TIMESTAMP",          INT64_MIN,  INT64_MAX,      tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64},
+  {TSDB_DATA_TYPE_NCHAR,     5,  8,            "NCHAR",              0,          0,              tsCompressString,    tsDecompressString,    getStatics_nchr},
+  {TSDB_DATA_TYPE_UTINYINT,  16, CHAR_BYTES,   "TINYINT UNSIGNED",   0,          UINT8_MAX,      tsCompressTinyint,   tsDecompressTinyint,   getStatics_u8},
+  {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES,  "SMALLINT UNSIGNED",  0,          UINT16_MAX,     tsCompressSmallint,  tsDecompressSmallint,  getStatics_u16},
+  {TSDB_DATA_TYPE_UINT,      12, INT_BYTES,    "INT UNSIGNED",       0,          UINT32_MAX,     tsCompressInt,       tsDecompressInt,       getStatics_u32},
+  {TSDB_DATA_TYPE_UBIGINT,   15, LONG_BYTES,   "BIGINT UNSIGNED",    0,          UINT64_MAX,     tsCompressBigint,    tsDecompressBigint,    getStatics_u64},
 };
 
 char tTokenTypeSwitcher[13] = {
@@ -405,6 +405,32 @@ char tTokenTypeSwitcher[13] = {
     TSDB_DATA_TYPE_NCHAR,   // TK_NCHAR
 };
 
+float floatMin = -FLT_MAX, floatMax = FLT_MAX;
+double doubleMin = -DBL_MAX, doubleMax = DBL_MAX;
+
+FORCE_INLINE void* getDataMin(int32_t type) {
+  switch (type) {
+    case TSDB_DATA_TYPE_FLOAT:
+      return &floatMin;
+    case TSDB_DATA_TYPE_DOUBLE:
+      return &doubleMin;
+    default:
+      return &tDataTypes[type].minValue;
+  }
+}
+
+FORCE_INLINE void* getDataMax(int32_t type) {
+  switch (type) {
+    case TSDB_DATA_TYPE_FLOAT:
+      return &floatMax;
+    case TSDB_DATA_TYPE_DOUBLE:
+      return &doubleMax;
+    default:
+      return &tDataTypes[type].maxValue;
+  }
+}
+
+
 bool isValidDataType(int32_t type) {
   return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT;
 }
@@ -566,6 +592,53 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
   }
 }
 
+void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
+  if (optr == TSDB_BINARY_OP_ADD) {
+    switch (type) {
+      case TSDB_DATA_TYPE_TINYINT:
+        *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_UTINYINT:
+        *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2);        
+        break;
+      case TSDB_DATA_TYPE_SMALLINT:
+        *((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_USMALLINT:
+        *((uint16_t *)dst) = GET_UINT16_VAL(s1) + GET_UINT16_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_INT:
+        *((int32_t *)dst) = GET_INT32_VAL(s1) + GET_INT32_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_UINT:
+        *((uint32_t *)dst) = GET_UINT32_VAL(s1) + GET_UINT32_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_BIGINT:
+        *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_UBIGINT:
+        *((uint64_t *)dst) = GET_UINT64_VAL(s1) + GET_UINT64_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_TIMESTAMP:
+        *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2);
+        break;
+      case TSDB_DATA_TYPE_FLOAT:
+        SET_FLOAT_VAL(dst, GET_FLOAT_VAL(s1) + GET_FLOAT_VAL(s2));
+        break;
+      case TSDB_DATA_TYPE_DOUBLE:
+        SET_DOUBLE_VAL(dst, GET_DOUBLE_VAL(s1) + GET_DOUBLE_VAL(s2));
+        break;
+      default: {
+        assert(0);
+        break;
+      }
+    }
+  } else {
+    assert(0);
+  }
+}
+
+
 void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) {
   switch (type) {
     case TSDB_DATA_TYPE_INT:

src/common/src/tvariant.c

@@ -23,6 +23,13 @@
 #include "tutil.h"
 #include "tvariant.h"
 
+#define SET_EXT_INFO(converted, res, minv, maxv, exti) do {                                                       \
+                                                        if (converted == NULL || exti == NULL || *converted == false) { break; }  \
+                                                        if ((res) < (minv)) { *exti = -1; break; }                                  \
+                                                        if ((res) > (maxv)) { *exti = 1; break; }                                   \
+                                                        assert(0);                                                                  \
+                                                       } while (0)
+
 void tVariantCreate(tVariant *pVar, SStrToken *token) {
   int32_t ret = 0;
   int32_t type = token->type;
@@ -184,7 +191,7 @@ void tVariantDestroy(tVariant *pVar) {
   }
 
   // NOTE: this is only for string array
-  if (pVar->nType == TSDB_DATA_TYPE_ARRAY) {
+  if (pVar->nType == TSDB_DATA_TYPE_POINTER_ARRAY) {
     size_t num = taosArrayGetSize(pVar->arr);
     for(size_t i = 0; i < num; i++) {
       void* p = taosArrayGetP(pVar->arr, i);
@@ -192,6 +199,9 @@ void tVariantDestroy(tVariant *pVar) {
     }
     taosArrayDestroy(pVar->arr);
     pVar->arr = NULL;
+  } else if (pVar->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
+    taosArrayDestroy(pVar->arr);
+    pVar->arr = NULL;
   }
 }
 
@@ -220,7 +230,7 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
 
   if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
     pDst->i64 = pSrc->i64;
-  } else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) {  // this is only for string array
+  } else if (pSrc->nType == TSDB_DATA_TYPE_POINTER_ARRAY) {  // this is only for string array
     size_t num = taosArrayGetSize(pSrc->arr);
     pDst->arr = taosArrayInit(num, sizeof(char*));
     for(size_t i = 0; i < num; i++) {
@@ -228,9 +238,18 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) {
       char* n = strdup(p);
       taosArrayPush(pDst->arr, &n);
     }
+  } else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
+      size_t num = taosArrayGetSize(pSrc->arr);
+      pDst->arr = taosArrayInit(num, sizeof(int64_t));
+      pDst->nLen = pSrc->nLen;
+      assert(pSrc->nLen == num);
+      for(size_t i = 0; i < num; i++) {
+        int64_t *p = taosArrayGet(pSrc->arr, i);
+        taosArrayPush(pDst->arr, p);
+      }
   }
 
-  if (pDst->nType != TSDB_DATA_TYPE_ARRAY) {
+  if (pDst->nType != TSDB_DATA_TYPE_POINTER_ARRAY && pDst->nType != TSDB_DATA_TYPE_VALUE_ARRAY) {
     pDst->nLen = tDataTypes[pDst->nType].bytes;
   }
 }
@@ -450,7 +469,7 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val
   return 0;
 }
 
-static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr) {
+static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr, bool *converted) {
   if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
     setNull((char *)result, type, tDataTypes[type].bytes);
     return 0;
@@ -540,6 +559,10 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
     }
   }
 
+  if (converted) {
+    *converted = true;
+  }
+
   bool code = false;
 
   uint64_t ui = 0;
@@ -602,6 +625,18 @@ static int32_t convertToBool(tVariant *pVariant, int64_t *pDest) {
  * to column type defined in schema
  */
 int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix) {
+  return tVariantDumpEx(pVariant, payload, type, includeLengthPrefix, NULL, NULL);
+}
+
+/*
+ * transfer data from variant serve as the implicit data conversion: from input sql string pVariant->nType
+ * to column type defined in schema
+ */
+int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo) {
+  if (converted) {
+    *converted = false;
+  }
+  
   if (pVariant == NULL || (pVariant->nType != 0 && !isValidDataType(pVariant->nType))) {
     return -1;
   }
@@ -620,7 +655,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
     
     case TSDB_DATA_TYPE_TINYINT: {
-      if (convertToInteger(pVariant, &result, type, true, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, INT8_MIN + 1, INT8_MAX, extInfo);
         return -1;
       }
       *((int8_t *)payload) = (int8_t) result;
@@ -628,7 +664,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
 
     case TSDB_DATA_TYPE_UTINYINT: {
-      if (convertToInteger(pVariant, &result, type, false, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {        
+        SET_EXT_INFO(converted, result, 0, UINT8_MAX - 1, extInfo);
         return -1;
       }
       *((uint8_t *)payload) = (uint8_t) result;
@@ -636,7 +673,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
     
     case TSDB_DATA_TYPE_SMALLINT: {
-      if (convertToInteger(pVariant, &result, type, true, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, INT16_MIN + 1, INT16_MAX, extInfo);
         return -1;
       }
       *((int16_t *)payload) = (int16_t)result;
@@ -644,7 +682,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
 
     case TSDB_DATA_TYPE_USMALLINT: {
-      if (convertToInteger(pVariant, &result, type, false, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, 0, UINT16_MAX - 1, extInfo);
         return -1;
       }
       *((uint16_t *)payload) = (uint16_t)result;
@@ -652,7 +691,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
     
     case TSDB_DATA_TYPE_INT: {
-      if (convertToInteger(pVariant, &result, type, true, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, INT32_MIN + 1, INT32_MAX, extInfo);
         return -1;
       }
       *((int32_t *)payload) = (int32_t)result;
@@ -660,7 +700,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
 
     case TSDB_DATA_TYPE_UINT: {
-      if (convertToInteger(pVariant, &result, type, false, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
+        SET_EXT_INFO(converted, result, 0, UINT32_MAX - 1, extInfo);
         return -1;
       }
       *((uint32_t *)payload) = (uint32_t)result;
@@ -668,7 +709,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
     
     case TSDB_DATA_TYPE_BIGINT: {
-      if (convertToInteger(pVariant, &result, type, true, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) {
+        SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo);
         return -1;
       }
       *((int64_t *)payload) = (int64_t)result;
@@ -676,7 +718,8 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
     }
 
     case TSDB_DATA_TYPE_UBIGINT: {
-      if (convertToInteger(pVariant, &result, type, false, false) < 0) {
+      if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) {
+        SET_EXT_INFO(converted, (uint64_t)result, 0, UINT64_MAX - 1, extInfo);
         return -1;
       }
       *((uint64_t *)payload) = (uint64_t)result;
@@ -696,11 +739,37 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
             return -1;
           }
 
+          if (converted) {
+            *converted = true;
+          }
+          
+          if (value > FLT_MAX || value < -FLT_MAX) {
+            SET_EXT_INFO(converted, value, -FLT_MAX, FLT_MAX, extInfo);
+            return -1;
+          }
           SET_FLOAT_VAL(payload, value);
         }
       } else if (pVariant->nType == TSDB_DATA_TYPE_BOOL || IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) {
+        if (converted) {
+          *converted = true;
+        }
+        
+        if (pVariant->i64 > FLT_MAX || pVariant->i64 < -FLT_MAX) {          
+          SET_EXT_INFO(converted, pVariant->i64, -FLT_MAX, FLT_MAX, extInfo);
+          return -1;
+        }
+
         SET_FLOAT_VAL(payload, pVariant->i64);
       } else if (IS_FLOAT_TYPE(pVariant->nType)) {
+        if (converted) {
+          *converted = true;
+        }
+        
+        if (pVariant->dKey > FLT_MAX || pVariant->dKey < -FLT_MAX) {          
+          SET_EXT_INFO(converted, pVariant->dKey, -FLT_MAX, FLT_MAX, extInfo);
+          return -1;
+        }
+      
         SET_FLOAT_VAL(payload, pVariant->dKey);
       } else if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
         *((uint32_t *)payload) = TSDB_DATA_FLOAT_NULL;
@@ -824,6 +893,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
   return 0;
 }
 
+
 /*
  * In variant, bool/smallint/tinyint/int/bigint share the same attribution of
  * structure, also ignore the convert the type required
@@ -848,7 +918,7 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) {
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_SMALLINT: {
-      convertToInteger(pVariant, &(pVariant->i64), type, true, true);
+      convertToInteger(pVariant, &(pVariant->i64), type, true, true, NULL);
       pVariant->nType = TSDB_DATA_TYPE_BIGINT;
       break;
     }

src/connector/jdbc/pom.xml

@@ -113,7 +113,6 @@
                     </includes>
                     <excludes>
                         <exclude>**/AppMemoryLeakTest.java</exclude>
-                        <exclude>**/AuthenticationTest.java</exclude>
                         <exclude>**/ConnectMultiTaosdByRestfulWithDifferentTokenTest.java</exclude>
                         <exclude>**/DatetimeBefore1970Test.java</exclude>
                         <exclude>**/FailOverTest.java</exclude>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -14,6 +14,8 @@
  *****************************************************************************/
 package com.taosdata.jdbc;
 
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.sql.*;
 import java.util.*;
 import java.util.logging.Logger;
@@ -127,6 +129,11 @@ public class TSDBDriver extends AbstractDriver {
             return null;
         }
 
+        if (!props.containsKey(TSDBDriver.PROPERTY_KEY_USER))
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED);
+        if (!props.containsKey(TSDBDriver.PROPERTY_KEY_PASSWORD))
+            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED);
+
         try {
             TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE),
                     (String) props.get(PROPERTY_KEY_CHARSET), (String) props.get(PROPERTY_KEY_TIME_ZONE));

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java

@@ -33,6 +33,8 @@ public class TSDBError {
         TSDBErrorMap.put(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE, "numeric value out of range");
         TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE, "unknown taos type in tdengine");
         TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TIMESTAMP_PRECISION, "unknown timestamp precision");
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, "user is required");
+        TSDBErrorMap.put(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, "password is required");
 
         TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN, "unknown error");
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java

@@ -29,6 +29,9 @@ public class TSDBErrorNumbers {
     public static final int ERROR_UNKNOWN_TIMESTAMP_PRECISION = 0x2316;     // unknown timestamp precision
     public static final int ERROR_RESTFul_Client_Protocol_Exception = 0x2317;
     public static final int ERROR_RESTFul_Client_IOException = 0x2318;
+    public static final int ERROR_USER_IS_REQUIRED = 0x2319;        // user is required
+    public static final int ERROR_PASSWORD_IS_REQUIRED = 0x231a;    // password is required
+
 
     public static final int ERROR_UNKNOWN = 0x2350;    //unknown error
 
@@ -67,6 +70,8 @@ public class TSDBErrorNumbers {
         errorNumbers.add(ERROR_UNKNOWN_TAOS_TYPE);
         errorNumbers.add(ERROR_UNKNOWN_TIMESTAMP_PRECISION);
         errorNumbers.add(ERROR_RESTFul_Client_IOException);
+        errorNumbers.add(ERROR_USER_IS_REQUIRED);
+        errorNumbers.add(ERROR_PASSWORD_IS_REQUIRED);
 
         errorNumbers.add(ERROR_RESTFul_Client_Protocol_Exception);
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -36,7 +36,6 @@ public class TSDBJNIConnector {
 
     static {
         System.loadLibrary("taos");
-        System.out.println("java.library.path:" + System.getProperty("java.library.path"));
     }
 
     public boolean isClosed() {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java

@@ -7,6 +7,7 @@ import com.taosdata.jdbc.utils.HttpClientPoolUtil;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.sql.*;
 import java.util.Properties;
 import java.util.logging.Logger;
@@ -40,8 +41,13 @@ public class RestfulDriver extends AbstractDriver {
 
         String loginUrl = "http://" + host + ":" + port + "/rest/login/" + props.getProperty(TSDBDriver.PROPERTY_KEY_USER) + "/" + props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD) + "";
         try {
-            String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), "UTF-8");
-            String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), "UTF-8");
+            if (!props.containsKey(TSDBDriver.PROPERTY_KEY_USER))
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED);
+            if (!props.containsKey(TSDBDriver.PROPERTY_KEY_PASSWORD))
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED);
+
+            String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), StandardCharsets.UTF_8.displayName());
+            String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), StandardCharsets.UTF_8.displayName());
             loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + user + "/" + password + "";
         } catch (UnsupportedEncodingException e) {
             e.printStackTrace();

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java

@@ -7,6 +7,7 @@ import com.taosdata.jdbc.AbstractStatement;
 import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.TSDBError;
 import com.taosdata.jdbc.TSDBErrorNumbers;
+import com.taosdata.jdbc.enums.TimestampFormat;
 import com.taosdata.jdbc.utils.HttpClientPoolUtil;
 import com.taosdata.jdbc.utils.SqlSyntaxValidator;
 
@@ -45,9 +46,7 @@ public class RestfulStatement extends AbstractStatement {
         if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql))
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql);
 
-        final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
-
-        return executeOneUpdate(url, sql);
+        return executeOneUpdate(sql);
     }
 
     @Override
@@ -62,34 +61,25 @@ public class RestfulStatement extends AbstractStatement {
     public boolean execute(String sql) throws SQLException {
         if (isClosed())
             throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED);
-        if (!SqlSyntaxValidator.isValidForExecute(sql))
-            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE, "not a valid sql for execute: " + sql);
 
         //如果执行了use操作应该将当前Statement的catalog设置为新的database
         boolean result = true;
-        String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
-        if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("TIMESTAMP")) {
-            url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt";
-        }
-        if (conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).equals("UTC")) {
-            url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc";
-        }
 
         if (SqlSyntaxValidator.isUseSql(sql)) {
-            HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
+            HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
             this.database = sql.trim().replace("use", "").trim();
             this.conn.setCatalog(this.database);
             result = false;
         } else if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) {
             executeOneQuery(sql);
         } else if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) {
-            executeOneUpdate(url, sql);
+            executeOneUpdate(sql);
             result = false;
         } else {
             if (SqlSyntaxValidator.isValidForExecuteQuery(sql)) {
-                executeQuery(sql);
+                executeOneQuery(sql);
             } else {
-                executeUpdate(sql);
+                executeOneUpdate(sql);
                 result = false;
             }
         }
@@ -97,19 +87,25 @@ public class RestfulStatement extends AbstractStatement {
         return result;
     }
 
-    private ResultSet executeOneQuery(String sql) throws SQLException {
-        if (!SqlSyntaxValidator.isValidForExecuteQuery(sql))
-            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql);
+    private String getUrl() throws SQLException {
+        TimestampFormat timestampFormat = TimestampFormat.valueOf(conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT).trim().toUpperCase());
+        String url;
+        switch (timestampFormat) {
+            case TIMESTAMP:
+                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt";
+                break;
+            case UTC:
+                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc";
+                break;
+            default:
+                url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
+        }
+        return url;
+    }
 
+    private ResultSet executeOneQuery(String sql) throws SQLException {
         // row data
-        String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
-        String timestampFormat = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIMESTAMP_FORMAT);
-        if ("TIMESTAMP".equalsIgnoreCase(timestampFormat))
-            url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlt";
-        if ("UTC".equalsIgnoreCase(timestampFormat))
-            url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sqlutc";
-
-        String result = HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
+        String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
         JSONObject resultJson = JSON.parseObject(result);
         if (resultJson.getString("status").equals("error")) {
             throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc"));
@@ -119,11 +115,8 @@ public class RestfulStatement extends AbstractStatement {
         return resultSet;
     }
 
-    private int executeOneUpdate(String url, String sql) throws SQLException {
-        if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql))
-            throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql);
-
-        String result = HttpClientPoolUtil.execute(url, sql, this.conn.getToken());
+    private int executeOneUpdate(String sql) throws SQLException {
+        String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken());
         JSONObject jsonObject = JSON.parseObject(result);
         if (jsonObject.getString("status").equals("error")) {
             throw TSDBError.createSQLException(jsonObject.getInteger("code"), jsonObject.getString("desc"));
@@ -134,7 +127,7 @@ public class RestfulStatement extends AbstractStatement {
     }
 
     private int getAffectedRows(JSONObject jsonObject) throws SQLException {
-        // create ... SQLs should return 0 , and Restful result is this:
+        // create ... SQLs should return 0 , and Restful result like this:
         // {"status": "succ", "head": ["affected_rows"], "data": [[0]], "rows": 1}
         JSONArray head = jsonObject.getJSONArray("head");
         if (head.size() != 1 || !"affected_rows".equals(head.getString(0)))

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java

@@ -16,8 +16,7 @@ package com.taosdata.jdbc.utils;
 
 public class SqlSyntaxValidator {
 
-    private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe", "reset"};
-    private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set"};
+    private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set", "reset"};
     private static final String[] querySQL = {"select", "show", "describe"};
 
     private static final String[] databaseUnspecifiedShow = {"databases", "dnodes", "mnodes", "variables"};
@@ -38,14 +37,6 @@ public class SqlSyntaxValidator {
         return false;
     }
 
-    public static boolean isValidForExecute(String sql) {
-        for (String prefix : SQL) {
-            if (sql.trim().toLowerCase().startsWith(prefix))
-                return true;
-        }
-        return false;
-    }
-
     public static boolean isDatabaseUnspecifiedQuery(String sql) {
         for (String databaseObj : databaseUnspecifiedShow) {
             if (sql.trim().toLowerCase().matches("show\\s+" + databaseObj + ".*"))
@@ -63,9 +54,5 @@ public class SqlSyntaxValidator {
         return sql.trim().toLowerCase().startsWith("use");
     }
 
-    public static boolean isSelectSql(String sql) {
-        return sql.trim().toLowerCase().startsWith("select");
-    }
-
 
 }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java

@@ -69,6 +69,8 @@ public class SubscribeTest {
     @Before
     public void createDatabase() throws SQLException {
         Properties properties = new Properties();
+        properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+        properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java

 package com.taosdata.jdbc.cases;
 
+import com.taosdata.jdbc.TSDBErrorNumbers;
+import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.sql.*;
@@ -12,6 +15,47 @@ public class AuthenticationTest {
     private static final String password = "taos?data";
     private Connection conn;
 
+    @Test
+    public void connectWithoutUserByJni() {
+        try {
+            DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?");
+        } catch (SQLException e) {
+            Assert.assertEquals(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, e.getErrorCode());
+            Assert.assertEquals("ERROR (2319): user is required", e.getMessage());
+        }
+    }
+
+    @Test
+    public void connectWithoutUserByRestful() {
+        try {
+            DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?");
+        } catch (SQLException e) {
+            Assert.assertEquals(TSDBErrorNumbers.ERROR_USER_IS_REQUIRED, e.getErrorCode());
+            Assert.assertEquals("ERROR (2319): user is required", e.getMessage());
+        }
+    }
+
+    @Test
+    public void connectWithoutPasswordByJni() {
+        try {
+            DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root");
+        } catch (SQLException e) {
+            Assert.assertEquals(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, e.getErrorCode());
+            Assert.assertEquals("ERROR (231a): password is required", e.getMessage());
+        }
+    }
+
+    @Test
+    public void connectWithoutPasswordByRestful() {
+        try {
+            DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root");
+        } catch (SQLException e) {
+            Assert.assertEquals(TSDBErrorNumbers.ERROR_PASSWORD_IS_REQUIRED, e.getErrorCode());
+            Assert.assertEquals("ERROR (231a): password is required", e.getMessage());
+        }
+    }
+
+    @Ignore
     @Test
     public void test() {
         // change password

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java

@@ -29,6 +29,8 @@ public class BatchInsertTest {
     public void before() {
         try {
             Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java

@@ -21,6 +21,8 @@ public class ImportTest {
     public static void before() {
         try {
             Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java

@@ -270,6 +270,41 @@ public class InsertSpecialCharacterJniTest {
         }
     }
 
+    @Ignore
+    @Test
+    public void testSingleQuotaEscape() throws SQLException {
+        final long now = System.currentTimeMillis();
+        final String sql = "insert into t? using ? tags(?) values(?, ?, ?) t? using " + tbname2 + " tags(?) values(?,?,?) ";
+        try (PreparedStatement pstmt = conn.prepareStatement(sql)) {
+            // t1
+            pstmt.setInt(1, 1);
+            pstmt.setString(2, tbname2);
+            pstmt.setString(3, special_character_str_5);
+            pstmt.setTimestamp(4, new Timestamp(now));
+            pstmt.setBytes(5, special_character_str_5.getBytes());
+            // t2
+            pstmt.setInt(7, 2);
+            pstmt.setString(8, special_character_str_5);
+            pstmt.setTimestamp(9, new Timestamp(now));
+            pstmt.setString(11, special_character_str_5);
+
+            int ret = pstmt.executeUpdate();
+            Assert.assertEquals(2, ret);
+        }
+
+        String query = "select * from ?.t? where ? < ? and ts >= ? and f1 is not null";
+        try (PreparedStatement pstmt = conn.prepareStatement(query)) {
+            pstmt.setString(1, dbName);
+            pstmt.setInt(2, 1);
+            pstmt.setString(3, "ts");
+            pstmt.setTimestamp(4, new Timestamp(System.currentTimeMillis()));
+            pstmt.setTimestamp(5, new Timestamp(0));
+
+            ResultSet rs = pstmt.executeQuery();
+            Assert.assertNotNull(rs);
+        }
+    }
+
     @Test
     public void testCase10() throws SQLException {
         final long now = System.currentTimeMillis();
@@ -293,13 +328,12 @@ public class InsertSpecialCharacterJniTest {
             Assert.assertEquals(2, ret);
         }
         //query t1
-        String query = "select * from ?.t? where ts < ? and ts >= ? and ? is not null";
+        String query = "select * from ?.t? where ts < ? and ts >= ? and f1 is not null";
         try (PreparedStatement pstmt = conn.prepareStatement(query)) {
             pstmt.setString(1, dbName);
             pstmt.setInt(2, 1);
             pstmt.setTimestamp(3, new Timestamp(System.currentTimeMillis()));
             pstmt.setTimestamp(4, new Timestamp(0));
-            pstmt.setString(5, "f1");
 
             ResultSet rs = pstmt.executeQuery();
             rs.next();
@@ -311,12 +345,11 @@ public class InsertSpecialCharacterJniTest {
             Assert.assertNull(f2);
         }
         // query t2
-        query = "select * from t? where ts < ? and ts >= ? and ? is not null";
+        query = "select * from t? where ts < ? and ts >= ? and f2 is not null";
         try (PreparedStatement pstmt = conn.prepareStatement(query)) {
             pstmt.setInt(1, 2);
             pstmt.setTimestamp(2, new Timestamp(System.currentTimeMillis()));
             pstmt.setTimestamp(3, new Timestamp(0));
-            pstmt.setString(4, "f2");
 
             ResultSet rs = pstmt.executeQuery();
             rs.next();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java

@@ -293,13 +293,12 @@ public class InsertSpecialCharacterRestfulTest {
             Assert.assertEquals(2, ret);
         }
         //query t1
-        String query = "select * from ?.t? where ts < ? and ts >= ? and ? is not null";
+        String query = "select * from ?.t? where ts < ? and ts >= ? and f1 is not null";
         try (PreparedStatement pstmt = conn.prepareStatement(query)) {
             pstmt.setString(1, dbName);
             pstmt.setInt(2, 1);
             pstmt.setTimestamp(3, new Timestamp(System.currentTimeMillis()));
             pstmt.setTimestamp(4, new Timestamp(0));
-            pstmt.setString(5, "f1");
 
             ResultSet rs = pstmt.executeQuery();
             rs.next();
@@ -311,12 +310,11 @@ public class InsertSpecialCharacterRestfulTest {
             Assert.assertNull(f2);
         }
         // query t2
-        query = "select * from t? where ts < ? and ts >= ? and ? is not null";
+        query = "select * from t? where ts < ? and ts >= ? and f2 is not null";
         try (PreparedStatement pstmt = conn.prepareStatement(query)) {
             pstmt.setInt(1, 2);
             pstmt.setTimestamp(2, new Timestamp(System.currentTimeMillis()));
             pstmt.setTimestamp(3, new Timestamp(0));
-            pstmt.setString(4, "f2");
 
             ResultSet rs = pstmt.executeQuery();
             rs.next();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java

@@ -22,6 +22,8 @@ public class QueryDataTest {
     public void createDatabase() {
         try {
             Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ResetQueryCacheTest.java

 package com.taosdata.jdbc.cases;
 
-import com.taosdata.jdbc.TSDBDriver;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 
-import java.sql.*;
-import java.util.Properties;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 public class ResetQueryCacheTest {
 
-    static Connection connection;
-    static Statement statement;
-    static String host = "127.0.0.1";
-
-    @Before
-    public void init() {
-        try {
-            Properties properties = new Properties();
-            properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
-            properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
-            properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-            connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
-            statement = connection.createStatement();
-        } catch (SQLException e) {
-            return;
-        }
+    @Test
+    public void jni() throws SQLException {
+        // given
+        Connection connection = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:0/?user=root&password=taosdata&timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8");
+        Statement statement = connection.createStatement();
+
+        // when
+        boolean execute = statement.execute("reset query cache");
+
+        // then
+        assertFalse(execute);
+        assertEquals(0, statement.getUpdateCount());
+
+        statement.close();
+        connection.close();
     }
 
     @Test
-    public void testResetQueryCache() throws SQLException {
-        String resetSql = "reset query cache";
-        statement.execute(resetSql);
-    }
+    public void restful() throws SQLException {
+        // given
+        Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://127.0.0.1:6041/?user=root&password=taosdata&timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8");
+        Statement statement = connection.createStatement();
+
+        // when
+        boolean execute = statement.execute("reset query cache");
+
+        // then
+        assertFalse(execute);
+        assertEquals(0, statement.getUpdateCount());
 
-    @After
-    public void close() {
-        try {
-            if (statement != null)
-                statement.close();
-            if (connection != null)
-                connection.close();
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
+        statement.close();
+        connection.close();
     }
 
 }
\ No newline at end of file

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java

@@ -20,6 +20,8 @@ public class SelectTest {
     public void createDatabaseAndTable() {
         try {
             Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java

@@ -24,6 +24,8 @@ public class StableTest {
     public static void createDatabase() {
         try {
             Properties properties = new Properties();
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
+            properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
             properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java

-package com.taosdata.jdbc.utils;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class SqlSyntaxValidatorTest {
-
-    @Test
-    public void isSelectSQL() {
-        Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather"));
-        Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather"));
-        Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather "));
-        Assert.assertFalse(SqlSyntaxValidator.isSelectSql("insert into test.weather values(now, 1.1, 2)"));
-    }
-
-    @Test
-    public void isUseSQL() {
-        Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test"));
-    }
-
-}
\ No newline at end of file

src/inc/taosdef.h

@@ -34,6 +34,7 @@ extern "C" {
 
 #define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX})
 #define TSWINDOW_DESC_INITIALIZER ((STimeWindow) {INT64_MAX, INT64_MIN})
+#define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX))
 
 #define TSKEY_INITIAL_VAL    INT64_MIN
 

src/inc/taoserror.h

@@ -272,7 +272,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW       TAOS_DEF_ERROR_CODE(0, 0x070A)  //"Too many time window in query")
 #define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER         TAOS_DEF_ERROR_CODE(0, 0x070B)  //"Query buffer limit has reached")
 #define TSDB_CODE_QRY_INCONSISTAN               TAOS_DEF_ERROR_CODE(0, 0x070C)  //"File inconsistency in replica")
-#define TSDB_CODE_QRY_SYS_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x070D)  //"System error")
+#define TSDB_CODE_QRY_INVALID_TIME_CONDITION    TAOS_DEF_ERROR_CODE(0, 0x070D)  //"invalid time condition")
+#define TSDB_CODE_QRY_SYS_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x070E)  //"System error")
 
 
 // grant

src/inc/taosmsg.h

@@ -489,7 +489,8 @@ typedef struct {
   int16_t     numOfCols;        // the number of columns will be load from vnode
   SInterval   interval;
   SSessionWindow sw;            // session window
-  uint32_t    tagCondLen;       // tag length in current query
+  uint16_t    tagCondLen;       // tag length in current query
+  uint16_t    colCondLen;       // column length in current query
   uint32_t    tbnameCondLen;    // table name filter condition string length
   int16_t     numOfGroupCols;   // num of group by columns
   int16_t     orderByIdx;

src/inc/ttype.h

@@ -47,7 +47,8 @@ typedef struct {
 
 
 // this data type is internally used only in 'in' query to hold the values
-#define TSDB_DATA_TYPE_ARRAY      (1000)
+#define TSDB_DATA_TYPE_POINTER_ARRAY      (1000)
+#define TSDB_DATA_TYPE_VALUE_ARRAY      (1001)
 
 #define GET_TYPED_DATA(_v, _finalType, _type, _data) \
   do {                                               \
@@ -181,6 +182,8 @@ typedef struct tDataTypeDescriptor {
   int16_t nameLen;
   int32_t bytes;
   char *  name;
+  int64_t minValue;
+  int64_t maxValue;
   int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
                   char algorithm, char *const buffer, int bufferSize);
   int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output,
@@ -200,6 +203,9 @@ const void *getNullValue(int32_t type);
 
 void assignVal(char *val, const char *src, int32_t len, int32_t type);
 void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf);
+void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
+void* getDataMin(int32_t type);
+void* getDataMax(int32_t type);
 
 int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned);
 

src/plugins/http/src/httpParser.c

@@ -101,13 +101,17 @@ char *httpGetStatusDesc(int32_t statusCode) {
 }
 
 static void httpCleanupString(HttpString *str) {
-  free(str->str);
-  str->str = NULL;
-  str->pos = 0;
-  str->size = 0;
+  if (str->str) {
+    free(str->str);
+    str->str = NULL;
+    str->pos = 0;
+    str->size = 0;
+  }
 }
 
 static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) {
+  char *new_str = NULL;
+
   if (str->size == 0) {
     str->pos = 0;
     str->size = len + 1;
@@ -115,7 +119,16 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) {
   } else if (str->pos + len + 1 >= str->size) {
     str->size += len;
     str->size *= 4;
-    str->str = realloc(str->str, str->size);
+
+    new_str = realloc(str->str, str->size);
+    if (new_str == NULL && str->str) {
+      // if str->str was not NULL originally,
+      // the old allocated memory was left unchanged,
+      // see man 3 realloc
+      free(str->str);
+    }
+
+    str->str = new_str;
   } else {
   }
 
@@ -317,7 +330,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const
 
 static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) {
   HttpContext *pContext = parser->pContext;
-  HttpString * buf = &parser->body;
+  HttpString *buf = &parser->body;
   if (parser->parseCode != TSDB_CODE_SUCCESS) return -1;
 
   if (buf->size <= 0) {
@@ -326,6 +339,7 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) {
   }
 
   int32_t newSize = buf->pos + len + 1;
+  char *newStr = NULL;
   if (newSize >= buf->size) {
     if (buf->size >= HTTP_BUFFER_SIZE) {
       httpError("context:%p, fd:%d, failed parse body, exceeding buffer size %d", pContext, pContext->fd, buf->size);
@@ -336,7 +350,12 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) {
     newSize = MAX(newSize, HTTP_BUFFER_INIT);
     newSize *= 4;
     newSize = MIN(newSize, HTTP_BUFFER_SIZE);
-    buf->str = realloc(buf->str, newSize);
+    newStr = realloc(buf->str, newSize);
+    if (newStr == NULL && buf->str) {
+      free(buf->str);
+    }
+
+    buf->str = newStr;
     buf->size = newSize;
 
     if (buf->str == NULL) {
@@ -374,13 +393,20 @@ static HTTP_PARSER_STATE httpTopStack(HttpParser *parser) {
 
 static int32_t httpPushStack(HttpParser *parser, HTTP_PARSER_STATE state) {
   HttpStack *stack = &parser->stacks;
+  int8_t *newStacks = NULL;
   if (stack->size == 0) {
     stack->pos = 0;
     stack->size = 32;
     stack->stacks = malloc(stack->size * sizeof(int8_t));
   } else if (stack->pos + 1 > stack->size) {
     stack->size *= 2;
-    stack->stacks = realloc(stack->stacks, stack->size * sizeof(int8_t));
+
+    newStacks = realloc(stack->stacks, stack->size * sizeof(int8_t));
+    if (newStacks == NULL && stack->stacks) {
+      free(stack->stacks);
+    }
+
+    stack->stacks = newStacks;
   } else {
   }
 

src/plugins/http/src/httpUtil.c

@@ -188,13 +188,17 @@ bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferS
 bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) {
   HttpSqlCmds *multiCmds = pContext->multiCmds;
 
-  if (cmdSize > HTTP_MAX_CMD_SIZE) {
+  if (cmdSize <= 0 || cmdSize > HTTP_MAX_CMD_SIZE) {
     httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user,
               cmdSize, HTTP_MAX_CMD_SIZE);
     return false;
   }
 
-  multiCmds->cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd));
+  HttpSqlCmd *new_cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd));
+  if (new_cmds == NULL && multiCmds->cmds) {
+    free(multiCmds->cmds);
+  }
+  multiCmds->cmds = new_cmds;
   if (multiCmds->cmds == NULL) {
     httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize);
     return false;
@@ -208,13 +212,17 @@ bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) {
 bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize) {
   HttpSqlCmds *multiCmds = pContext->multiCmds;
 
-  if (bufferSize > HTTP_MAX_BUFFER_SIZE) {
+  if (bufferSize <= 0 || bufferSize > HTTP_MAX_BUFFER_SIZE) {
     httpError("context:%p, fd:%d, user:%s, mulitcmd buffer size:%d large then %d", pContext, pContext->fd,
               pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE);
     return false;
   }
 
-  multiCmds->buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize);
+  char *new_buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize);
+  if (new_buffer == NULL && multiCmds->buffer) {
+    free(multiCmds->buffer);
+  }
+  multiCmds->buffer = new_buffer;
   if (multiCmds->buffer == NULL) {
     httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, bufferSize);
     return false;

src/query/inc/qExecutor.h

@@ -252,8 +252,10 @@ typedef struct SQueryAttr {
   int32_t          numOfFilterCols;
   int64_t*         fillVal;
   SOrderedPrjQueryInfo prjInfo;        // limit value for each vgroup, only available in global order projection query.
-  SSingleColumnFilterInfo* pFilterInfo;
 
+  SSingleColumnFilterInfo* pFilterInfo;
+  SFilterInfo     *pFilters;
+  
   void*            tsdb;
   SMemRef          memRef;
   STableGroupInfo  tableGroupInfo;       // table <tid, last_key> list  SArray<STableKeyInfo>
@@ -382,6 +384,7 @@ typedef struct SQInfo {
 typedef struct SQueryParam {
   char            *sql;
   char            *tagCond;
+  char            *colCond;
   char            *tbnameCond;
   char            *prevResult;
   SArray          *pTableIdList;
@@ -390,6 +393,8 @@ typedef struct SQueryParam {
   SExprInfo       *pExprs;
   SExprInfo       *pSecExprs;
 
+  SFilterInfo     *pFilters;
+
   SColIndex       *pGroupColIndex;
   SColumnInfo     *pTagColumnInfo;
   SGroupbyExpr *pGroupbyExpr;
@@ -573,6 +578,7 @@ SSDataBlock* doSLimit(void* param, bool* newgroup);
 
 int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);
 void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock);
+void doSetFilterColInfo(SFilterInfo     *pFilters, SSDataBlock* pBlock);
 bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p);
 void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
 
@@ -594,9 +600,11 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
 int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo,
                                            SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo);
 
+int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters);
+
 SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code);
 SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
-                        SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo);
+                        SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo);
 
 int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start,
                   int32_t prevResultLen, void* merger);

src/query/inc/qTableMeta.h

@@ -3,6 +3,7 @@
 
 #include "tsdb.h"  //todo tsdb should not be here
 #include "qSqlparser.h"
+#include "qFilter.h"
 
 typedef struct SFieldInfo {
   int16_t      numOfOutput;   // number of column in result
@@ -16,6 +17,14 @@ typedef struct SCond {
   char *   cond;
 } SCond;
 
+typedef struct STblCond {
+  uint64_t uid;
+  int16_t  idx;  //table index
+  int32_t  len;  // length of tag query condition data
+  char *   cond;
+} STblCond;
+
+
 typedef struct SJoinNode {
   uint64_t uid;
   int16_t  tagColId;
@@ -89,6 +98,11 @@ typedef struct STableMetaInfo {
 struct   SQInfo;      // global merge operator
 struct   SQueryAttr;     // query object
 
+typedef struct STableFilter {
+  uint64_t uid;
+  SFilterInfo info;
+} STableFilter;
+
 typedef struct SQueryInfo {
   int16_t          command;       // the command may be different for each subclause, so keep it seperately.
   uint32_t         type;          // query/insert type
@@ -106,8 +120,11 @@ typedef struct SQueryInfo {
   SLimitVal        slimit;
   STagCond         tagCond;
 
+  SArray *         colCond;
+
   SOrderVal        order;
   int16_t          numOfTables;
+  int16_t          curTableIdx;
   STableMetaInfo **pTableMetaInfo;
   struct STSBuf   *tsBuf;
 

src/query/src/qSqlParser.c

@@ -423,6 +423,9 @@ tSqlExpr *tSqlExprClone(tSqlExpr *pSrc) {
     pExpr->pRight = tSqlExprClone(pSrc->pRight);
   }
 
+  memset(&pExpr->value, 0, sizeof(pExpr->value));
+  tVariantAssign(&pExpr->value, &pSrc->value);
+
   //we don't clone paramList now because clone is only used for between/and
   assert(pSrc->Expr.paramList == NULL);
   return pExpr;
@@ -478,9 +481,7 @@ static void doDestroySqlExprNode(tSqlExpr *pExpr) {
     return;
   }
 
-  if (pExpr->tokenId == TK_STRING) {
-    tVariantDestroy(&pExpr->value);
-  }
+  tVariantDestroy(&pExpr->value);
 
   tSqlExprListDestroy(pExpr->Expr.paramList);
   free(pExpr);

src/query/src/queryMain.c

@@ -103,6 +103,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
     }
   }
 
+  if (param.colCond != NULL) {
+    if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, &param.pFilters)) != TSDB_CODE_SUCCESS) {
+      goto _over;
+    }
+  }
+
   param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code);
   if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) {
     goto _over;
@@ -162,13 +168,19 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
 
   assert(pQueryMsg->stableQuery == isSTableQuery);
   (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
-                              param.pTagColumnInfo, vgId, param.sql, qId, param.pUdfInfo);
+                              param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo);
 
   param.sql    = NULL;
   param.pExprs = NULL;
   param.pSecExprs = NULL;
   param.pGroupbyExpr = NULL;
   param.pTagColumnInfo = NULL;
+  param.pFilters = NULL;
+
+  if ((*pQInfo) == NULL) {
+    code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    goto _over;
+  }
   param.pUdfInfo = NULL;
 
   code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, &param, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL);
@@ -178,6 +190,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
     taosArrayDestroy(param.pGroupbyExpr->columnInfo);
   }
 
+  tfree(param.colCond);
+  
   destroyUdfInfo(param.pUdfInfo);
 
   taosArrayDestroy(param.pTableIdList);
@@ -190,6 +204,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
     freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters);
   }
 
+  filterFreeInfo(param.pFilters);
+
   //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;
   if (code != TSDB_CODE_SUCCESS) {
     *pQInfo = NULL;

src/query/tests/CMakeLists.txt

@@ -27,3 +27,4 @@ SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w)
+SET_SOURCE_FILES_PROPERTIES(./rangeMergeTest.cpp PROPERTIES COMPILE_FLAGS -w)

src/tsdb/src/tsdbCommit.c

@@ -722,7 +722,7 @@ static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) {
     return -1;
   }
 
-  pCommith->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
+  pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
   if (pCommith->pDataCols == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     tsdbDestroyCommitH(pCommith);
@@ -920,7 +920,6 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo
     SDataCol * pDataCol = pDataCols->cols + ncol;
     SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull;
 
-    // if (isNEleNull(pDataCol, rowsToWrite)) {  // all data to commit are NULL, just ignore it
     if (isAllRowsNull(pDataCol)) {  // all data to commit are NULL, just ignore it
       continue;
     }
@@ -1277,6 +1276,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
 
     if (key1 < key2) {
       for (int i = 0; i < pDataCols->numOfCols; i++) {
+        //TODO: dataColAppendVal may fail
         dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
                          pTarget->maxPoints);
       }
@@ -1308,6 +1308,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
         ASSERT(!isRowDel);
 
         for (int i = 0; i < pDataCols->numOfCols; i++) {
+          //TODO: dataColAppendVal may fail
           dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
                            pTarget->maxPoints);
         }

src/tsdb/src/tsdbCompact.c

@@ -296,7 +296,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) {
       return -1;
     }
 
-    pComph->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
+    pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
     if (pComph->pDataCols == NULL) {
       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
       tsdbDestroyCompactH(pComph);

src/tsdb/src/tsdbMemTable.c

@@ -702,11 +702,12 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
 }
 
 //row1 has higher priority
-static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, STSchema **ppSchema1, STSchema **ppSchema2, STable* pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) {
+static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo,
+                                     STSchema **ppSchema1, STSchema **ppSchema2,
+                                     STable* pTable, int32_t* pPoints, SMemRow* pLastRow) {
   
   //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows!
   if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) {
-    (*pAffectedRows)++;
     (*pPoints)++;
     return NULL;
   }
@@ -715,7 +716,6 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep
     void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1));
     if(pMem == NULL) return NULL;
     memRowCpy(pMem, row1);
-    (*pAffectedRows)++;
     (*pPoints)++;
     *pLastRow = pMem;
     return pMem;
@@ -750,18 +750,16 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep
   if(pMem == NULL) return NULL;
   memRowCpy(pMem, tmp);
 
-  (*pAffectedRows)++;
   (*pPoints)++;
-
   *pLastRow = pMem;
   return pMem;
 }
 
 static void* tsdbInsertDupKeyMergePacked(void** args) {
-  return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7], args[8]);
+  return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]);
 }
 
-static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) {
+static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, SMemRow* pLastRow) {
 
   if(pSkipList->insertHandleFn == NULL) {
     tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9);
@@ -769,17 +767,16 @@ static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STa
     dupHandleSavedFunc->args[3] = NULL;
     dupHandleSavedFunc->args[4] = NULL;
     dupHandleSavedFunc->args[5] = pTable;
-    dupHandleSavedFunc->args[6] = pAffectedRows;
-    dupHandleSavedFunc->args[7] = pPoints;
-    dupHandleSavedFunc->args[8] = pLastRow;
     pSkipList->insertHandleFn = dupHandleSavedFunc;
   }
+  pSkipList->insertHandleFn->args[6] = pPoints;
+  pSkipList->insertHandleFn->args[7] = pLastRow;
 }
 
 static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *pAffectedRows) {
 
   STsdbMeta       *pMeta = pRepo->tsdbMeta;
-  int64_t          points = 0;
+  int32_t          points = 0;
   STable          *pTable = NULL;
   SSubmitBlkIter   blkIter = {0};
   SMemTable       *pMemTable = NULL;
@@ -830,9 +827,10 @@ static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *
 
   SMemRow lastRow = NULL;
   int64_t osize = SL_SIZE(pTableData->pData);
-  tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, pAffectedRows, &points, &lastRow);
+  tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow);
   tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext);
   int64_t dsize = SL_SIZE(pTableData->pData) - osize;
+  (*pAffectedRows) += points;
 
 
   if(lastRow != NULL) {

src/tsdb/src/tsdbMeta.c

@@ -17,7 +17,6 @@
 #define TSDB_SUPER_TABLE_SL_LEVEL 5
 #define DEFAULT_TAG_INDEX_COLUMN 0
 
-static int     tsdbCompareSchemaVersion(const void *key1, const void *key2);
 static char *  getTagIndexKey(const void *pData);
 static STable *tsdbNewTable();
 static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable);

src/util/src/terror.c

@@ -280,6 +280,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC,                  "Multiple retrieval of
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW,      "Too many time window in query")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER,        "Query buffer limit has reached")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN,              "File inconsistance in replica")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION,   "One valid time range condition expected")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR,                "System error")