Merge pull request #10333 from taosdata/feature/qnode

Feature/qnode

include/common/tcompare.h

@@ -23,8 +23,12 @@
 extern "C" {
 #endif
 
-int32_t       compareStrPatternComp(const void* pLeft, const void* pRight);
-int32_t       compareWStrPatternComp(const void* pLeft, const void* pRight);
+int32_t compareStrPatternMatch(const void* pLeft, const void* pRight);
+int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight);
+
+int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight);
+int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight);
+
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char* a, const char* b, int32_t type, size_t size);

include/common/tname.h

@@ -16,6 +16,11 @@
 #ifndef TDENGINE_TNAME_H
 #define TDENGINE_TNAME_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
 #include "tdef.h"
 #include "tmsg.h"
 
@@ -59,4 +64,9 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId);
 
 SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name);
 
+#ifdef __cplusplus
+}
+#endif
+
+
 #endif  // TDENGINE_TNAME_H

include/common/ttypes.h

@@ -78,9 +78,12 @@ typedef struct {
       case TSDB_DATA_TYPE_UINT:                      \
         (_v) = (_finalType)GET_UINT32_VAL(_data);    \
         break;                                       \
-      default:                                       \
+      case TSDB_DATA_TYPE_INT:                       \
         (_v) = (_finalType)GET_INT32_VAL(_data);     \
         break;                                       \
+      default:                                       \
+        (_v) = (_finalType)varDataLen(_data);        \
+        break;                                       \
     }                                                \
   } while (0)
 
@@ -115,9 +118,11 @@ typedef struct {
       case TSDB_DATA_TYPE_UINT:                \
         *(uint32_t *)(_v) = (uint32_t)(_data); \
         break;                                 \
-      default:                                 \
+      case TSDB_DATA_TYPE_INT:                 \
         *(int32_t *)(_v) = (int32_t)(_data);   \
         break;                                 \
+      default:                                 \
+        break;                                 \
     }                                          \
   } while (0)
 
@@ -138,6 +143,9 @@ typedef struct {
 #define IS_VALID_FLOAT(_t)      ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)     ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 
+#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
+#define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL))
+
 static FORCE_INLINE bool isNull(const void *val, int32_t type) {
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
@@ -205,6 +213,7 @@ void* getDataMax(int32_t type);
 
 
 #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL)
+#define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL)
 
 #ifdef __cplusplus
 }

include/libs/function/function.h

@@ -226,13 +226,18 @@ typedef struct SAggFunctionInfo {
   int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
 } SAggFunctionInfo;
 
-struct SScalarFuncParam;
+typedef struct SScalarParam {
+  void*   data;
+  int32_t num;
+  int32_t type;
+  int32_t bytes;
+} SScalarParam;
 
 typedef struct SScalarFunctionInfo {
   char      name[FUNCTIONS_NAME_MAX_LENGTH];
   int8_t    type;              // scalar function or aggregation function
   uint32_t  functionId;        // index of scalar function
-  void     (*process)(struct SScalarFuncParam* pOutput, size_t numOfInput, const struct SScalarFuncParam *pInput);
+  void     (*process)(struct SScalarParam* pOutput, size_t numOfInput, const struct SScalarParam *pInput);
 } SScalarFunctionInfo;
 
 typedef struct SMultiFunctionsDesc {
@@ -285,10 +290,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num);
 bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry);
 bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry);
 
-struct SScalarFunctionSupport* createScalarFuncSupport(int32_t num);
-void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num);
-struct SScalarFunctionSupport* getScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t index);
-
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // fill api
 struct SFillInfo;

include/libs/function/functionMgt.h

@@ -21,6 +21,7 @@ extern "C" {
 #endif
 
 #include "querynodes.h"
+#include "function.h"
 
 typedef enum EFunctionType {
   // aggregate function
@@ -117,6 +118,13 @@ typedef struct SFuncExecFuncs {
   FExecFinalize finalize;
 } SFuncExecFuncs;
 
+typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
+typedef struct SScalarFuncExecFuncs {
+  FScalarExecProcess process;
+} SScalarFuncExecFuncs;
+
+
 int32_t fmFuncMgtInit();
 
 int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
@@ -134,6 +142,7 @@ bool fmIsTimeorderFunc(int32_t funcId);
 int32_t fmFuncScanType(int32_t funcId);
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
+int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
 
 #ifdef __cplusplus
 }

include/libs/nodes/querynodes.h

@@ -306,6 +306,7 @@ bool nodesIsJsonOp(const SOperatorNode* pOp);
 
 bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
+void *nodesGetValueFromNode(SValueNode *pNode);
 
 #ifdef __cplusplus
 }

include/libs/scalar/filter.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TDENGINE_FILTER_H
+#define TDENGINE_FILTER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SFilterInfo SFilterInfo;
+
+typedef struct SFilterColumnParam{
+  int32_t numOfCols;
+  SArray* pDataBlock;
+} SFilterColumnParam;
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_FILTER_H
\ No newline at end of file

include/libs/scalar/scalar.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TDENGINE_SCALAR_H
+#define TDENGINE_SCALAR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "function.h"
+#include "nodes.h"
+#include "querynodes.h"
+
+typedef struct SFilterInfo SFilterInfo;
+
+
+int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
+int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
+int32_t scalarGetOperatorParamNum(EOperatorType type);
+
+int32_t vectorGetConvertType(int32_t type1, int32_t type2);
+int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_SCALAR_H

include/util/compare.h

@@ -49,10 +49,19 @@ int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size,
 
 int32_t taosArrayCompareString(const void *a, const void *b);
 
-int32_t setCompareBytes1(const void *pLeft, const void *pRight);
-int32_t setCompareBytes2(const void *pLeft, const void *pRight);
-int32_t setCompareBytes4(const void *pLeft, const void *pRight);
-int32_t setCompareBytes8(const void *pLeft, const void *pRight);
+int32_t setChkInBytes1(const void *pLeft, const void *pRight);
+int32_t setChkInBytes2(const void *pLeft, const void *pRight);
+int32_t setChkInBytes4(const void *pLeft, const void *pRight);
+int32_t setChkInBytes8(const void *pLeft, const void *pRight);
+
+int32_t setChkNotInBytes1(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes2(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes4(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes8(const void *pLeft, const void *pRight);
+
+int32_t compareChkInString(const void *pLeft, const void *pRight);
+int32_t compareChkNotInString(const void *pLeft, const void *pRight);
+
 
 int32_t compareInt8Val(const void *pLeft, const void *pRight);
 int32_t compareInt16Val(const void *pLeft, const void *pRight);
@@ -74,7 +83,6 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight);
 int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight);
 int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight);
 
-int32_t compareFindItemInSet(const void *pLeft, const void *pRight);
 
 int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
 int32_t compareInt16ValDesc(const void *pLeft, const void *pRight);
@@ -92,6 +100,9 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight);
 
+__compar_fn_t getComparFunc(int32_t type, int32_t optr);
+
+
 #ifdef __cplusplus
 }
 #endif

include/util/tdef.h

@@ -118,16 +118,18 @@ do { \
 #define TSDB_RELATION_GREATER_EQUAL 5
 #define TSDB_RELATION_NOT_EQUAL   6
 #define TSDB_RELATION_LIKE        7
-#define TSDB_RELATION_ISNULL      8
-#define TSDB_RELATION_NOTNULL     9
-#define TSDB_RELATION_IN          10
-
-#define TSDB_RELATION_AND         11
-#define TSDB_RELATION_OR          12
-#define TSDB_RELATION_NOT         13
-
-#define TSDB_RELATION_MATCH       14
-#define TSDB_RELATION_NMATCH      15
+#define TSDB_RELATION_NOT_LIKE    8
+#define TSDB_RELATION_ISNULL      9
+#define TSDB_RELATION_NOTNULL     10
+#define TSDB_RELATION_IN          11
+#define TSDB_RELATION_NOT_IN      12
+
+#define TSDB_RELATION_AND         13
+#define TSDB_RELATION_OR          14
+#define TSDB_RELATION_NOT         15
+
+#define TSDB_RELATION_MATCH       16
+#define TSDB_RELATION_NMATCH      17
 
 #define TSDB_BINARY_OP_ADD        4000
 #define TSDB_BINARY_OP_SUBTRACT   4001

include/util/thash.h

@@ -55,6 +55,8 @@ uint32_t taosIntHash_64(const char *key, uint32_t len);
 
 _hash_fn_t taosGetDefaultHashFunction(int32_t type);
 
+_equal_fn_t taosGetDefaultEqualFunction(int32_t type);
+
 typedef struct SHashNode {
   struct SHashNode *next;
   uint32_t          hashVal;     // the hash value of key
@@ -258,6 +260,8 @@ void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen);
  */
 void taosHashRelease(SHashObj *pHashObj, void *p);
 
+void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
+
 
 #ifdef __cplusplus
 }

include/util/types.h

@@ -66,6 +66,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
 // #else
   #define GET_FLOAT_VAL(x)       (*(float *)(x))
   #define GET_DOUBLE_VAL(x)      (*(double *)(x))
+  #define SET_BIGINT_VAL(x, y)  { (*(int64_t *)(x))  = (int64_t)(y);       }
   #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
   #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
   #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }

source/common/src/tcompare.c

@@ -15,116 +15,6 @@
 
 #include "tcompare.h"
 
-int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
-  SPatternCompareInfo pInfo = {'%', '_'};
-
-  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
-  char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char));
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-
-  size_t sz = varDataLen(pLeft);
-  char *buf = malloc(sz + 1);
-  memcpy(buf, varDataVal(pLeft), sz);
-  buf[sz] = 0;
-
-  int32_t ret = patternMatch(pattern, buf, sz, &pInfo);
-  free(buf);
-  free(pattern);
-  return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
-}
-
-int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
-  SPatternCompareInfo pInfo = {'%', '_'};
-
-  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
-
-  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-
-  int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
-  free(pattern);
-
-  return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
-}
-
-__compar_fn_t getComparFunc(int32_t type, int32_t optr) {
-  __compar_fn_t comparFn = NULL;
-
-  if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
-    switch (type) {
-      case TSDB_DATA_TYPE_BOOL:
-      case TSDB_DATA_TYPE_TINYINT:
-      case TSDB_DATA_TYPE_UTINYINT:
-        return setCompareBytes1;
-      case TSDB_DATA_TYPE_SMALLINT:
-      case TSDB_DATA_TYPE_USMALLINT:
-        return setCompareBytes2;
-      case TSDB_DATA_TYPE_INT:
-      case TSDB_DATA_TYPE_UINT:
-      case TSDB_DATA_TYPE_FLOAT:
-        return setCompareBytes4;
-      case TSDB_DATA_TYPE_BIGINT:
-      case TSDB_DATA_TYPE_UBIGINT:
-      case TSDB_DATA_TYPE_DOUBLE:
-      case TSDB_DATA_TYPE_TIMESTAMP:
-        return setCompareBytes8;
-      default:
-        assert(0);
-    }
-  }
-
-  switch (type) {
-    case TSDB_DATA_TYPE_BOOL:
-    case TSDB_DATA_TYPE_TINYINT:   comparFn = compareInt8Val;   break;
-    case TSDB_DATA_TYPE_SMALLINT:  comparFn = compareInt16Val;  break;
-    case TSDB_DATA_TYPE_INT:       comparFn = compareInt32Val;  break;
-    case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val;  break;
-    case TSDB_DATA_TYPE_FLOAT:     comparFn = compareFloatVal;  break;
-    case TSDB_DATA_TYPE_DOUBLE:    comparFn = compareDoubleVal; break;
-    case TSDB_DATA_TYPE_BINARY: {
-      if (optr == TSDB_RELATION_MATCH) {
-        comparFn = compareStrRegexCompMatch;
-      } else if (optr == TSDB_RELATION_NMATCH) {
-        comparFn = compareStrRegexCompNMatch;
-      } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
-        comparFn = compareStrPatternComp;
-      } else if (optr == TSDB_RELATION_IN) {
-        comparFn = compareFindItemInSet;
-      } else { /* normal relational comparFn */
-        comparFn = compareLenPrefixedStr;
-      }
-
-      break;
-    }
-
-    case TSDB_DATA_TYPE_NCHAR: {
-      if (optr == TSDB_RELATION_MATCH) {
-        comparFn = compareStrRegexCompMatch;
-      } else if (optr == TSDB_RELATION_NMATCH) {
-        comparFn = compareStrRegexCompNMatch;
-      } else if (optr == TSDB_RELATION_LIKE) {
-        comparFn = compareWStrPatternComp;
-      } else if (optr == TSDB_RELATION_IN) {
-        comparFn = compareFindItemInSet;
-      } else {
-        comparFn = compareLenPrefixedWStr;
-      }
-      break;
-    }
-
-    case TSDB_DATA_TYPE_UTINYINT:  comparFn = compareUint8Val; break;
-    case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break;
-    case TSDB_DATA_TYPE_UINT:      comparFn = compareUint32Val;break;
-    case TSDB_DATA_TYPE_UBIGINT:   comparFn = compareUint64Val;break;
-
-    default:
-      comparFn = compareInt32Val;
-      break;
-  }
-
-  return comparFn;
-}
 
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) {
   __compar_fn_t comparFn = NULL;

source/libs/CMakeLists.txt

@@ -14,3 +14,4 @@ add_subdirectory(qcom)
 add_subdirectory(qworker)
 add_subdirectory(tfs)
 add_subdirectory(nodes)
+add_subdirectory(scalar)
\ No newline at end of file

source/libs/executor/CMakeLists.txt

@@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC})
 #        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor"
 #        )
 target_link_libraries(executor
-            PRIVATE os util common function parser planner qcom vnode
+            PRIVATE os util common function parser planner qcom vnode scalar nodes
         )
 
 target_include_directories(

source/libs/executor/inc/executorimpl.h

@@ -29,9 +29,9 @@ extern "C" {
 #include "executil.h"
 #include "executor.h"
 #include "planner.h"
+#include "scalar.h"
 #include "taosdef.h"
 #include "tarray.h"
-#include "tfilter.h"
 #include "thash.h"
 #include "tlockfree.h"
 #include "tpagedbuf.h"

source/libs/executor/src/executorimpl.c

@@ -2148,7 +2148,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT
   // NOTE: pTableCheckInfo need to update the query time range and the lastKey info
   pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
 
-  pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput);
+  //pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput);
 
   if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL ||
       pRuntimeEnv->prevRow == NULL  || pRuntimeEnv->tagVal == NULL) {
@@ -2174,7 +2174,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT
   return TSDB_CODE_SUCCESS;
 
 _clean:
-  destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput);
+  //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput);
   tfree(pRuntimeEnv->pResultRowHashTable);
   tfree(pRuntimeEnv->keyBuf);
   tfree(pRuntimeEnv->prevRow);
@@ -2212,7 +2212,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) {
 
   //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId);
 
-  destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput);
+  //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput);
 //  destroyUdfInfo(pRuntimeEnv->pUdfInfo);
   destroyResultBuf(pRuntimeEnv->pResultBuf);
   doFreeQueryHandle(pRuntimeEnv);

source/libs/executor/test/CMakeLists.txt

@@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
 TARGET_LINK_LIBRARIES(
         executorTest
-        PUBLIC os util common transport gtest taos qcom executor function planner
+        PRIVATE os util common transport gtest taos qcom executor function planner scalar nodes
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/libs/executor/test/executorTests.cpp

@@ -32,6 +32,8 @@
 #include "trpc.h"
 #include "stub.h"
 #include "executor.h"
+#include "tmsg.h"
+#include "tname.h"
 
 namespace {
 

source/libs/function/inc/tunaryoperator.h

@@ -20,10 +20,10 @@
 extern "C" {
 #endif
 
-#include "tscalarfunction.h"
+//#include "tscalarfunction.h"
 
-typedef void (*_unary_scalar_fn_t)(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput);
-_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator);
+//typedef void (*_unary_scalar_fn_t)(SScalarParam *pLeft, SScalarParam* pOutput);
+//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator);
 
 #ifdef __cplusplus
 }

source/libs/function/src/taggfunction.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tscalarfunction.h"
 #include "os.h"
 #include "taosdef.h"
 #include "tmsg.h"
@@ -3221,6 +3220,7 @@ static void diff_function(SqlFunctionCtx *pCtx) {
   }
 }
 
+#if 0
 char *getArithColumnData(void *param, const char* name, int32_t colId) {
   SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param;
   
@@ -3235,15 +3235,16 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) {
   assert(index >= 0);
   return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes;
 }
+#endif
 
 static void arithmetic_function(SqlFunctionCtx *pCtx) {
   GET_RES_INFO(pCtx)->numOfRes += pCtx->size;
-  SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz;
+  //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz;
 
-  SScalarFuncParam output = {0};
+  SScalarParam output = {0};
   output.data = pCtx->pOutput;
 
-  evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData);
+  //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData);
 }
 
 #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \

source/libs/function/src/tfunction.c

@@ -3,7 +3,6 @@
 #include "function.h"
 #include "thash.h"
 #include "taggfunction.h"
-#include "tscalarfunction.h"
 
 static SHashObj* functionHashTable = NULL;
 static SHashObj* udfHashTable = NULL;
@@ -18,12 +17,14 @@ static void doInitFunctionHashTable() {
     taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&ptr, POINTER_BYTES);
   }
 
+/*
   numOfEntries = tListLen(scalarFunc);
   for(int32_t i = 0; i < numOfEntries; ++i) {
     int32_t len = (int32_t) strlen(scalarFunc[i].name);
     SScalarFunctionInfo* ptr = &scalarFunc[i];
     taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES);
   }
+*/
 
   udfHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, true);
 }

source/libs/function/src/tunaryoperator.c

@@ -4,10 +4,10 @@
 
 
 // TODO dynamic define these functions
-_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) {
-  assert(0);
-}
+//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) {
+//  assert(0);
+//}
 
-bool isStringOperatorFn(int32_t op) {
-  return op == FUNCTION_LENGTH;
-}
+//bool isStringOperatorFn(int32_t op) {
+//  return op == FUNCTION_LENGTH;
+//}

source/libs/nodes/src/nodesUtilFuncs.c

@@ -17,6 +17,7 @@
 #include "plannodes.h"
 #include "taos.h"
 #include "taoserror.h"
+#include "taos.h"
 #include "thash.h"
 
 static SNode* makeNode(ENodeType type, size_t size) {
@@ -172,6 +173,36 @@ void nodesDestroyList(SNodeList* pList) {
   tfree(pList);
 }
 
+void *nodesGetValueFromNode(SValueNode *pNode) {
+  switch (pNode->node.resType.type) {
+    case TSDB_DATA_TYPE_BOOL:
+      return (void *)&pNode->datum.b;
+    case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      return (void *)&pNode->datum.i;
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_UBIGINT:
+      return (void *)&pNode->datum.u;
+    case TSDB_DATA_TYPE_FLOAT:
+    case TSDB_DATA_TYPE_DOUBLE: 
+      return (void *)&pNode->datum.d;
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+    case TSDB_DATA_TYPE_VARCHAR:
+    case TSDB_DATA_TYPE_VARBINARY: 
+      return (void *)pNode->datum.p;
+    default:
+      break;
+  }
+
+  return NULL;
+}
+
 bool nodesIsExprNode(const SNode* pNode) {
   ENodeType type = nodeType(pNode);
   return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type);

source/libs/scalar/CMakeLists.txt

+aux_source_directory(src SCALAR_SRC)
+
+add_library(scalar STATIC ${SCALAR_SRC})
+target_include_directories(
+    scalar
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_link_libraries(scalar
+        PRIVATE os util common nodes function qcom
+        )
+
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})
\ No newline at end of file

source/libs/executor/inc/tfilter.h → source/libs/scalar/inc/filterInt.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_QFILTER_H
-#define TDENGINE_QFILTER_H
+#ifndef TDENGINE_FILTER_INT_H
+#define TDENGINE_FILTER_INT_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -22,7 +22,10 @@ extern "C" {
 
 #include "thash.h"
 #include "tname.h"
-#include "function.h"
+#include "common.h"
+#include "scalar.h"
+#include "querynodes.h"
+#include "query.h"
 
 #define FILTER_DEFAULT_GROUP_SIZE 4
 #define FILTER_DEFAULT_UNIT_SIZE 4
@@ -83,6 +86,12 @@ enum {
   RANGE_TYPE_MR_CTX = 3,
 };
 
+enum {
+  FI_ACTION_NO_NEED = 1,
+  FI_ACTION_CONTINUE,
+  FI_ACTION_STOP,
+};
+
 typedef struct OptrStr {
   uint16_t optr;
   char    *str;
@@ -95,17 +104,11 @@ typedef struct SFilterRange {
   char eflag;
 } SFilterRange;
 
-typedef struct SFilterColRange {  
-  uint16_t idx;  //column field idx
-  bool isNull;
-  bool notNull;
-  bool isRange;
-  SFilterRange ra;
-} SFilterColRange;
-
 typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
 typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
 typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t);
+typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
+typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **);
 
 typedef struct SFilterRangeCompare {
   int64_t s;
@@ -153,37 +156,39 @@ typedef struct SFilterField {
 } SFilterField;
 
 typedef struct SFilterFields {
-  uint16_t size;
-  uint16_t num;
+  uint32_t size;
+  uint32_t num;
   SFilterField *fields;
 } SFilterFields;
 
 typedef struct SFilterFieldId {
   uint16_t type;
-  uint16_t idx;
+  uint32_t idx;
 } SFilterFieldId;
 
 typedef struct SFilterGroup {
-  uint16_t  unitSize;
-  uint16_t  unitNum;
-  uint16_t *unitIdxs;
+  uint32_t  unitSize;
+  uint32_t  unitNum;
+  uint32_t *unitIdxs;
   uint8_t  *unitFlags;  // !unit result
 } SFilterGroup;
 
 typedef struct SFilterColInfo {
   uint8_t type;
   int32_t dataType;
+  uint8_t optr;   // for equal operation in the relation of RELATION_IN
+  int64_t value;  // for equal operation in the relation of RELATION_IN
   void   *info;
 } SFilterColInfo;
 
 typedef struct SFilterGroupCtx {
-  uint16_t         colNum;
-  uint16_t        *colIdx;
+  uint32_t         colNum;
+  uint32_t        *colIdx;
   SFilterColInfo  *colInfo;
 } SFilterGroupCtx;
 
 typedef struct SFilterColCtx {
-  uint16_t  colIdx;
+  uint32_t  colIdx;
   void*     ctx;
 } SFilterColCtx;
 
@@ -217,16 +222,32 @@ typedef struct SFilterPCtx {
   SHashObj *unitHash;
 } SFilterPCtx;
 
+typedef struct SFltTreeStat {
+  int32_t code;
+  bool    scalarMode;
+} SFltTreeStat;
+
+typedef struct SFltScalarCtx {
+  SNode *node;
+} SFltScalarCtx;
+
+typedef struct SFltBuildGroupCtx {
+  SFilterInfo *info;
+  SArray      *group;
+  int32_t      code;
+} SFltBuildGroupCtx;
+
 typedef struct SFilterInfo {
+  bool              scalarMode;
+  SFltScalarCtx     sclCtx;
   uint32_t          options;
   uint32_t          status;  
-  uint16_t          unitSize;
-  uint16_t          unitNum;
-  uint16_t          groupNum;
-  uint16_t          colRangeNum;
+  uint32_t          unitSize;
+  uint32_t          unitNum;
+  uint32_t          groupNum;
+  uint32_t          colRangeNum;
   SFilterFields     fields[FLD_TYPE_MAX];
   SFilterGroup     *groups;
-  uint16_t         *cgroups;
   SFilterUnit      *units;
   SFilterComUnit   *cunits;
   uint8_t          *unitRes;    // result
@@ -234,16 +255,15 @@ typedef struct SFilterInfo {
   SFilterRangeCtx **colRange;
   filter_exec_func  func;          
   uint8_t           blkFlag;
-  uint16_t          blkGroupNum;
-  uint16_t         *blkUnits;
+  uint32_t          blkGroupNum;
+  uint32_t         *blkUnits;
   int8_t           *blkUnitRes;
+  void             *pTable;
 
   SFilterPCtx       pctx;
 } SFilterInfo;
 
-#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t))
-
-#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR)
+#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON)
 #define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
 
 #define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
@@ -259,7 +279,7 @@ typedef struct SFilterInfo {
 #define FILTER_CLR_FLAG(st, f) st &= (~f)
 
 #define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src)
-#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0)
+#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint32_t *)(_t + 1) = idx1; *(uint32_t *)(_t + 3) = idx2; } while (0)
 #define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE))))
 #define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0)
 
@@ -269,25 +289,33 @@ typedef struct SFilterInfo {
 #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
 #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
 
-#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
-#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
-#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
+#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_NULL)
+
+#define fltFatal(...)  qFatal(__VA_ARGS__)
+#define fltError(...)  qError(__VA_ARGS__)
+#define fltWarn(...)   qWarn(__VA_ARGS__)
+#define fltInfo(...)   qInfo(__VA_ARGS__)
+#define fltDebug(...)  qDebug(__VA_ARGS__)
+#define fltTrace(...)  qTrace(__VA_ARGS__)
+
+
+#define FLT_CHK_JMP(c) do { if (c) { goto _return; } } while (0)
+#define FLT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define FLT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define FLT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
 
-#define CHK_RETV(c) do { if (c) { return; } } while (0)
-#define CHK_RET(c, r) do { if (c) { return r; } } while (0)
-#define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
-#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
-#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0)
 
 #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
 #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
-#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type)
-#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes)
-#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId)
-#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc))
-#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
-#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
+#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type)
+#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes)
+#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId)
+#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId)
+#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc))
+#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri))
+#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
 #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
+#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
 #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
 
 #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
@@ -300,6 +328,7 @@ typedef struct SFilterInfo {
 #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
+#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
 #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
 #define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
 #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
@@ -314,7 +343,7 @@ typedef struct SFilterInfo {
 #define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0)
 #define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0)
 
-#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0)
+#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0)
 
 #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) 
 
@@ -322,18 +351,25 @@ typedef struct SFilterInfo {
 #define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL)
 #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
 
-
-extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
+#if 0
+extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options);
 extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
-extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock);
+extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp);
+extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp);
 extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
 extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
 extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
 extern void filterFreeInfo(SFilterInfo *info);
 extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
+#else
+//REMOVE THESE!!!!!!!!!!!!!!!!!!!!
+#include "function.h"
+#endif
+extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
+extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_QFILTER_H
+#endif  // TDENGINE_FILTER_INT_H

source/libs/scalar/inc/sclInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TDENGINE_SCALARINT_H
+#define TDENGINE_SCALARINT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "common.h"
+#include "thash.h"
+#include "query.h"
+
+typedef struct SScalarCtx {
+  int32_t      code;
+  SSDataBlock *pSrc; 
+  SHashObj    *pRes;  /* element is SScalarParam */
+} SScalarCtx;
+
+#define SCL_DEFAULT_OP_NUM 10
+
+#define sclFatal(...)  qFatal(__VA_ARGS__)
+#define sclError(...)  qError(__VA_ARGS__)
+#define sclWarn(...)   qWarn(__VA_ARGS__)
+#define sclInfo(...)   qInfo(__VA_ARGS__)
+#define sclDebug(...)  qDebug(__VA_ARGS__)
+#define sclTrace(...)  qTrace(__VA_ARGS__)
+
+#define SCL_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define SCL_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+
+
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_SCALARINT_H
\ No newline at end of file

source/libs/function/inc/tscalarfunction.h → source/libs/scalar/inc/sclfunc.h

@@ -20,13 +20,7 @@ extern "C" {
 #endif
 
 #include "function.h"
-
-typedef struct SScalarFuncParam {
-  void*   data;
-  int32_t num;
-  int32_t type;
-  int32_t bytes;
-} SScalarFuncParam;
+#include "scalar.h"
 
 typedef struct SScalarFunctionSupport {
   struct SExprInfo   *pExprInfo;
@@ -39,7 +33,7 @@ typedef struct SScalarFunctionSupport {
 
 extern struct SScalarFunctionInfo scalarFunc[8];
 
-int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput,
+int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput,
                           void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t));
 
 

source/libs/function/inc/tbinoperator.h → source/libs/scalar/inc/sclvector.h

@@ -20,9 +20,9 @@
 extern "C" {
 #endif
 
-#include "tscalarfunction.h"
+#include "sclfunc.h"
 
-typedef void (*_bin_scalar_fn_t)(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *output, int32_t order);
+typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order);
 _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator);
 bool isBinaryStringOp(int32_t op);
 

source/libs/executor/src/tfilter.c → source/libs/scalar/src/filter.c

@@ -17,7 +17,8 @@
 #include "thash.h"
 //#include "queryLog.h"
 #include "tcompare.h"
-#include "tfilter.h"
+#include "filterInt.h"
+#include "filter.h"
 
 OptrStr gOptrStr[] = {
   {TSDB_RELATION_INVALID,                  "invalid"},
@@ -28,48 +29,35 @@ OptrStr gOptrStr[] = {
   {TSDB_RELATION_GREATER_EQUAL,            ">="},
   {TSDB_RELATION_NOT_EQUAL,                "!="},
   {TSDB_RELATION_LIKE,                     "like"},
-  {TSDB_RELATION_MATCH,                    "match"},
-  {TSDB_RELATION_MATCH,                    "nmatch"},
   {TSDB_RELATION_ISNULL,                   "is null"},
   {TSDB_RELATION_NOTNULL,                  "not null"},
   {TSDB_RELATION_IN,                       "in"},
   {TSDB_RELATION_AND,                      "and"},
   {TSDB_RELATION_OR,                       "or"},
-  {TSDB_RELATION_NOT,                      "not"}
-};
-
-static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) {
-  const SSchema *sch1 = desc1;
-  const SSchema *sch2 = desc2;
-
-  return sch1->colId != sch2->colId;
-}
-
-static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) {
-  const SVariant *val1 = desc1;
-  const SVariant *val2 = desc2;
-
-  return taosVariantCompare(val1, val2);
-}
-
-
-filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = {
-  NULL,
-  filterFieldColDescCompare,
-  filterFieldValDescCompare
+  {TSDB_RELATION_NOT,                      "not"},
+  {TSDB_RELATION_MATCH,                    "match"},
+  {TSDB_RELATION_NMATCH,                   "nmatch"}
 };
 
 bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
-  return cfunc(maxv, minr) >= 0;
+  int32_t result = cfunc(maxv, minr);
+  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+  return result >= 0;
 }
 bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
-  return cfunc(maxv, minr) > 0;
+  int32_t result = cfunc(maxv, minr);
+  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+  return result > 0;
 }
 bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
-  return cfunc(minv, maxr) <= 0;
+  int32_t result = cfunc(minv, maxr);
+  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+  return result <= 0;
 }
 bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
-  return cfunc(minv, maxr) < 0;
+  int32_t result = cfunc(minv, maxr);
+  //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+  return result < 0;
 }
 bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
   return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0;
@@ -156,9 +144,11 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) {
 }
 
 __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal,
-  compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, 
+  compareDoubleVal, compareLenPrefixedStr, compareStrPatternMatch, compareChkInString, compareWStrPatternMatch, 
   compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
-  setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch
+  setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, 
+  compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
+  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch
 };
 
 int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@@ -187,6 +177,29 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
     }
   }
 
+  if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
+    switch (type) {
+      case TSDB_DATA_TYPE_BOOL:
+      case TSDB_DATA_TYPE_TINYINT:  
+      case TSDB_DATA_TYPE_UTINYINT:  
+        return 21;
+      case TSDB_DATA_TYPE_SMALLINT:
+      case TSDB_DATA_TYPE_USMALLINT:
+        return 22;
+      case TSDB_DATA_TYPE_INT:
+      case TSDB_DATA_TYPE_UINT:
+      case TSDB_DATA_TYPE_FLOAT:        
+        return 23;
+      case TSDB_DATA_TYPE_BIGINT:        
+      case TSDB_DATA_TYPE_UBIGINT:        
+      case TSDB_DATA_TYPE_DOUBLE:        
+      case TSDB_DATA_TYPE_TIMESTAMP:        
+        return 24;
+      default:
+        assert(0);
+    }
+  }
+  
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:   comparFn = 1;   break;
@@ -203,8 +216,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         comparFn = 20;
       } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
         comparFn = 7;
+      } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */
+        comparFn = 26;
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = 8;
+      } else if (optr == TSDB_RELATION_NOT_IN) {
+        comparFn = 25;
       } else { /* normal relational comparFn */
         comparFn = 6;
       }
@@ -219,8 +236,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
         comparFn = 20;
       } else if (optr == TSDB_RELATION_LIKE) {
         comparFn = 9;
+      } else if (optr == TSDB_RELATION_LIKE) {
+        comparFn = 27;
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = 8;
+      } else if (optr == TSDB_RELATION_NOT_IN) {
+        comparFn = 25;
       } else {
         comparFn = 10;
       }
@@ -240,6 +261,10 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
   return comparFn;
 }
 
+__compar_fn_t filterGetCompFunc(int32_t type, int32_t optr) {
+  return gDataCompare[filterGetCompFuncIdx(type, optr)];
+}
+
 
 static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) {
   SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight);
@@ -254,7 +279,7 @@ int32_t filterInitUnitsFields(SFilterInfo *info) {
   
   info->fields[FLD_TYPE_COLUMN].num = 0;
   info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE;
-  info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE);
+  info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField));
   info->fields[FLD_TYPE_VALUE].num = 0;
   info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE;
   info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField));
@@ -281,7 +306,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt
 
 void* filterInitRangeCtx(int32_t type, int32_t options) {
   if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-    //qError("not supported range type:%d", type);
+    qError("not supported range type:%d", type);
     return NULL;
   }
   
@@ -648,7 +673,7 @@ int32_t filterGetRangeRes(void* h, SFilterRange *ra) {
   }
 
   if (num == 0) {
-    //qError("no range result");
+    qError("no range result");
     return TSDB_CODE_QRY_APP_ERROR;
   }
   
@@ -738,8 +763,15 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) {
   int32_t leftSize = (int32_t)taosArrayGetSize(left);
   int32_t rightSize = (int32_t)taosArrayGetSize(right);
 
-//  CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group");
-//  CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group");
+  if (taosArrayGetSize(left) <= 0) {
+    fltDebug("empty group");
+    FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  if (taosArrayGetSize(right) <= 0) { 
+    fltDebug("empty group");
+    FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
   
   for (int32_t l = 0; l < leftSize; ++l) {
     SFilterGroup *gp1 = taosArrayGet(left, l);
@@ -756,8 +788,8 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) {
 }
 
 int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) {
-  for (uint16_t i = 0; i < fields->num; ++i) {
-    if (0 == gDescCompare[type](fields->fields[i].desc, v)) {
+  for (uint32_t i = 0; i < fields->num; ++i) {
+    if (nodesEqualNode(fields->fields[i].desc, v)) {
       return i;
     }
   }
@@ -769,7 +801,7 @@ int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) {
 int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) {
   if (type == FLD_TYPE_VALUE) {
     if (info->pctx.valHash == false) {
-      //qError("value hash is empty");
+      qError("value hash is empty");
       return -1;
     }
 
@@ -782,10 +814,11 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d
   return -1;
 }
 
-
+// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0
+// Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better.
 int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) {
   int32_t idx = -1;
-  uint16_t *num;
+  uint32_t *num;
 
   num = &info->fields[type].num;
 
@@ -847,21 +880,25 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte
 }
 
 
-int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) {
-//  CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node");
-//  CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR);
+int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *fid) {
+  if (node == NULL) {
+    fltError("empty node");
+    FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+  
+  if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE) {
+    FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
   
   int32_t type;
   void *v;
 
-  if (node->nodeType == TEXPR_BINARYEXPR_NODE) {
+  if (nodeType(node) == QUERY_NODE_COLUMN_REF) {
     type = FLD_TYPE_COLUMN;
-    v = node->pSchema;
-    node->pSchema = NULL;
+    v = node;
   } else {
     type = FLD_TYPE_VALUE;
-    v = node->pVal;
-    node->pVal = NULL;
+    v = node;
   }
 
   filterAddField(info, v, NULL, type, fid, 0, true);
@@ -869,7 +906,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) {
+int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) {
   if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) {
     if (info->pctx.unitHash == NULL) {
       info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false);
@@ -878,14 +915,14 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
       FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1);
       void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v));
       if (hu) {
-        *uidx = *(uint16_t *)hu;
+        *uidx = *(uint32_t *)hu;
         return TSDB_CODE_SUCCESS;
       }
     }
   }
 
   if (info->unitNum >= info->unitSize) {
-    uint16_t psize = info->unitSize;
+    uint32_t psize = info->unitSize;
     info->unitSize += FILTER_DEFAULT_UNIT_SIZE;
     info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit));
     memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE);
@@ -903,7 +940,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
     SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u);  
     assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE));
   } else {
-    assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR);
+    if(optr != TSDB_RELATION_ISNULL && optr != TSDB_RELATION_NOTNULL && optr != FILTER_DUMMY_EMPTY_OPTR){
+      return -1;
+    }
   }
   
   SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u);
@@ -926,7 +965,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
 
 
 
-int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) {
+int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) {
   if (group->unitNum >= group->unitSize) {
     group->unitSize += FILTER_DEFAULT_UNIT_SIZE;
     group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs));
@@ -937,257 +976,37 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) {
-  SBufferReader br = tbufInitReader(buf, len, false); 
-  uint32_t sType  = tbufReadUint32(&br);     
-  SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false);
-  int32_t code = 0;
-
-  int dummy = -1;
-  SVariant tmpVar = {0};  
-  size_t  t = 0;
-  int32_t sz = tbufReadInt32(&br);
-  void *pvar = NULL;  
-  int64_t val = 0;
-  int32_t bufLen = 0;
-  if (IS_NUMERIC_TYPE(sType)) {
-    bufLen = 60;  // The maximum length of string that a number is converted to.
-  } else {
-    bufLen = 128;
-  }
-
-  char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE);
-    
-  for (int32_t i = 0; i < sz; i++) {
-    switch (sType) {
-    case TSDB_DATA_TYPE_BOOL:
-    case TSDB_DATA_TYPE_UTINYINT:
-    case TSDB_DATA_TYPE_TINYINT: {
-      *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); 
-      t = sizeof(val);
-      pvar = &val;
-      break;
-    }
-    case TSDB_DATA_TYPE_USMALLINT:
-    case TSDB_DATA_TYPE_SMALLINT: {
-      *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); 
-      t = sizeof(val);
-      pvar = &val;
-      break;
-    }
-    case TSDB_DATA_TYPE_UINT:
-    case TSDB_DATA_TYPE_INT: {
-      *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); 
-      t = sizeof(val);
-      pvar = &val;
-      break;
-    }
-    case TSDB_DATA_TYPE_TIMESTAMP:
-    case TSDB_DATA_TYPE_UBIGINT:
-    case TSDB_DATA_TYPE_BIGINT: {
-      *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); 
-      t = sizeof(val);
-      pvar = &val;
-      break;
-    }
-    case TSDB_DATA_TYPE_DOUBLE: {
-      *(double *)&val = tbufReadDouble(&br);
-      t = sizeof(val);
-      pvar = &val;
-      break;
-    }
-    case TSDB_DATA_TYPE_FLOAT: {
-      *(float *)&val = (float)tbufReadDouble(&br);
-      t = sizeof(val);
-      pvar = &val;
-      break;
-    }
-    case TSDB_DATA_TYPE_BINARY: {
-      pvar = (char *)tbufReadBinary(&br, &t);
-      break;
-    }
-    case TSDB_DATA_TYPE_NCHAR: {
-      pvar = (char *)tbufReadBinary(&br, &t);      
-      break;
-    }
-    default:
-      taosHashCleanup(pObj);
-      *q = NULL;
-      assert(0);
-    }
-    
-    taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType);
-
-    if (bufLen < t) {
-      tmp = realloc(tmp, t * TSDB_NCHAR_SIZE);
-      bufLen = (int32_t)t;
-    }
-
-    bool converted = false;
-    char extInfo = 0;
-
-    switch (tType) {
-      case TSDB_DATA_TYPE_BOOL:
-      case TSDB_DATA_TYPE_UTINYINT:
-      case TSDB_DATA_TYPE_TINYINT: {
-//        if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
-//          if (converted) {
-//            taosVariantDestroy(&tmpVar);
-//            memset(&tmpVar, 0, sizeof(tmpVar));
-//            continue;
-//          }
-//
-//          goto _return;
-//        }
-        pvar = &val;
-        t = sizeof(val);
-        break;
-      }
-      case TSDB_DATA_TYPE_USMALLINT:
-      case TSDB_DATA_TYPE_SMALLINT: {
-//        if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
-//          if (converted) {
-//            taosVariantDestroy(&tmpVar);
-//            memset(&tmpVar, 0, sizeof(tmpVar));
-//            continue;
-//          }
-//
-//          goto _return;
-//        }
-        pvar = &val;
-        t = sizeof(val);
-        break;
-      }
-      case TSDB_DATA_TYPE_UINT:
-      case TSDB_DATA_TYPE_INT: {
-//        if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
-//          if (converted) {
-//            taosVariantDestroy(&tmpVar);
-//            memset(&tmpVar, 0, sizeof(tmpVar));
-//            continue;
-//          }
-//
-//          goto _return;
-//        }
-        pvar = &val;
-        t = sizeof(val);
-        break;
-      }
-      case TSDB_DATA_TYPE_TIMESTAMP:
-      case TSDB_DATA_TYPE_UBIGINT:
-      case TSDB_DATA_TYPE_BIGINT: {
-        if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) {
-          goto _return;
-        }
-        pvar = &val;
-        t = sizeof(val);
-        break;
-      }
-      case TSDB_DATA_TYPE_DOUBLE: {
-        if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) {
-          goto _return;
-        }
-        pvar = &val;
-        t = sizeof(val);
-        break;
-      }
-      case TSDB_DATA_TYPE_FLOAT: {
-//        if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
-//          if (converted) {
-//            taosVariantDestroy(&tmpVar);
-//            memset(&tmpVar, 0, sizeof(tmpVar));
-//            continue;
-//          }
-          
-//          goto _return;
-//        }
-        pvar = &val;
-        t = sizeof(val);
-        break;
-      }
-      case TSDB_DATA_TYPE_BINARY: {
-        if (taosVariantDump(&tmpVar, tmp, tType, true)) {
-          goto _return;
-        }
-        t = varDataLen(tmp);
-        pvar = varDataVal(tmp);
-        break;
-      }
-      case TSDB_DATA_TYPE_NCHAR: {
-        if (taosVariantDump(&tmpVar, tmp, tType, true)) {
-          goto _return;
-        }
-        t = varDataLen(tmp);
-        pvar = varDataVal(tmp);        
-        break;
-      }
-      default:
-        goto _return;
-    }
-    
-    taosHashPut(pObj, (char *)pvar, t,  &dummy, sizeof(dummy));
-    taosVariantDestroy(&tmpVar);
-    memset(&tmpVar, 0, sizeof(tmpVar));
-  }
-
-  *q = (void *)pObj;
-  pObj = NULL;
-  
-_return:  
-  taosVariantDestroy(&tmpVar);
-  taosHashCleanup(pObj);
-  tfree(tmp);
-  
-  return code;
-}
-
-
-
-int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) {
+int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
+  SOperatorNode *node = (SOperatorNode *)tree;
+  int32_t ret = TSDB_CODE_SUCCESS;
   SFilterFieldId left = {0}, right = {0};
+  filterAddFieldFromNode(info, node->pLeft, &left);
+  uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
+  int32_t len = 0;
+  uint32_t uidx = 0;
+  int32_t code = 0;
 
-  filterAddFieldFromNode(info, tree->_node.pLeft, &left);  
-
-  SVariant* var = tree->_node.pRight->pVal;
-  int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
-  size_t len = 0;
-  uint16_t uidx = 0;
-
-  if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) {
-    void *data = NULL;
-    filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type);
-//    CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param");
-
-    if (taosHashGetSize((SHashObj *)data) <= 0) {
-      filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx);
-      
-      SFilterGroup fgroup = {0};
-      filterAddUnitToGroup(&fgroup, uidx);
+  if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) {
+    SNodeListNode *listNode = (SNodeListNode *)node->pRight;
+    void *fdata = NULL;
+    SListCell *cell = listNode->pNodeList->pHead;
+    SScalarParam in = {.num = 1}, out = {.num = 1, .type = type};
     
-      taosArrayPush(group, &fgroup);
-      taosHashCleanup(data);
+    for (int32_t i = 0; i < listNode->pNodeList->length; ++i) {
+      SValueNode *valueNode = (SValueNode *)cell->pNode;
+      in.type = valueNode->node.resType.type;
+      in.bytes = valueNode->node.resType.bytes;
+      in.data = nodesGetValueFromNode(valueNode);
+      out.data = malloc(sizeof(int64_t));
 
-      return TSDB_CODE_SUCCESS;
+      code = vectorConvertImpl(&in, &out);
+      if (code) {
+        fltError("convert from %d to %d failed", in.type, out.type);
+        tfree(out.data);
+        FLT_ERR_RET(code);
       }
       
-    void *p = taosHashIterate((SHashObj *)data, NULL);
-    while(p) {
-      void* key = NULL;
-      len = 0;
-
-      taosHashGetKey(p, &key, &len);
-      void *fdata = NULL;
-    
-      if (IS_VAR_DATA_TYPE(type)) {
-        fdata = malloc(len + VARSTR_HEADER_SIZE);
-        varDataLen(fdata) = len;
-        memcpy(varDataVal(fdata), key, len);
-        len += VARSTR_HEADER_SIZE;
-      } else {
-        fdata = malloc(sizeof(int64_t));
-        SIMPLE_COPY_VALUES(fdata, key);
       len = tDataTypes[type].bytes;
-      }
 
       filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true);
 
@@ -1197,15 +1016,11 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
       filterAddUnitToGroup(&fgroup, uidx);
       
       taosArrayPush(group, &fgroup);
-      
-      p = taosHashIterate((SHashObj *)data, p);
     }
-
-    taosHashCleanup(data);
   } else {
-    filterAddFieldFromNode(info, tree->_node.pRight, &right);  
+    filterAddFieldFromNode(info, node->pRight, &right);
     
-    filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);  
+    FLT_ERR_RET(filterAddUnit(info, node->opType, &left, &right, &uidx));
 
     SFilterGroup fgroup = {0};
     filterAddUnitToGroup(&fgroup, uidx);
@@ -1217,7 +1032,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
 }
 
 
-int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) {
+int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) {
   SFilterFieldId left, right, *pright = &right;
   int32_t type = FILTER_UNIT_DATA_TYPE(u);
   uint16_t flag = FLD_DESC_NO_FREE;
@@ -1230,7 +1045,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
     void *data = FILTER_UNIT_VAL_DATA(src, u);
     if (IS_VAR_DATA_TYPE(type)) {
       if (FILTER_UNIT_OPTR(u) ==  TSDB_RELATION_IN) {
-        filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false);
+        filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right.
 
         t = FILTER_GET_FIELD(dst, right);
         
@@ -1252,7 +1067,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
   return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx);
 }
 
-int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) {
+int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) {
   SFilterUnit *u = &info->units[uidx];
   
   u->compare.optr2 = optr;
@@ -1262,9 +1077,9 @@ int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *righ
 }
 
 
-int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) {
+int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) {
   SFilterFieldId left, right, right2;
-  uint16_t uidx = 0;
+  uint32_t uidx = 0;
 
   SFilterField *col = FILTER_GET_COL_FIELD(src, cidx);
 
@@ -1442,61 +1257,80 @@ static void filterFreeGroup(void *pItem) {
 }
 
 
-int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) {
+EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
   int32_t code = TSDB_CODE_SUCCESS;
-  SArray* leftGroup = NULL;
-  SArray* rightGroup = NULL;
+  SArray* preGroup = NULL;
+  SArray* newGroup = NULL;
+  SArray* resGroup = NULL;
+  ENodeType nType = nodeType(nType);
+  SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext;
 
-  if (tree->nodeType != TEXPR_BINARYEXPR_NODE) {
-    //qError("invalid nodeType:%d", tree->nodeType); 
-    return TSDB_CODE_QRY_APP_ERROR;
+  if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) {
+    SLogicConditionNode *node = (SLogicConditionNode *)pNode;
+    if (LOGIC_COND_TYPE_AND == node->condType) {
+      SListCell *cell = node->pParameterList->pHead;
+      for (int32_t i = 0; i < node->pParameterList->length; ++i) {
+        newGroup = taosArrayInit(4, sizeof(SFilterGroup));
+        resGroup = taosArrayInit(4, sizeof(SFilterGroup));
+        
+        SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup};
+        nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx);
+        FLT_ERR_JRET(tctx.code);
+        
+        FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup));
+        
+        taosArrayDestroyEx(newGroup, filterFreeGroup);
+        taosArrayDestroyEx(preGroup, filterFreeGroup);
+        
+        preGroup = resGroup;
       }
 
-  if (tree->_node.optr == TSDB_RELATION_AND) {
-    leftGroup = taosArrayInit(4, sizeof(SFilterGroup));
-    rightGroup = taosArrayInit(4, sizeof(SFilterGroup));
-    ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup));
-    ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup));
+      taosArrayAddAll(ctx->group, resGroup);
 
-    ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup));
+      taosArrayDestroyEx(newGroup, filterFreeGroup);
+      taosArrayDestroyEx(preGroup, filterFreeGroup);
+      taosArrayDestroyEx(resGroup, filterFreeGroup);
       
-    taosArrayDestroyEx(leftGroup, filterFreeGroup);
-    taosArrayDestroyEx(rightGroup, filterFreeGroup);
+      return DEAL_RES_IGNORE_CHILD;
+    }
 
-    return TSDB_CODE_SUCCESS;
+    if (LOGIC_COND_TYPE_OR == node->condType) {
+      SListCell *cell = node->pParameterList->pHead;
+      for (int32_t i = 0; i < node->pParameterList->length; ++i) {
+        nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext);
+        FLT_ERR_JRET(ctx->code);
+      }
+      
+      return DEAL_RES_IGNORE_CHILD;
     }
 
-  if (tree->_node.optr == TSDB_RELATION_OR) {
-    ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group));
-    ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group));
+    fltError("invalid condition type, type:%d", node->condType);
 
-    return TSDB_CODE_SUCCESS;
+    return DEAL_RES_ERROR;
   }
 
-  code = filterAddGroupUnitFromNode(info, tree, group);  
+  if (QUERY_NODE_OPERATOR == nType) {
+    FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group));  
     
+    return DEAL_RES_CONTINUE;
+  }
 
-_return:
+  fltError("invalid node type for filter, type:%d", nodeType(pNode));
 
-  taosArrayDestroyEx(leftGroup, filterFreeGroup);
-  taosArrayDestroyEx(rightGroup, filterFreeGroup);
+  code = TSDB_CODE_QRY_INVALID_INPUT;
 
-  return code;
-}
+_return:
 
-#if 0
-int32_t filterInitUnitFunc(SFilterInfo *info) {
-  for (uint16_t i = 0; i < info->unitNum; ++i) {
-    SFilterUnit* unit = &info->units[i];
+  taosArrayDestroyEx(newGroup, filterFreeGroup);
+  taosArrayDestroyEx(preGroup, filterFreeGroup);
+  taosArrayDestroyEx(resGroup, filterFreeGroup);
 
-    info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr);
-  }
+  ctx->code = code;
 
-  return TSDB_CODE_SUCCESS;
+  return DEAL_RES_ERROR;
 }
-#endif
 
-int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
+int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
   int32_t n = 0;
 
   switch (type) {
@@ -1574,34 +1408,38 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le
 
 void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) {
   if (qDebugFlag & DEBUG_DEBUG) {
-//    CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg);
+    if (info == NULL) {
+      fltDebug("%s - FilterInfo: EMPTY", msg);
+      return;
+    }
 
     if (options == 0) {
-//      //qDebug("%s - FilterInfo:", msg);
-//      //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
-      for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+      qDebug("%s - FilterInfo:", msg);
+      qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
+      for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
         SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i];
-        SSchema *sch = field->desc;
-//        //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name);
+        SColumnRefNode *refNode = (SColumnRefNode *)field->desc;
+        qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId);
       }
 
-      //qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num);
-      for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) {
+      qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num);
+      for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) {
         SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i];
         if (field->desc) {
-          SVariant *var = field->desc;
-          if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
-            //qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); 
+          SValueNode *var = (SValueNode *)field->desc;
+          SDataType *dType = &var->node.resType;
+          if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) {
+            qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); 
           } else {
-            //qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO
+            qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, dType->type, var->datum.i); //TODO
           }
         } else if (field->data) {
-          //qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO
+          qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO
         }
       }
 
-      //qDebug("UNIT  Num:%u", info->unitNum);
-      for (uint16_t i = 0; i < info->unitNum; ++i) {
+      qDebug("UNIT  Num:%u", info->unitNum);
+      for (uint32_t i = 0; i < info->unitNum; ++i) {
         SFilterUnit *unit = &info->units[i];
         int32_t type = FILTER_UNIT_DATA_TYPE(unit);
         int32_t len = 0;
@@ -1609,8 +1447,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
         char str[512] = {0};
         
         SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
-        SSchema *sch = left->desc;
-        len = sprintf(str, "UNIT[%d] => [%d][%s]  %s  [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str);
+        SColumnRefNode *refNode = (SColumnRefNode *)left->desc;
+        if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_NMATCH){
+          len = sprintf(str, "UNIT[%d] => [%d][%d]  %s  [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str);
+        }
 
         if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
           SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit);
@@ -1619,7 +1459,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
             tlen = varDataLen(data);
             data += VARSTR_HEADER_SIZE;
           }
-          converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen);
+          if (data) fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen);
         } else {
           strcat(str, "NULL");
         }
@@ -1627,7 +1467,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
 
         if (unit->compare.optr2) {
           strcat(str, " && ");
-          sprintf(str + strlen(str), "[%d][%s]  %s  [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str);
+          if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_NMATCH){
+            sprintf(str + strlen(str), "[%d][%d]  %s  [", refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr2].str);
+          }
           
           if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
             SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit);
@@ -1636,23 +1478,23 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
               tlen = varDataLen(data);
               data += VARSTR_HEADER_SIZE;
             }
-            converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen);
+            fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen);
           } else {
             strcat(str, "NULL");
           }
           strcat(str, "]");
         }
         
-        //qDebug("%s", str); //TODO
+        qDebug("%s", str); //TODO
       }
 
-      //qDebug("GROUP Num:%u", info->groupNum);
-      for (uint16_t i = 0; i < info->groupNum; ++i) {
+      qDebug("GROUP Num:%u", info->groupNum);
+      for (uint32_t i = 0; i < info->groupNum; ++i) {
         SFilterGroup *group = &info->groups[i];
-        //qDebug("Group%d : unit num[%u]", i, group->unitNum);
+        qDebug("Group%d : unit num[%u]", i, group->unitNum);
 
-        for (uint16_t u = 0; u < group->unitNum; ++u) {
-          //qDebug("unit id:%u", group->unitIdxs[u]);
+        for (uint32_t u = 0; u < group->unitNum; ++u) {
+          qDebug("unit id:%u", group->unitIdxs[u]);
         }
       }
 
@@ -1660,12 +1502,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
     }
 
     if (options == 1) {
-      //qDebug("%s - RANGE info:", msg);
+      qDebug("%s - RANGE info:", msg);
 
-      //qDebug("RANGE Num:%u", info->colRangeNum);
-      for (uint16_t i = 0; i < info->colRangeNum; ++i) {
+      qDebug("RANGE Num:%u", info->colRangeNum);
+      for (uint32_t i = 0; i < info->colRangeNum; ++i) {
         SFilterRangeCtx *ctx = info->colRange[i];
-        //qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange);
+        qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange);
         if (ctx->isrange) {      
           SFilterRangeNode *r = ctx->rs;
           while (r) {
@@ -1675,7 +1517,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
               strcat(str,"(NULL)");
             } else {
               FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"[");
-              converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen);
+              fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen);
               FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]");
             }
             strcat(str, " - ");
@@ -1683,10 +1525,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
               strcat(str, "(NULL)");
             } else {
               FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"[");
-              converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen);
+              fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen);
               FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]");
             }
-            //qDebug("range: %s", str);        
+            qDebug("range: %s", str);        
             
             r = r->next;
           }
@@ -1696,25 +1538,25 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
       return;
     }
 
-    //qDebug("%s - Block Filter info:", msg);
+    qDebug("%s - Block Filter info:", msg);
 
     if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) {
-      //qDebug("Flag:%s", "ALL");
+      qDebug("Flag:%s", "ALL");
       return;
     } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) {
-      //qDebug("Flag:%s", "EMPTY");
+      qDebug("Flag:%s", "EMPTY");
       return;
     } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){
-      //qDebug("Flag:%s", "ACTIVE");
+      qDebug("Flag:%s", "ACTIVE");
     }
 
-    //qDebug("GroupNum:%d", info->blkGroupNum);
-    uint16_t *unitIdx = info->blkUnits;
-    for (uint16_t i = 0; i < info->blkGroupNum; ++i) {
-      //qDebug("Group[%d] UnitNum: %d:", i, *unitIdx);
-      uint16_t unitNum = *(unitIdx++);
-      for (uint16_t m = 0; m < unitNum; ++m) {
-        //qDebug("uidx[%d]", *(unitIdx++));
+    qDebug("GroupNum:%d", info->blkGroupNum);
+    uint32_t *unitIdx = info->blkUnits;
+    for (uint32_t i = 0; i < info->blkGroupNum; ++i) {
+      qDebug("Group[%d] UnitNum: %d:", i, *unitIdx);
+      uint32_t unitNum = *(unitIdx++);
+      for (uint32_t m = 0; m < unitNum; ++m) {
+        qDebug("uidx[%d]", *(unitIdx++));
       }
     }
   }
@@ -1800,21 +1642,23 @@ void filterFreePCtx(SFilterPCtx *pctx) {
 }
 
 void filterFreeInfo(SFilterInfo *info) {
-  CHK_RETV(info == NULL);
+  if (info == NULL) {
+    return;
+  }
 
   tfree(info->cunits);
   tfree(info->blkUnitRes);
   tfree(info->blkUnits);
   
   for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) {
-    for (uint16_t f = 0; f < info->fields[i].num; ++f) {
+    for (uint32_t f = 0; f < info->fields[i].num; ++f) {
       filterFreeField(&info->fields[i].fields[f], i);
     }
     
     tfree(info->fields[i].fields);
   }
 
-  for (int32_t i = 0; i < info->groupNum; ++i) {
+  for (uint32_t i = 0; i < info->groupNum; ++i) {
     filterFreeGroup(&info->groups[i]);    
   }
   
@@ -1826,7 +1670,7 @@ void filterFreeInfo(SFilterInfo *info) {
 
   tfree(info->unitFlags);
 
-  for (uint16_t i = 0; i < info->colRangeNum; ++i) {
+  for (uint32_t i = 0; i < info->colRangeNum; ++i) {
     filterFreeRangeCtx(info->colRange[i]);
   }
 
@@ -1862,9 +1706,72 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t fltGenerateSetFromList(void **data, void *pNode, uint32_t type) {
+  SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false);
+  if (NULL == pObj) {
+    fltError("taosHashInit failed, size:%d", 256);
+    FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); 
+
+  int32_t code = 0;
+  SNodeListNode *nodeList = (SNodeListNode *)pNode;
+  SListCell *cell = nodeList->pNodeList->pHead;
+  SScalarParam in = {.num = 1}, out = {.num = 1, .type = type};
+  int8_t dummy = 0;
+  int32_t bufLen = 60;
+  out.data = malloc(bufLen);
+  int32_t len = 0;
+  void *buf = NULL;
+  
+  for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) {
+    SValueNode *valueNode = (SValueNode *)cell->pNode;
+
+    if (valueNode->node.resType.type != type) {
+      in.type = valueNode->node.resType.type;
+      in.bytes = valueNode->node.resType.bytes;
+      in.data = nodesGetValueFromNode(valueNode);
     
-int32_t filterInitValFieldData(SFilterInfo *info) {
-  for (uint16_t i = 0; i < info->unitNum; ++i) {
+      code = vectorConvertImpl(&in, &out);
+      if (code) {
+        fltError("convert from %d to %d failed", in.type, out.type);
+        FLT_ERR_JRET(code);
+      }
+
+      if (IS_VAR_DATA_TYPE(type)) {
+        len = varDataLen(out.data);
+      } else {
+        len = tDataTypes[type].bytes;
+      }
+
+      buf = out.data;
+    } else {
+      buf = nodesGetValueFromNode(valueNode);
+      len = valueNode->node.resType.bytes;
+    }
+    
+    if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) {
+      fltError("taosHashPut failed");
+      FLT_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+    }
+  }
+
+  tfree(out.data);
+  *data = pObj;
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  tfree(out.data);
+  taosHashCleanup(pObj);
+  
+  FLT_RET(code);
+}
+
+int32_t fltInitValFieldData(SFilterInfo *info) {
+  for (uint32_t i = 0; i < info->unitNum; ++i) {
     SFilterUnit* unit = &info->units[i];
     if (unit->right.type != FLD_TYPE_VALUE) {
       assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR);
@@ -1878,7 +1785,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
     uint32_t type = FILTER_UNIT_DATA_TYPE(unit);
     SFilterField* fi = right;
     
-    SVariant* var = fi->desc;
+    SValueNode* var = (SValueNode *)fi->desc;
 
     if (var == NULL) {
       assert(fi->data != NULL);
@@ -1886,45 +1793,65 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
     }
 
     if (unit->compare.optr == TSDB_RELATION_IN) {
-      filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type);
-//      CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param");
+      FLT_ERR_RET(fltGenerateSetFromList((void **)&fi->data, fi->desc, type));
+      if (fi->data == NULL) {
+        fltError("failed to convert in param");
+        FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
 
       FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH);
       
       continue;
     }
 
+    SDataType *dType = &var->node.resType;
+
     if (type == TSDB_DATA_TYPE_BINARY) {
-      size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;
+      size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE;
       fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE);
     } else if (type == TSDB_DATA_TYPE_NCHAR) {
-      size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;    
+      size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE;    
       fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
-    } else {
-      if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {  //TIME RANGE
-        fi->data = calloc(var->nLen, tDataTypes[type].bytes);
-        for (int32_t a = 0; a < var->nLen; ++a) {
+    } else if (type != TSDB_DATA_TYPE_JSON){
+      if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) {  //TIME RANGE
+/*      
+        fi->data = calloc(dType->bytes, tDataTypes[type].bytes);
+        for (int32_t a = 0; a < dType->bytes; ++a) {
           int64_t *v = taosArrayGet(var->arr, a);
           assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type);
         }
-
+*/
         continue;
       } else {
         fi->data = calloc(1, sizeof(int64_t));
       }
+    } else{     // type == TSDB_DATA_TYPE_JSON
+      // fi->data = null;  use fi->desc as data, because json value is variable, so use tVariant (fi->desc)
     }
 
+    if(type != TSDB_DATA_TYPE_JSON){
       bool converted = false;
       char extInfo = 0;
-//    if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) {
-//      if (converted) {
-//        filterHandleValueExtInfo(unit, extInfo);
-//
-//        continue;
-//      }
-//      //qError("dump value to type[%d] failed", type);
-//      return TSDB_CODE_TSC_INVALID_OPERATION;
-//    }
+      SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes};
+      SScalarParam out = {.data = fi->data, .num = 1, .type = type};
+      if (vectorConvertImpl(&in, &out)) {
+        qError("convert value to type[%d] failed", type);
+        return TSDB_CODE_TSC_INVALID_OPERATION;
+      }
+    }
+
+    // match/nmatch for nchar type need convert from ucs4 to mbs
+    if(type == TSDB_DATA_TYPE_NCHAR &&
+        (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
+      char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
+      int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
+      if (len < 0){
+        qError("filterInitValFieldData taosUcs4ToMbs error 1");
+        return TSDB_CODE_FAILED;
+      }
+      varDataSetLen(newValData, len);
+      varDataCopy(fi->data, newValData);
+    }
   }
 
   return TSDB_CODE_SUCCESS;
@@ -2025,25 +1952,25 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c
 }
 
 int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) {
-  CHK_JMP(ctx1->status != ctx2->status);
-  CHK_JMP(ctx1->isnull != ctx2->isnull);
-  CHK_JMP(ctx1->notnull != ctx2->notnull);
-  CHK_JMP(ctx1->isrange != ctx2->isrange);
+  FLT_CHK_JMP(ctx1->status != ctx2->status);
+  FLT_CHK_JMP(ctx1->isnull != ctx2->isnull);
+  FLT_CHK_JMP(ctx1->notnull != ctx2->notnull);
+  FLT_CHK_JMP(ctx1->isrange != ctx2->isrange);
 
   SFilterRangeNode *r1 = ctx1->rs;
   SFilterRangeNode *r2 = ctx2->rs;
   
   while (r1 && r2) {
-    CHK_JMP(r1->ra.sflag != r2->ra.sflag);
-    CHK_JMP(r1->ra.eflag != r2->ra.eflag);
-    CHK_JMP(r1->ra.s != r2->ra.s);
-    CHK_JMP(r1->ra.e != r2->ra.e);
+    FLT_CHK_JMP(r1->ra.sflag != r2->ra.sflag);
+    FLT_CHK_JMP(r1->ra.eflag != r2->ra.eflag);
+    FLT_CHK_JMP(r1->ra.s != r2->ra.s);
+    FLT_CHK_JMP(r1->ra.e != r2->ra.e);
     
     r1 = r1->next;
     r2 = r2->next;
   }
 
-  CHK_JMP(r1 != r2);
+  FLT_CHK_JMP(r1 != r2);
 
   *equal = true;
 
@@ -2055,7 +1982,7 @@ _return:
 }
 
 
-int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) {
+int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colIdx, bool *empty) {
   SArray* colArray = (SArray *)gRes->colInfo[colIdx].info;
   int32_t size = (int32_t)taosArrayGetSize(colArray);
   int32_t type = gRes->colInfo[colIdx].dataType;
@@ -2066,11 +1993,15 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colI
     uint8_t optr = FILTER_UNIT_OPTR(u);
 
     filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL);
-    CHK_JMP(*empty);
+    FLT_CHK_JMP(*empty);
 
     if (!FILTER_NO_MERGE_OPTR(optr)) {
       filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND);
-      CHK_JMP(MR_EMPTY_RES(ctx));
+      FLT_CHK_JMP(MR_EMPTY_RES(ctx));
+    }
+    if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){
+      gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL;
+      SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u));
     }
   }
 
@@ -2092,11 +2023,11 @@ _return:
 
 int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) {
   bool empty = false;
-  uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t));
-  uint16_t colIdxi = 0;
-  uint16_t gResIdx = 0;
+  uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t));
+  uint32_t colIdxi = 0;
+  uint32_t gResIdx = 0;
   
-  for (uint16_t i = 0; i < info->groupNum; ++i) {
+  for (uint32_t i = 0; i < info->groupNum; ++i) {
     SFilterGroup* g = info->groups + i;
 
     gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx));
@@ -2104,9 +2035,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
     colIdxi = 0;
     empty = false;
     
-    for (uint16_t j = 0; j < g->unitNum; ++j) {
+    for (uint32_t j = 0; j < g->unitNum; ++j) {
       SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j);
-      uint16_t cidx = FILTER_UNIT_COL_IDX(u);
+      uint32_t cidx = FILTER_UNIT_COL_IDX(u);
 
       if (gRes[gResIdx]->colInfo[cidx].info == NULL) {
         gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES);
@@ -2122,10 +2053,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
     }
 
     if (colIdxi > 1) {
-      qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
+      qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
     }
 
-    for (uint16_t l = 0; l < colIdxi; ++l) {
+    for (uint32_t l = 0; l < colIdxi; ++l) {
       int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType;
 
       if (FILTER_NO_MERGE_DATA_TYPE(type)) {
@@ -2164,7 +2095,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
 }
 
 void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) {
-  uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0;
+  uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
   bool equal = false;
   
   for (; m < gRes1->colNum; ++m) {
@@ -2188,6 +2119,15 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool
         return;
       }
 
+      // for long in operation
+      if (gRes1->colInfo[idx1].optr == TSDB_RELATION_EQUAL && gRes2->colInfo[idx2].optr == TSDB_RELATION_EQUAL) {
+        SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info;
+        if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){
+          *conflict = true;
+          return;
+        }
+      }
+      
       ++n;
       equal = true;
       break;
@@ -2204,7 +2144,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool
 }
 
 
-int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) {
+int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint32_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) {
   SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx);
   int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
 
@@ -2234,10 +2174,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter
 
   FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE);
 
-  uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0;
+  uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
   bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum;
   bool equal = false;
-  uint16_t equal1 = 0, equal2 = 0, merNum = 0;
+  uint32_t equal1 = 0, equal2 = 0, merNum = 0;
   SFilterRangeCtx *ctx = NULL;
   SFilterColCtx colCtx = {0};
   SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx));
@@ -2258,7 +2198,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter
       
       filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all);
 
-      CHK_JMP(*all);
+      FLT_CHK_JMP(*all);
 
       if (numEqual) {
         if ((*gRes1)->colNum == 1) {
@@ -2278,7 +2218,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter
             ++equal2;
           }
 
-          CHK_JMP(equal1 != merNum && equal2 != merNum);
+          FLT_CHK_JMP(equal1 != merNum && equal2 != merNum);
           colCtx.colIdx = idx1;
           colCtx.ctx = ctx;
           ctx = NULL;
@@ -2290,7 +2230,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter
           ++equal1;
         }
 
-        CHK_JMP(equal1 != merNum);
+        FLT_CHK_JMP(equal1 != merNum);
         colCtx.colIdx = idx1;
         colCtx.ctx = ctx;        
         ctx = NULL;
@@ -2351,7 +2291,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
   qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx);
 
   int32_t pEnd = 0, cStart = 0, cEnd = 0;
-  uint16_t pColNum = 0, cColNum = 0; 
+  uint32_t pColNum = 0, cColNum = 0; 
   int32_t movedNum = 0;
   bool all = false;
 
@@ -2371,7 +2311,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
           assert(m < n);
           filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all);
 
-          CHK_JMP(all);
+          FLT_CHK_JMP(all);
 
           if (gRes[n] == NULL) {
             if (n < ((*gResNum) - 1)) {
@@ -2392,7 +2332,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
         assert(m < n);
         filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all);
 
-        CHK_JMP(all);
+        FLT_CHK_JMP(all);
         
         if (gRes[n] == NULL) {
           if (n < ((*gResNum) - 1)) {
@@ -2417,7 +2357,6 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
     }
     
     cStart = i;
-    cEnd = i;
     cColNum = gRes[i]->colNum;    
   }
 
@@ -2433,7 +2372,7 @@ _return:
 int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) {
   size_t groupSize = taosArrayGetSize(group);
 
-  info->groupNum = (uint16_t)groupSize;
+  info->groupNum = (uint32_t)groupSize;
 
   if (info->groupNum > 0) {
     info->groups = calloc(info->groupNum, sizeof(*info->groups));
@@ -2450,7 +2389,7 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) {
 
 int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) {
   if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) {
-    //qDebug("no need rewrite");
+    qDebug("no need rewrite");
     return TSDB_CODE_SUCCESS;
   }
   
@@ -2462,7 +2401,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
   SFilterGroupCtx *res = NULL;
   SFilterColInfo *colInfo = NULL;
   int32_t optr = 0;
-  uint16_t uidx = 0;
+  uint32_t uidx = 0;
 
   memset(info, 0, sizeof(*info));
   
@@ -2482,7 +2421,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
 
     SFilterGroup ng = {0};
     
-    for (uint16_t m = 0; m < res->colNum; ++m) {
+    for (uint32_t m = 0; m < res->colNum; ++m) {
       colInfo = &res->colInfo[res->colIdx[m]];
       if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) {
         assert(colInfo->type == RANGE_TYPE_UNIT);
@@ -2518,13 +2457,13 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
 }
 
 int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) {
-  uint16_t *idxs = NULL;
-  uint16_t colNum = 0;
+  uint32_t *idxs = NULL;
+  uint32_t colNum = 0;
   SFilterGroupCtx *res = NULL;
-  uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum));
+  uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum));
 
   for (int32_t i = 0; i < gResNum; ++i) {
-    for (uint16_t m = 0; m < gRes[i]->colNum; ++m) {
+    for (uint32_t m = 0; m < gRes[i]->colNum; ++m) {
       SFilterColInfo  *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]];
       if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) {
         continue;
@@ -2534,7 +2473,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
     }
   }
 
-  for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+  for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
     if (idxNum[i] < gResNum) {
       continue;
     }
@@ -2548,16 +2487,16 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
     idxs[colNum++] = i;
   }
 
-  CHK_JMP(colNum <= 0);
+  FLT_CHK_JMP(colNum <= 0);
 
   info->colRangeNum = colNum;
   info->colRange = calloc(colNum, POINTER_BYTES);
 
   for (int32_t i = 0; i < gResNum; ++i) {
     res = gRes[i];
-    uint16_t n = 0;
+    uint32_t n = 0;
 
-    for (uint16_t m = 0; m < info->colRangeNum; ++m) {
+    for (uint32_t m = 0; m < info->colRangeNum; ++m) {
       for (; n < res->colNum; ++n) {
         if (res->colIdx[n] < idxs[m]) {
           continue;
@@ -2569,7 +2508,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
         if (info->colRange[m] == NULL) {
           info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0);
           SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]);
-          info->colRange[m]->colId = ((SSchema*)fi->desc)->colId;
+          info->colRange[m]->colId = FILTER_GET_COL_FIELD_ID(fi);
         }
 
         assert(colInfo->type == RANGE_TYPE_MR_CTX);
@@ -2588,7 +2527,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
           --info->colRangeNum;
           --m;
 
-          CHK_JMP(info->colRangeNum <= 0);          
+          FLT_CHK_JMP(info->colRangeNum <= 0);          
         }
 
         ++n; 
@@ -2605,7 +2544,7 @@ _return:
 }
 
 int32_t filterPostProcessRange(SFilterInfo *info) {
-  for (uint16_t i = 0; i < info->colRangeNum; ++i) {
+  for (uint32_t i = 0; i < info->colRangeNum; ++i) {
     SFilterRangeCtx* ctx = info->colRange[i];
     SFilterRangeNode *r = ctx->rs;
     while (r) {
@@ -2619,13 +2558,11 @@ int32_t filterPostProcessRange(SFilterInfo *info) {
 
 
 int32_t filterGenerateComInfo(SFilterInfo *info) {
-  uint16_t n = 0;
-  
   info->cunits = malloc(info->unitNum * sizeof(*info->cunits));
   info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum);
   info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum);
 
-  for (uint16_t i = 0; i < info->unitNum; ++i) {
+  for (uint32_t i = 0; i < info->unitNum; ++i) {
     SFilterUnit *unit = &info->units[i];
 
     info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr);
@@ -2635,7 +2572,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
     info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
     
     if (unit->right.type == FLD_TYPE_VALUE) {
+      if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){   // json value is tVariant
+        info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit);
+      }else{
         info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
+      }
     } else {
       info->cunits[i].valData = NULL;
     }
@@ -2649,29 +2590,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
     info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit);
   }
   
-  uint16_t cgroupNum = info->groupNum + 1;
-  
-  for (uint16_t i = 0; i < info->groupNum; ++i) {
-    cgroupNum += info->groups[i].unitNum;
-  }
-
-  info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups));
-  
-  for (uint16_t i = 0; i < info->groupNum; ++i) {
-    info->cgroups[n++] = info->groups[i].unitNum;
-
-    for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) {
-      info->cgroups[n++] = info->groups[i].unitIdxs[m];
-    }
-  }
-
-  info->cgroups[n] = 0;
-  
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t filterUpdateComUnits(SFilterInfo *info) {
-  for (uint16_t i = 0; i < info->unitNum; ++i) {
+  for (uint32_t i = 0; i < info->unitNum; ++i) {
     SFilterUnit *unit = &info->units[i];
 
     info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
@@ -2686,7 +2609,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
 
   memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum);
   
-  for (int32_t k = 0; k < info->unitNum; ++k) {
+  for (uint32_t k = 0; k < info->unitNum; ++k) {
     int32_t index = -1;
     SFilterComUnit *cunit = &info->cunits[k];
 
@@ -2739,10 +2662,12 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
 
     SColumnDataAgg* pDataBlockst = &pDataStatis[index];
     void *minVal, *maxVal;
+    float minv = 0;
+    float maxv = 0;
 
     if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) {
-      float minv = (float)(*(double *)(&pDataBlockst->min));
-      float maxv = (float)(*(double *)(&pDataBlockst->max));
+      minv = (float)(*(double *)(&pDataBlockst->min));
+      maxv = (float)(*(double *)(&pDataBlockst->max));
        
       minVal = &minv;
       maxVal = &maxv;
@@ -2797,12 +2722,15 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
 
   }
 
-//  CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY");
+  if (rmUnit == 0) {
+    fltDebug("NO Block Filter APPLY");
+    FLT_RET(TSDB_CODE_SUCCESS);
+  }
 
   info->blkGroupNum = info->groupNum;
   
-  uint16_t *unitNum = info->blkUnits;
-  uint16_t *unitIdx = unitNum + 1;
+  uint32_t *unitNum = info->blkUnits;
+  uint32_t *unitIdx = unitNum + 1;
   int32_t all = 0, empty = 0;
   
   for (uint32_t g = 0; g < info->groupNum; ++g) {
@@ -2812,7 +2740,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
     empty = 0;
     
     for (uint32_t u = 0; u < group->unitNum; ++u) {
-      uint16_t uidx = group->unitIdxs[u];
+      uint32_t uidx = group->unitIdxs[u];
       if (info->blkUnitRes[uidx] == 1) {
         --(*unitNum);
         all = 1;
@@ -2859,9 +2787,11 @@ _return:
 bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
   SFilterInfo *info = (SFilterInfo *)pinfo;
   bool all = true;
-  uint16_t *unitIdx = NULL;
+  uint32_t *unitIdx = NULL;
 
+  if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
+  }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
     //FILTER_UNIT_CLR_F(info);
@@ -2869,7 +2799,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p,
     unitIdx = info->blkUnits;
     
     for (uint32_t g = 0; g < info->blkGroupNum; ++g) {
-      uint16_t unitNum = *(unitIdx++);
+      uint32_t unitNum = *(unitIdx++);
       for (uint32_t u = 0; u < unitNum; ++u) {
         SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)];
         void *colData = (char *)cunit->colData + cunit->dataSize * i;
@@ -2964,12 +2894,25 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
     return all;
   }
 
+  if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
+  }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint16_t uidx = info->groups[0].unitIdxs[0];
+    uint32_t uidx = info->groups[0].unitIdxs[0];
     void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
+    if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
+      if (!colData){  // for json->'key' is null
+        (*p)[i] = 1;
+      }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){  // for json is null
+        colData = POINTER_SHIFT(colData, CHAR_BYTES);
         (*p)[i] = isNull(colData, info->cunits[uidx].dataType);
+      }else{
+        (*p)[i] = 0;
+      }
+    }else{
+      (*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType));
+    }
     if ((*p)[i] == 0) {
       all = false;
     }    
@@ -2985,12 +2928,27 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
     return all;
   }
 
+  if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
+  }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint16_t uidx = info->groups[0].unitIdxs[0];
+    uint32_t uidx = info->groups[0].unitIdxs[0];
     void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
+
+    if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
+      if (!colData) {   // for json->'key' is not null
+        (*p)[i] = 0;
+      }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){   // for json is not null
+        colData = POINTER_SHIFT(colData, CHAR_BYTES);
         (*p)[i] = !isNull(colData, info->cunits[uidx].dataType);
+      }else{    // for json->'key' is not null
+        (*p)[i] = 1;
+      }
+    }else {
+      (*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType));
+    }
+
     if ((*p)[i] == 0) {
       all = false;
     }
@@ -3013,10 +2971,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD
     return all;
   }
 
+  if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
+  }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
-    if (isNull(colData, info->cunits[0].dataType)) {
+    if (colData == NULL || isNull(colData, info->cunits[0].dataType)) {
       all = false;
       colData += dataSize;
       continue;
@@ -3042,17 +3002,33 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa
     return all;
   }
   
+  if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
+  }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint16_t uidx = info->groups[0].unitIdxs[0];
+    uint32_t uidx = info->groups[0].unitIdxs[0];
     void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
-    if (isNull(colData, info->cunits[uidx].dataType)) {
+    if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) {
+      (*p)[i] = 0;
       all = false;
       continue;
     }
+    // match/nmatch for nchar type need convert from ucs4 to mbs
 
+    if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){
+      char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
+      int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
+      if (len < 0){
+        qError("castConvert1 taosUcs4ToMbs error");
+      }else{
+        varDataSetLen(newColData, len);
+        (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData);
+      }
+      tfree(newColData);
+    }else{
       (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData);
+    }
 
     if ((*p)[i] == 0) {
       all = false;
@@ -3071,7 +3047,9 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
     return all;
   }
 
+  if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
+  }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
     //FILTER_UNIT_CLR_F(info);
@@ -3079,7 +3057,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
     for (uint32_t g = 0; g < info->groupNum; ++g) {
       SFilterGroup *group = &info->groups[g];
       for (uint32_t u = 0; u < group->unitNum; ++u) {
-        uint16_t uidx = group->unitIdxs[u];
+        uint32_t uidx = group->unitIdxs[u];
         SFilterComUnit *cunit = &info->cunits[uidx];
         void *colData = (char *)cunit->colData + cunit->dataSize * i;
       
@@ -3088,7 +3066,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
         //} else {
           uint8_t optr = cunit->optr;
 
-          if (isNull(colData, cunit->dataType)) {
+          if (colData == NULL || isNull(colData, cunit->dataType)) {
             (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false;
           } else {
             if (optr == TSDB_RELATION_NOTNULL) {
@@ -3098,8 +3076,20 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
             } else if (cunit->rfunc >= 0) {
               (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]);
             } else {
+              if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){
+                char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
+                int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
+                if (len < 0){
+                  qError("castConvert1 taosUcs4ToMbs error");
+                }else{
+                  varDataSetLen(newColData, len);
+                  (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData);
+                }
+                tfree(newColData);
+              }else{
                 (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
               }
+            }
           
           //FILTER_UNIT_SET_R(info, uidx, p[i]);
           //FILTER_UNIT_SET_F(info, uidx);
@@ -3123,11 +3113,6 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
   return all;
 }
 
-
-FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
-  return (*info->func)(info, numOfRows, p, statis, numOfCols);
-}
-
 int32_t filterSetExecFunc(SFilterInfo *info) {
   if (FILTER_ALL_RES(info)) {
     info->func = filterExecuteImplAll;
@@ -3174,13 +3159,13 @@ int32_t filterPreprocess(SFilterInfo *info) {
   filterMergeGroups(info, gRes, &gResNum);
 
   if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) {
-//    qInfo("Final - FilterInfo: [ALL]");
+    fltInfo("Final - FilterInfo: [ALL]");
     goto _return;
   }
 
   
   if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) {
-//    qInfo("Final - FilterInfo: [EMPTY]");
+    fltInfo("Final - FilterInfo: [EMPTY]");
     goto _return;
   }  
 
@@ -3207,25 +3192,19 @@ _return:
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) {
-//  CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
-//  CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
 
+int32_t fltSetColFieldDataImpl(SFilterInfo *info, void *param, filer_get_col_from_id fp, bool fromColId) {
   if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
     return TSDB_CODE_SUCCESS;
   }
 
-  for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+  for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
     SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
-    SSchema* sch = fi->desc;
-    
-    for (int32_t j = 0; j < numOfCols; ++j) {
-      SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j);
-      if (sch->colId == pColInfo->info.colId) {
-        fi->data = pColInfo->pData;
 
-        break;
-      }
+    if (fromColId) {
+      (*fp)(param, FILTER_GET_COL_FIELD_ID(fi), &fi->data);
+    } else {
+      (*fp)(param, FILTER_GET_COL_FIELD_SLOT_ID(fi), &fi->data);
     }
   }
 
@@ -3235,45 +3214,32 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat
 }
 
 
-int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) {
+int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) {
   int32_t code = TSDB_CODE_SUCCESS;
-  SFilterInfo *info = NULL;
-  
-//  CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param");
-
-  if (*pinfo == NULL) {
-    *pinfo = calloc(1, sizeof(SFilterInfo));
-  }
-
-  info = *pinfo;
-
-  info->options = options;
   
   SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup));
 
   filterInitUnitsFields(info);
 
-  code = filterTreeToGroup(tree, info, group);
-
-  ERR_JRET(code);
+  SFltBuildGroupCtx tctx = {.info = info, .group = group};
+  nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx);
+  FLT_ERR_JRET(tctx.code);
 
   filterConvertGroupFromArray(info, group);
+  taosArrayDestroy(group);
 
-  ERR_JRET(filterInitValFieldData(info));
+  FLT_ERR_JRET(fltInitValFieldData(info));
 
   if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) {
     filterDumpInfoToString(info, "Before preprocess", 0);
 
-    ERR_JRET(filterPreprocess(info));
+    FLT_ERR_JRET(filterPreprocess(info));
     
-    CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL));
+    FLT_CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL));
 
     if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) {
-      taosArrayDestroy(group);
       return code;
     }
-
-    //ERR_JRET(filterInitUnitFunc(info));
   }  
 
   info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes));
@@ -3281,17 +3247,11 @@ int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t option
 
   filterDumpInfoToString(info, "Final", 0);
 
-  taosArrayDestroy(group);
-
   return code;
 
 _return:
-//  qInfo("No filter, code:%d", code);
 
-  taosArrayDestroy(group);
-  filterFreeInfo(*pinfo);
-
-  *pinfo = NULL;
+  qInfo("No filter, code:%d", code);
   
   return code;
 }
@@ -3311,7 +3271,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
   bool ret = true;
   void *minVal, *maxVal;
   
-  for (int32_t k = 0; k < info->colRangeNum; ++k) {
+  for (uint32_t k = 0; k < info->colRangeNum; ++k) {
     int32_t index = -1;
     SFilterRangeCtx *ctx = info->colRange[k];
     for(int32_t i = 0; i < numOfCols; ++i) {
@@ -3331,30 +3291,35 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
       break;
     }
 
-    if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) {
+    if (pDataStatis[index].numOfNull <= 0) {
+      if (ctx->isnull && !ctx->notnull && !ctx->isrange) {
         ret = false;
         break;
       }
-    
-    // all data in current column are NULL, no need to check its boundary value
+    } else if (pDataStatis[index].numOfNull > 0) {
       if (pDataStatis[index].numOfNull == numOfRows) {
-
-      // if isNULL query exists, load the null data column
         if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) {
           ret = false;
           break;
         }
 
         continue;
+      } else {
+        if (ctx->isnull) {
+          continue;
+        }
+      }
     }
 
     SColumnDataAgg* pDataBlockst = &pDataStatis[index];
 
     SFilterRangeNode *r = ctx->rs;
+    float minv = 0;
+    float maxv = 0;
 
     if (ctx->type == TSDB_DATA_TYPE_FLOAT) {
-      float minv = (float)(*(double *)(&pDataBlockst->min));
-      float maxv = (float)(*(double *)(&pDataBlockst->max));
+      minv = (float)(*(double *)(&pDataBlockst->min));
+      maxv = (float)(*(double *)(&pDataBlockst->max));
        
       minVal = &minv;
       maxVal = &maxv;
@@ -3371,7 +3336,9 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
       r = r->next;
     }
     
-    CHK_RET(!ret, ret);
+    if (!ret) {
+      return ret;
+    }
   }
 
   return ret;
@@ -3389,7 +3356,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow       *win) {
   int32_t code = 0;
   bool empty = false, all = false;
 
-  for (int32_t i = 0; i < info->groupNum; ++i) {
+  for (uint32_t i = 0; i < info->groupNum; ++i) {
     SFilterGroup *group = &info->groups[i];
     if (group->unitNum > 1) {
       cur = tmpc;
@@ -3399,14 +3366,14 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow       *win) {
       optr = TSDB_RELATION_OR;
     }
 
-    for (int32_t u = 0; u < group->unitNum; ++u) {
-      uint16_t uidx = group->unitIdxs[u];
+    for (uint32_t u = 0; u < group->unitNum; ++u) {
+      uint32_t uidx = group->unitIdxs[u];
       SFilterUnit *unit = &info->units[uidx];
 
       uint8_t raOptr = FILTER_UNIT_OPTR(unit);
       
       filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL);
-      CHK_JMP(empty);
+      FLT_CHK_JMP(empty);
       
       if (FILTER_NO_MERGE_OPTR(raOptr)) {
         continue;
@@ -3441,11 +3408,11 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow       *win) {
   } else {
     filterGetRangeNum(prev, &num);
     if (num > 1) {
-      //qError("only one time range accepted, num:%d", num);
-      ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION);
+      qError("only one time range accepted, num:%d", num);
+      FLT_ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION);
     }
 
-    CHK_JMP(num < 1);
+    FLT_CHK_JMP(num < 1);
 
     SFilterRange tra;
     filterGetRangeRes(prev, &tra);
@@ -3456,7 +3423,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow       *win) {
   filterFreeRangeCtx(prev);
   filterFreeRangeCtx(tmpc);
 
-  //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
+  qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
   return TSDB_CODE_SUCCESS;
 
 _return:
@@ -3466,14 +3433,18 @@ _return:
   filterFreeRangeCtx(prev);
   filterFreeRangeCtx(tmpc);
 
-  //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
+  qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
 
   return code;
 }
 
 
 int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) {
-  for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+  if (FILTER_EMPTY_RES(info) || FILTER_ALL_RES(info)) {
+    return TSDB_CODE_SUCCESS;
+  }
+ 
+  for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
     SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
     int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
     if (type == TSDB_DATA_TYPE_NCHAR) {
@@ -3486,7 +3457,20 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
         char *src = FILTER_GET_COL_FIELD_DATA(fi, j);
         char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j);
         int32_t len = 0;
-        taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
+        char *varSrc = varDataVal(src);
+        size_t k = 0, varSrcLen = varDataLen(src);
+        while (k < varSrcLen && varSrc[k++] == -1) {}
+        if (k == varSrcLen) {
+          /* NULL */
+          varDataLen(dst) = (VarDataLenT) varSrcLen;
+          varDataCopy(dst, src);
+          continue;
+        }
+        bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
+        if(!ret) {
+          qError("filterConverNcharColumns taosMbsToUcs4 error");
+          return TSDB_CODE_FAILED;
+        }
         varDataLen(dst) = len;
       }
 
@@ -3504,7 +3488,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
 }
 
 int32_t filterFreeNcharColumns(SFilterInfo* info) {
-  for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+  for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
     SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
     int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
     if (type == TSDB_DATA_TYPE_NCHAR) {
@@ -3515,7 +3499,198 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) {
   return TSDB_CODE_SUCCESS;
 }
 
+EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
+  SFltTreeStat *stat = (SFltTreeStat *)pContext;
+
+  if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) {
+    SLogicConditionNode *node = (SLogicConditionNode *)*pNode;
+    SListCell *cell = node->pParameterList->pHead;
+    for (int32_t i = 0; i < node->pParameterList->length; ++i) {
+      if (NULL == cell || NULL == cell->pNode) {
+        fltError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode);
+        stat->code = TSDB_CODE_QRY_INVALID_INPUT;
+        return DEAL_RES_ERROR;
+      }
+    
+      if ((QUERY_NODE_OPERATOR != nodeType(cell->pNode)) && (QUERY_NODE_LOGIC_CONDITION != nodeType(cell->pNode))) {
+        stat->scalarMode = true;
+      }
+      
+      cell = cell->pNext;
+    }
+
+    return DEAL_RES_CONTINUE;
+  }
+
+  if (stat->scalarMode) {
+    return DEAL_RES_CONTINUE;
+  }
+  
+  if (QUERY_NODE_VALUE == nodeType(*pNode)) {
+    return DEAL_RES_CONTINUE;
+  }
+
+  if (QUERY_NODE_FUNCTION == nodeType(*pNode)) {
+    stat->scalarMode = true;
+    return DEAL_RES_CONTINUE;
+  }
+
+  if (QUERY_NODE_OPERATOR == nodeType(*pNode)) {
+    SOperatorNode *node = (SOperatorNode *)*pNode;
+    if (!FLT_IS_COMPARISON_OPERATOR(node->opType)) {
+      stat->scalarMode = true;
+      return DEAL_RES_CONTINUE;
+    }
+
+    if (NULL == node->pRight) {
+      if (scalarGetOperatorParamNum(node->opType) > 1) {
+        fltError("invalid operator, pRight:%p, type:%d", node->pRight, nodeType(node));
+        stat->code = TSDB_CODE_QRY_APP_ERROR;
+        return DEAL_RES_ERROR;
+      }
+      
+      if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) {
+        stat->scalarMode = true;
+        return DEAL_RES_CONTINUE;
+      }
+    } else {
+      if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) {
+        stat->scalarMode = true;
+        return DEAL_RES_CONTINUE;
+      }
+
+      if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) {
+        stat->scalarMode = true;
+        return DEAL_RES_CONTINUE;
+      }      
+
+      if (nodeType(node->pLeft) == nodeType(node->pRight)) {
+        stat->scalarMode = true;
+        return DEAL_RES_CONTINUE;
+      }
+
+      if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) {
+        SNode *t = node->pLeft;
+        node->pLeft = node->pRight;
+        node->pRight = t;
+      }
+
+      if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) {
+        fltError("invalid IN operator node, rightType:%d", nodeType(node->pRight));
+        stat->code = TSDB_CODE_QRY_APP_ERROR;
+        return DEAL_RES_ERROR;
+      }
+
+      if (OP_TYPE_IN != node->opType) {
+        SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft;
+        SValueNode *valueNode = (SValueNode *)node->pRight;
+        int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type);
+        if (0 != type && type != refNode->dataType.type) {
+          stat->scalarMode = true;
+          return DEAL_RES_CONTINUE;
+        }
+      }
+    }
+
+    return DEAL_RES_CONTINUE;
+  }  
+  
+  fltError("invalid node type for filter, type:%d", nodeType(*pNode));
+  
+  stat->code = TSDB_CODE_QRY_INVALID_INPUT;
+  
+  return DEAL_RES_ERROR;
+}
+
+int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
+  nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat);
+
+  FLT_RET(pStat->code);
+}
+
+int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
 
+}
+
+
+int32_t filterGetDataFromColId(void *param, int32_t id, void **data) {
+  int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols;
+  SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock;
+  
+  for (int32_t j = 0; j < numOfCols; ++j) {
+    SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j);
+    if (id == pColInfo->info.colId) {
+      *data = pColInfo->pData;
+      break;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
+  return fltSetColFieldDataImpl(info, param, fp, false);
+}
+
+int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
+  return fltSetColFieldDataImpl(info, param, fp, true);
+}
+
+
+
+int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) {
+  int32_t code = 0;
+  SFilterInfo *info = NULL;
+  
+  if (pNode == NULL || pInfo == NULL) {
+    fltError("invalid param");
+    FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  if (*pInfo == NULL) {
+    *pInfo = calloc(1, sizeof(SFilterInfo));
+    if (NULL == *pInfo) {
+      fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo));
+      FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+    }
+  }
+
+  info = *pInfo;
+  info->options = options;
+
+  SFltTreeStat stat = {0};
+  FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat));
+
+  if (!info->scalarMode) {
+    FLT_ERR_JRET(fltInitFromNode(pNode, info, options));
+  } else {
+    info->sclCtx.node = pNode;
+    FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat));
+  }
+  
+  return code;
+
+_return:
+  
+  filterFreeInfo(*pInfo);
+
+  *pInfo = NULL;
+
+  FLT_RET(code);
+}
+
+FORCE_INLINE bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
+  if (info->scalarMode) {
+    SScalarParam output = {0};
+    FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output));
+
+    *p = output.data;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols);
+}
 
 
 

source/libs/scalar/src/scalar.c

+#include "nodes.h"
+#include "common.h"
+#include "querynodes.h"
+#include "function.h"
+#include "functionMgt.h"
+#include "sclvector.h"
+#include "sclInt.h"
+
+int32_t scalarGetOperatorParamNum(EOperatorType type) {
+  if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type) {
+    return 1;
+  }
+
+  return 2;
+}
+
+void sclFreeRes(SHashObj *res) {
+  SScalarParam *p = NULL;
+  void *pIter = taosHashIterate(res, NULL);
+  while (pIter) {
+    p = (SScalarParam *)pIter;
+
+    if (p) {
+      tfree(p->data);
+    }
+    
+    pIter = taosHashIterate(res, pIter);
+  }
+  
+  taosHashCleanup(res);
+}
+
+void sclFreeParam(SScalarParam *param) {
+  tfree(param->data);
+}
+
+int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) {
+  switch (nodeType(node)) {
+    case QUERY_NODE_VALUE: {
+      SValueNode *valueNode = (SValueNode *)node;
+      param->data = nodesGetValueFromNode(valueNode);
+      param->num = 1;
+      param->type = valueNode->node.resType.type;
+      param->bytes = valueNode->node.resType.bytes;
+      
+      break;
+    }
+    case QUERY_NODE_NODE_LIST: {
+      SNodeListNode *nodeList = (SNodeListNode *)node;
+      //TODO BUILD HASH
+      break;
+    }
+    case QUERY_NODE_COLUMN_REF: {
+      if (NULL == ctx) {
+        sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
+        SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+      
+      SColumnRefNode *ref = (SColumnRefNode *)node;
+      if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) {
+        sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock));
+        SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+      }
+
+      SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId);
+      param->data = columnData->pData;
+      param->num = ctx->pSrc->info.rows;
+      param->type = columnData->info.type;
+      param->bytes = columnData->info.bytes;
+      
+      break;
+    }
+    case QUERY_NODE_LOGIC_CONDITION:
+    case QUERY_NODE_OPERATOR: {
+      if (NULL == ctx) {
+        sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
+        SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+
+      SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES);
+      if (NULL == res) {
+        sclError("no result for node, type:%d, node:%p", nodeType(node), node);
+        SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+      }
+
+      *param = *res;
+      
+      break;
+    }
+  }
+
+  if (param->num > *rowNum) {
+    if ((1 != param->num) && (1 < *rowNum)) {
+      sclError("different row nums, rowNum:%d, newRowNum:%d", *rowNum, param->num);
+      SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+    }
+    
+    *rowNum = param->num;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t sclParamMoveNext(SScalarParam *params, int32_t num) {
+  SScalarParam *param = NULL;
+  
+  for (int32_t i = 0; i < num; ++i) {
+    param = params + i;
+    
+    if (1 == param->num) {
+      continue;
+    }
+
+    if (IS_VAR_DATA_TYPE(param->type)) {
+      param->data = (char *)(param->data) + varDataTLen(param->data);
+    } else {
+      param->data = (char *)(param->data) + tDataTypes[param->type].bytes;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) {
+  int32_t code = 0;
+  SScalarParam *paramList = calloc(pParamList->length, sizeof(SScalarParam));
+  if (NULL == paramList) {
+    sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam)));
+    SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  SListCell *cell = pParamList->pHead;
+  for (int32_t i = 0; i < pParamList->length; ++i) {
+    if (NULL == cell || NULL == cell->pNode) {
+      sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode);
+      SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
+    }
+
+    SCL_ERR_JRET(sclInitParam(cell->pNode, &paramList[i], ctx, rowNum));
+    
+    cell = cell->pNext;
+  }
+
+  *pParams = paramList;
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  tfree(paramList);
+  SCL_RET(code);
+}
+
+int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) {
+  int32_t code = 0;
+  int32_t paramNum = scalarGetOperatorParamNum(node->opType);
+  if (NULL == node->pLeft || (paramNum == 2 && NULL == node->pRight)) {
+    sclError("invalid operation node, left:%p, right:%p", node->pLeft, node->pRight);
+    SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+  
+  SScalarParam *paramList = calloc(paramNum, sizeof(SScalarParam));
+  if (NULL == paramList) {
+    sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam)));
+    SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  SCL_ERR_JRET(sclInitParam(node->pLeft, &paramList[0], ctx, rowNum));
+  if (paramNum > 1) {
+    SCL_ERR_JRET(sclInitParam(node->pRight, &paramList[1], ctx, rowNum));
+  }
+
+  *pParams = paramList;
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  tfree(paramList);
+  SCL_RET(code);
+}
+
+
+int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) {
+  if (NULL == node->pParameterList || node->pParameterList->length <= 0) {
+    sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0);
+    SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  SScalarFuncExecFuncs ffpSet = {0};
+  int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
+  if (code) {
+    sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
+    SCL_ERR_RET(code);
+  }
+
+  SScalarParam *params = NULL;
+  int32_t rowNum = 0;
+  SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
+
+  output->type = node->node.resType.type;
+  output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes));
+  if (NULL == output->data) {
+    sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes)));
+    SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  for (int32_t i = 0; i < rowNum; ++i) {
+    code = (*ffpSet.process)(params, node->pParameterList->length, output);
+    if (code) {
+      sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
+      SCL_ERR_JRET(code);    
+    }
+
+    sclParamMoveNext(output, 1);
+    sclParamMoveNext(params, node->pParameterList->length);
+  }
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  tfree(params);
+  SCL_RET(code);
+}
+
+
+int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *output) {
+  if (NULL == node->pParameterList || node->pParameterList->length <= 0) {
+    sclError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0);
+    SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  if (TSDB_DATA_TYPE_BOOL != node->node.resType.type) {
+    sclError("invalid logic resType, type:%d", node->node.resType.type);
+    SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  if (LOGIC_COND_TYPE_NOT == node->condType && node->pParameterList->length > 1) {
+    sclError("invalid NOT operation parameter number, paramNum:%d", node->pParameterList->length);
+    SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  SScalarParam *params = NULL;
+  int32_t rowNum = 0;
+  int32_t code = 0;
+  
+  SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
+
+  output->type = node->node.resType.type;
+  output->data = calloc(rowNum, sizeof(bool));
+  if (NULL == output->data) {
+    sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool)));
+    SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+  
+  bool value = false;
+
+  for (int32_t i = 0; i < rowNum; ++i) {
+    for (int32_t m = 0; m < node->pParameterList->length; ++m) {
+      GET_TYPED_DATA(value, bool, params[m].type, params[m].data);
+      
+      if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) {
+        break;
+      } else if (LOGIC_COND_TYPE_OR == node->condType && value) {
+        break;
+      } else if (LOGIC_COND_TYPE_NOT == node->condType) {
+        value = !value;
+      }
+    }
+
+    *(bool *)output->data = value;
+
+    sclParamMoveNext(output, 1);    
+    sclParamMoveNext(params, node->pParameterList->length);
+  }
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  tfree(params);
+  SCL_RET(code);
+}
+
+int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *output) {
+  SScalarParam *params = NULL;
+  int32_t rowNum = 0;
+  int32_t code = 0;
+  
+  SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
+
+  output->type = node->node.resType.type;
+  output->data = calloc(rowNum, tDataTypes[output->type].bytes);
+  if (NULL == output->data) {
+    sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes);
+    SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+
+  _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType);
+
+  int32_t paramNum = scalarGetOperatorParamNum(node->opType);
+  SScalarParam* pLeft = &params[0];
+  SScalarParam* pRight = paramNum > 1 ? &params[1] : NULL;
+
+  for (int32_t i = 0; i < rowNum; ++i) {
+
+    OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC);
+
+    sclParamMoveNext(output, 1);    
+    sclParamMoveNext(pLeft, 1);
+    if (pRight) {
+      sclParamMoveNext(pRight, 1);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  tfree(params);
+  SCL_RET(code);
+}
+
+
+EDealRes sclRewriteFunction(SNode** pNode, void* pContext) {
+  SFunctionNode *node = (SFunctionNode *)*pNode;
+  SScalarParam output = {0};
+  
+  *(int32_t *)pContext = sclExecFuncion(node, NULL, &output);
+  if (*(int32_t *)pContext) {
+    return DEAL_RES_ERROR;
+  }
+
+  SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
+  if (NULL == res) {
+    sclError("make value node failed");
+    sclFreeParam(&output);
+    *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return DEAL_RES_ERROR;
+  }
+
+  res->node.resType = node->node.resType;
+
+  if (IS_VAR_DATA_TYPE(output.type)) {
+    res->datum.p = output.data;
+    output.data = NULL;
+  } else {
+    memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes);
+  }
+  
+  nodesDestroyNode(*pNode);
+  *pNode = (SNode*)res;
+
+  sclFreeParam(&output);
+
+  return DEAL_RES_CONTINUE;
+}
+
+EDealRes sclRewriteLogic(SNode** pNode, void* pContext) {
+  SLogicConditionNode *node = (SLogicConditionNode *)*pNode;
+  SScalarParam output = {0};
+
+  *(int32_t *)pContext = sclExecLogic(node, NULL, &output);
+  if (*(int32_t *)pContext) {
+    return DEAL_RES_ERROR;
+  }
+
+  SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
+  if (NULL == res) {
+    sclError("make value node failed");
+    sclFreeParam(&output);    
+    *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return DEAL_RES_ERROR;
+  }
+
+  res->node.resType = node->node.resType;
+
+  if (IS_VAR_DATA_TYPE(output.type)) {
+    res->datum.p = output.data;
+    output.data = NULL;
+  } else {
+    memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes);
+  }
+
+  nodesDestroyNode(*pNode);
+  *pNode = (SNode*)res;
+
+  sclFreeParam(&output);
+
+  return DEAL_RES_CONTINUE;
+}
+
+EDealRes sclRewriteOperator(SNode** pNode, void* pContext) {
+  SOperatorNode *node = (SOperatorNode *)*pNode;
+  SScalarParam output = {0};
+
+  *(int32_t *)pContext = sclExecOperator(node, NULL, &output);
+  if (*(int32_t *)pContext) {
+    return DEAL_RES_ERROR;
+  }
+
+  SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
+  if (NULL == res) {
+    sclError("make value node failed");    
+    sclFreeParam(&output);    
+    *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return DEAL_RES_ERROR;
+  }
+
+  res->node.resType = node->node.resType;
+
+  if (IS_VAR_DATA_TYPE(output.type)) {
+    res->datum.p = output.data;
+    output.data = NULL;
+  } else {
+    memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes);
+  }
+
+  nodesDestroyNode(*pNode);
+  *pNode = (SNode*)res;
+
+  sclFreeParam(&output);    
+
+  return DEAL_RES_CONTINUE;
+}
+
+
+EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) {
+  if (QUERY_NODE_VALUE == nodeType(*pNode)) {
+    return DEAL_RES_CONTINUE;
+  }
+
+  if (QUERY_NODE_FUNCTION == nodeType(*pNode)) {
+    return sclRewriteFunction(pNode, pContext);
+  }
+
+  if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) {
+    return sclRewriteLogic(pNode, pContext);
+  }
+
+  if (QUERY_NODE_OPERATOR == nodeType(*pNode)) {
+    return sclRewriteOperator(pNode, pContext);
+  }  
+  
+  sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode));
+  
+  *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT;
+  
+  return DEAL_RES_ERROR;
+}
+
+
+EDealRes sclWalkFunction(SNode* pNode, void* pContext) {
+  SScalarCtx *ctx = (SScalarCtx *)pContext;
+  SFunctionNode *node = (SFunctionNode *)pNode;
+  SScalarParam output = {0};
+  
+  ctx->code = sclExecFuncion(node, ctx, &output);
+  if (ctx->code) {
+    return DEAL_RES_ERROR;
+  }
+
+  if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) {
+    ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return DEAL_RES_ERROR;
+  }
+
+  return DEAL_RES_CONTINUE;
+}
+
+
+EDealRes sclWalkLogic(SNode* pNode, void* pContext) {
+  SScalarCtx *ctx = (SScalarCtx *)pContext;
+  SLogicConditionNode *node = (SLogicConditionNode *)pNode;
+  SScalarParam output = {0};
+  
+  ctx->code = sclExecLogic(node, ctx, &output);
+  if (ctx->code) {
+    return DEAL_RES_ERROR;
+  }
+
+  if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) {
+    ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return DEAL_RES_ERROR;
+  }
+
+  return DEAL_RES_CONTINUE;
+}
+
+
+EDealRes sclWalkOperator(SNode* pNode, void* pContext) {
+  SScalarCtx *ctx = (SScalarCtx *)pContext;
+  SOperatorNode *node = (SOperatorNode *)pNode;
+  SScalarParam output = {0};
+  
+  ctx->code = sclExecOperator(node, ctx, &output);
+  if (ctx->code) {
+    return DEAL_RES_ERROR;
+  }
+
+  if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) {
+    ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return DEAL_RES_ERROR;
+  }
+
+  return DEAL_RES_CONTINUE;
+}
+
+
+EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
+  if (QUERY_NODE_VALUE == nodeType(pNode)) {
+    return DEAL_RES_CONTINUE;
+  }
+
+  if (QUERY_NODE_FUNCTION == nodeType(pNode)) {
+    return sclWalkFunction(pNode, pContext);
+  }
+
+  if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) {
+    return sclWalkLogic(pNode, pContext);
+  }
+
+  if (QUERY_NODE_OPERATOR == nodeType(pNode)) {
+    return sclWalkOperator(pNode, pContext);
+  }
+
+  sclError("invalid node type for calculating constants, type:%d", nodeType(pNode));
+
+  SScalarCtx *ctx = (SScalarCtx *)pContext;
+  
+  ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+  
+  return DEAL_RES_ERROR;
+}
+
+
+
+int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) {
+  if (NULL == pNode) {
+    SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  int32_t code = 0;
+  
+  nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&code);
+
+  if (code) {
+    nodesDestroyNode(pNode);
+    SCL_ERR_RET(code);
+  }
+
+  *pRes = pNode;
+
+  SCL_RET(code);
+}
+
+int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) {
+  if (NULL == pNode || NULL == pSrc || NULL == pDst) {
+    SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+  }
+
+  int32_t code = 0;
+  SScalarCtx ctx = {.code = 0, .pSrc = pSrc};
+  
+  ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  if (NULL == ctx.pRes) {
+    sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM);
+    SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }
+  
+  nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx);
+
+  if (ctx.code) {
+    nodesDestroyNode(pNode);
+    sclFreeRes(ctx.pRes);
+    SCL_ERR_RET(ctx.code);
+  }
+
+  SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES);
+  if (NULL == res) {
+    sclError("no res for calculating, node:%p, type:%d", pNode, nodeType(pNode));
+    SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+  
+  *pDst = *res;
+
+  nodesDestroyNode(pNode);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+

source/libs/function/src/tscalarfunction.c → source/libs/scalar/src/sclfunc.c

-#include "tscalarfunction.h"
-#include "tbinoperator.h"
-#include "tunaryoperator.h"
+#include "sclfunc.h"
+#include "sclvector.h"
 
-static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) {
+static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) {
   dst->type = src->type;
   dst->bytes = src->bytes;
   dst->num = src->num;
 }
 
-static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) {
+static void tceil(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
   assignBasicParaInfo(pOutput, pLeft);
   assert(numOfInput == 1);
 
@@ -34,7 +33,7 @@ static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFun
   }
 }
 
-static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) {
+static void tfloor(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
   assignBasicParaInfo(pOutput, pLeft);
   assert(numOfInput == 1);
 
@@ -62,7 +61,7 @@ static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFu
   }
 }
 
-static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) {
+static void _tabs(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
   assignBasicParaInfo(pOutput, pLeft);
   assert(numOfInput == 1);
 
@@ -120,7 +119,7 @@ static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFun
   }
 }
 
-static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) {
+static void tround(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
   assignBasicParaInfo(pOutput, pLeft);
   assert(numOfInput == 1);
 
@@ -146,7 +145,7 @@ static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFu
   }
 }
 
-static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) {
+static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
   assert(numOfInput == 1);
 
   int64_t* out = (int64_t*) pOutput->data;
@@ -157,7 +156,7 @@ static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarF
   }
 }
 
-static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) {
+static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
   assert(numOfInput > 0);
 
   int32_t rowLen = 0;
@@ -189,11 +188,11 @@ static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarF
   }
 }
 
-static void tltrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) {
+static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
 
 }
 
-static void trtrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) {
+static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
 
 }
 
@@ -262,7 +261,7 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf
   }
 }
 
-static void setScalarFuncParam(SScalarFuncParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) {
+static void setScalarFuncParam(SScalarParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) {
   param->bytes = bytes;
   param->type = type;
   param->num  = numOfRows;
@@ -273,6 +272,7 @@ bool isStringOp(int32_t op) {
   return op == TSDB_BINARY_OP_CONCAT;
 }
 
+#if 0
 int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param,
                           char* (*getSourceDataBlock)(void*, const char*, int32_t)) {
   if (pExprs == NULL) {
@@ -361,6 +361,8 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa
 
   return 0;
 }
+#endif
+
 
 SScalarFunctionInfo scalarFunc[8] = {
     {"ceil",   FUNCTION_TYPE_SCALAR, FUNCTION_CEIL,   tceil},

source/libs/function/src/tbinoperator.c → source/libs/scalar/src/sclvector.c

@@ -16,8 +16,11 @@
 #include "os.h"
 
 #include "ttypes.h"
-#include "tbinoperator.h"
+#include "sclvector.h"
 #include "tcompare.h"
+#include "querynodes.h"
+#include "filterInt.h"
+#include "query.h"
 
 //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i]));                                
 
@@ -119,6 +122,70 @@ _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
 }
 
 
+
+typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index);
+
+int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) {
+  return (int64_t)*((int8_t *)src + index);
+}
+int64_t getVectorBigintValue_UTINYINT(void *src, int32_t index) {
+  return (int64_t)*((uint8_t *)src + index);
+}
+int64_t getVectorBigintValue_SMALLINT(void *src, int32_t index) {
+  return (int64_t)*((int16_t *)src + index);
+}
+int64_t getVectorBigintValue_USMALLINT(void *src, int32_t index) {
+  return (int64_t)*((uint16_t *)src + index);
+}
+int64_t getVectorBigintValue_INT(void *src, int32_t index) {
+  return (int64_t)*((int32_t *)src + index);
+}
+int64_t getVectorBigintValue_UINT(void *src, int32_t index) {
+  return (int64_t)*((uint32_t *)src + index);
+}
+int64_t getVectorBigintValue_BIGINT(void *src, int32_t index) {
+  return (int64_t)*((int64_t *)src + index);
+}
+int64_t getVectorBigintValue_UBIGINT(void *src, int32_t index) {
+  return (int64_t)*((uint64_t *)src + index);
+}
+int64_t getVectorBigintValue_FLOAT(void *src, int32_t index) {
+  return (int64_t)*((float *)src + index);
+}
+int64_t getVectorBigintValue_DOUBLE(void *src, int32_t index) {
+  return (int64_t)*((double *)src + index);
+}
+_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
+    _getBigintValue_fn_t p = NULL;
+    if(srcType==TSDB_DATA_TYPE_TINYINT) {
+        p = getVectorBigintValue_TINYINT;
+    }else if(srcType==TSDB_DATA_TYPE_UTINYINT) {
+        p = getVectorBigintValue_UTINYINT;
+    }else if(srcType==TSDB_DATA_TYPE_SMALLINT) {
+        p = getVectorBigintValue_SMALLINT;
+    }else if(srcType==TSDB_DATA_TYPE_USMALLINT) {
+        p = getVectorBigintValue_USMALLINT;
+    }else if(srcType==TSDB_DATA_TYPE_INT) {
+        p = getVectorBigintValue_INT;
+    }else if(srcType==TSDB_DATA_TYPE_UINT) {
+        p = getVectorBigintValue_UINT;
+    }else if(srcType==TSDB_DATA_TYPE_BIGINT) {
+        p = getVectorBigintValue_BIGINT;
+    }else if(srcType==TSDB_DATA_TYPE_UBIGINT) {
+        p = getVectorBigintValue_UBIGINT;
+    }else if(srcType==TSDB_DATA_TYPE_FLOAT) {
+        p = getVectorBigintValue_FLOAT;
+    }else if(srcType==TSDB_DATA_TYPE_DOUBLE) {
+        p = getVectorBigintValue_DOUBLE;
+    }else {
+        assert(0);
+    }
+    return p;
+}
+
+
+
+
 typedef void* (*_getValueAddr_fn_t)(void *src, int32_t index);
 
 void* getVectorValueAddr_TINYINT(void *src, int32_t index) {
@@ -180,7 +247,351 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) {
     return p;
 }
 
-void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
+
+int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) {
+  int16_t inType = pIn->type; 
+  int16_t inBytes = pIn->bytes;
+  char *input = pIn->data; 
+  int16_t outType = pOut->type; 
+  int16_t outBytes = pOut->bytes; 
+  char *output = pOut->data;
+
+  switch (outType) {
+    case TSDB_DATA_TYPE_BOOL:
+      if (inType == TSDB_DATA_TYPE_BINARY) {
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input));
+          
+          input += varDataLen(input) + VARSTR_HEADER_SIZE;
+          output += sizeof(bool);
+        }
+      } else if (inType == TSDB_DATA_TYPE_NCHAR) {      
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input));
+          
+          input += varDataLen(input) + VARSTR_HEADER_SIZE;
+          output += tDataTypes[outType].bytes;
+        }
+      } else {
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          uint64_t value = 0;
+          GET_TYPED_DATA(value, uint64_t, inType, input);
+          SET_TYPED_DATA(output, outType, value);
+
+          input += tDataTypes[inType].bytes;
+          output += tDataTypes[outType].bytes;
+        }
+      }
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP:    
+      if (inType == TSDB_DATA_TYPE_BINARY) {
+        int32_t bufSize = varDataLen(input) + 1;
+        char *tmp = malloc(bufSize);
+        if (NULL == tmp) {
+          return TSDB_CODE_QRY_OUT_OF_MEMORY;
+        }
+        
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          if (varDataLen(input) >= bufSize) {
+            bufSize = varDataLen(input) + 1;
+            tmp = realloc(tmp, bufSize);
+          }
+          
+          memcpy(tmp, varDataVal(input), varDataLen(input));
+          tmp[varDataLen(input)] = 0;
+          
+          int64_t value = strtoll(tmp, NULL, 10);
+          SET_TYPED_DATA(output, outType, value);
+          
+          input += varDataLen(input) + VARSTR_HEADER_SIZE;
+          output += tDataTypes[outType].bytes;
+        }
+        
+        tfree(tmp);
+      } else if (inType == TSDB_DATA_TYPE_NCHAR) {      
+        int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
+        char *tmp = calloc(1, bufSize);
+        if (NULL == tmp) {
+          return TSDB_CODE_QRY_OUT_OF_MEMORY;
+        }
+        
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) {
+            bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
+            tmp = realloc(tmp, bufSize);
+          }
+        
+          int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp);
+          if (len < 0){
+            qError("castConvert taosUcs4ToMbs error 1");
+            tfree(tmp);
+            return TSDB_CODE_QRY_APP_ERROR;
+          }
+          
+          tmp[len] = 0;
+          int64_t value = strtoll(tmp, NULL, 10);
+          SET_TYPED_DATA(output, outType, value);
+
+          input += varDataLen(input) + VARSTR_HEADER_SIZE;
+          output += tDataTypes[outType].bytes;
+        }
+        
+        tfree(tmp);
+      } else {
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          int64_t value = 0;
+          GET_TYPED_DATA(value, int64_t, inType, input);
+          SET_TYPED_DATA(output, outType, value);
+
+          input += tDataTypes[inType].bytes;
+          output += tDataTypes[outType].bytes;
+        }
+      }
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_UBIGINT:
+      if (inType == TSDB_DATA_TYPE_BINARY) {
+        int32_t bufSize = varDataLen(input) + 1;
+        char *tmp = malloc(bufSize);
+        if (NULL == tmp) {
+          return TSDB_CODE_QRY_OUT_OF_MEMORY;
+        }
+        
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          if (varDataLen(input) >= bufSize) {
+            bufSize = varDataLen(input) + 1;
+            tmp = realloc(tmp, bufSize);
+          }
+          
+          memcpy(tmp, varDataVal(input), varDataLen(input));
+          tmp[varDataLen(input)] = 0;
+          uint64_t value = strtoull(tmp, NULL, 10);
+          SET_TYPED_DATA(output, outType, value);
+          
+          input += varDataLen(input) + VARSTR_HEADER_SIZE;
+          output += tDataTypes[outType].bytes;
+        }
+        
+        tfree(tmp);
+      } else if (inType == TSDB_DATA_TYPE_NCHAR) {      
+        int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
+        char *tmp = calloc(1, bufSize);
+        if (NULL == tmp) {
+          return TSDB_CODE_QRY_OUT_OF_MEMORY;
+        }
+        
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) {
+            bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
+            tmp = realloc(tmp, bufSize);
+          }
+        
+          int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp);
+          if (len < 0){
+            qError("castConvert taosUcs4ToMbs error 1");
+            tfree(tmp);
+            return TSDB_CODE_QRY_APP_ERROR;
+          }
+          
+          tmp[len] = 0;
+          uint64_t value = strtoull(tmp, NULL, 10);
+          SET_TYPED_DATA(output, outType, value);
+
+          input += varDataLen(input) + VARSTR_HEADER_SIZE;
+          output += tDataTypes[outType].bytes;
+        }
+        
+        tfree(tmp);
+      } else {      
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          uint64_t value = 0;
+          GET_TYPED_DATA(value, uint64_t, inType, input);
+          SET_TYPED_DATA(output, outType, value);
+
+          input += tDataTypes[inType].bytes;
+          output += tDataTypes[outType].bytes;
+        }
+      }
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+    case TSDB_DATA_TYPE_DOUBLE:
+      if (inType == TSDB_DATA_TYPE_BINARY) {
+        int32_t bufSize = varDataLen(input) + 1;
+        char *tmp = malloc(bufSize);
+        if (NULL == tmp) {
+          return TSDB_CODE_QRY_OUT_OF_MEMORY;
+        }
+        
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          if (varDataLen(input) >= bufSize) {
+            bufSize = varDataLen(input) + 1;
+            tmp = realloc(tmp, bufSize);
+          }
+          
+          memcpy(tmp, varDataVal(input), varDataLen(input));
+          tmp[varDataLen(input)] = 0;
+          
+          double value = strtod(tmp, NULL);
+          SET_TYPED_DATA(output, outType, value);
+          
+          input += varDataLen(input) + VARSTR_HEADER_SIZE;
+          output += tDataTypes[outType].bytes;
+        }
+        
+        tfree(tmp);
+      } else if (inType == TSDB_DATA_TYPE_NCHAR) {      
+        int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
+        char *tmp = calloc(1, bufSize);
+        if (NULL == tmp) {
+          return TSDB_CODE_QRY_OUT_OF_MEMORY;
+        }
+        
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) {
+            bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1;
+            tmp = realloc(tmp, bufSize);
+          }
+        
+          int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp);
+          if (len < 0){
+            qError("castConvert taosUcs4ToMbs error 1");
+            tfree(tmp);
+            return TSDB_CODE_QRY_APP_ERROR;
+          }
+          
+          tmp[len] = 0;
+          double value = strtod(tmp, NULL);
+          SET_TYPED_DATA(output, outType, value);
+
+          input += varDataLen(input) + VARSTR_HEADER_SIZE;
+          output += tDataTypes[outType].bytes;
+        }
+        
+        tfree(tmp);
+      } else {
+        for (int32_t i = 0; i < pIn->num; ++i) {
+          int64_t value = 0;
+          GET_TYPED_DATA(value, int64_t, inType, input);
+          SET_TYPED_DATA(output, outType, value);
+
+          input += tDataTypes[inType].bytes;
+          output += tDataTypes[outType].bytes;
+        }
+      }
+      break;      
+    default:
+      qError("invalid convert output type:%d", outType);
+      return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = {
+/*         NULL BOOL TINY SMAL INT  BIG  FLOA DOUB BINA TIME NCHA UTIN USMA UINT UBIG VARC VARB JSON DECI BLOB */
+/*NULL*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+/*BOOL*/   0,   0,   0,   3,   4,   5,   6,   7,   7,   9,   7,   0,   12,  13,  14,  7,   7,   0,   0,   0,
+/*TINY*/   0,   0,   0,   3,   4,   5,   6,   7,   7,   9,   7,   3,   4,   5,   7,   7,   7,   0,   0,   0,
+/*SMAL*/   0,   0,   0,   0,   4,   5,   6,   7,   7,   9,   7,   3,   4,   5,   7,   7,   7,   0,   0,   0,
+/*INT */   0,   0,   0,   0,   0,   5,   6,   7,   7,   9,   7,   4,   4,   5,   7,   7,   7,   0,   0,   0,
+/*BIGI*/   0,   0,   0,   0,   0,   0,   6,   7,   7,   0,   7,   5,   5,   5,   7,   7,   7,   0,   0,   0,
+/*FLOA*/   0,   0,   0,   0,   0,   0,   0,   7,   7,   6,   7,   6,   6,   6,   6,   7,   7,   0,   0,   0,
+/*DOUB*/   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   7,   7,   7,   7,   7,   0,   0,   0,
+/*BINA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   0,   7,   7,   7,   7,   0,   0,   0,   0,   0,
+/*TIME*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   9,   9,   9,   7,   7,   7,   0,   0,   0,
+/*NCHA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   0,   0,   0,   0,   0,
+/*UTIN*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   12,  13,  14,  7,   7,   0,   0,   0,
+/*USMA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   13,  14,  7,   7,   0,   0,   0,
+/*UINT*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   14,  7,   7,   0,   0,   0,
+/*UBIG*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   0,   0,   0,
+/*VARC*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+/*VARB*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+/*JSON*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+/*DECI*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+/*BLOB*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0
+};
+
+int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
+  if (type1 == type2) {
+    return 0;
+  }
+
+  if (type1 < type2) {
+    return gConvertTypes[type1][type2];
+  }
+
+  return gConvertTypes[type2][type1];
+}
+
+int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) {
+  if (pLeft->type == pRight->type) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SScalarParam *param1 = NULL, *paramOut1 = NULL; 
+  SScalarParam *param2 = NULL, *paramOut2 = NULL;
+  int32_t code = 0;
+  
+  if (pLeft->type < pRight->type) {
+    param1 = pLeft;
+    param2 = pRight;
+    paramOut1 = pLeftOut;
+    paramOut2 = pRightOut;
+  } else {
+    param1 = pRight;
+    param2 = pLeft;
+    paramOut1 = pRightOut;
+    paramOut2 = pLeftOut;
+  }
+
+
+  int8_t type = vectorGetConvertType(param1->type, param2->type);
+  if (0 == type) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (type != param1->type) {
+    paramOut1->bytes = param1->bytes;
+    paramOut1->type = type;
+    paramOut1->num = param1->num;
+    paramOut1->data = malloc(paramOut1->num * tDataTypes[paramOut1->type].bytes);
+    if (NULL == paramOut1->data) {
+      return TSDB_CODE_QRY_OUT_OF_MEMORY;
+    }
+    
+    code = vectorConvertImpl(param1, paramOut1);
+    if (code) {
+      tfree(paramOut1->data);
+      return code;
+    }
+  }
+  
+  if (type != param2->type) {
+    paramOut2->bytes = param2->bytes;
+    paramOut2->type = type;
+    paramOut2->num = param2->num;
+    paramOut2->data = malloc(paramOut2->num * tDataTypes[paramOut2->type].bytes);
+    if (NULL == paramOut2->data) {
+      tfree(paramOut1->data);
+      return TSDB_CODE_QRY_OUT_OF_MEMORY;
+    }
+    
+    code = vectorConvertImpl(param2, paramOut2);
+    if (code) {
+      tfree(paramOut1->data);
+      tfree(paramOut2->data);
+      return code;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
   int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
   int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
 
@@ -219,7 +630,7 @@ void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int
   }
 }
 
-void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
+void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
   int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
   int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
 
@@ -256,7 +667,7 @@ void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int
     }
   }
 }
-void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
+void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
   int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
   int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
 
@@ -295,7 +706,7 @@ void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out
   }                                                                                                 
 }
 
-void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
+void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
   int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
   int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
 
@@ -341,7 +752,7 @@ void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out,
   }
 }
 
-void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
+void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
   int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
   int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
 
@@ -413,7 +824,7 @@ void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *ou
   }                                                                                                 
 }
 
-void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
+void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
   int32_t len = pLeft->bytes + pRight->bytes;
 
   int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
@@ -463,20 +874,298 @@ void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out,
 
 }
 
+
+void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
+
+  int64_t *output=(int64_t *)out;
+  _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
+  _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
+  _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type);
+  _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type);
+
+  if (pLeft->num == pRight->num) {
+    for (; i < pRight->num && i >= 0; i += step, output += 1) {
+      if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) ||
+          isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) {
+        SET_BIGINT_NULL(output);
+        continue;
+      }
+
+      SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i));
+    }
+  } else if (pLeft->num == 1) {
+    for (; i >= 0 && i < pRight->num; i += step, output += 1) {
+      if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) {
+        SET_BIGINT_NULL(output);
+        continue;
+      }
+      SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i));
+    }
+  } else if (pRight->num == 1) {
+    for (; i >= 0 && i < pLeft->num; i += step, output += 1) {
+      if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) {
+        SET_BIGINT_NULL(output);
+        continue;
+      }
+      SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0));
+    }
+  }
+}
+
+void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
+
+  int64_t *output=(int64_t *)out;
+  _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
+  _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
+  _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type);
+  _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type);
+
+  if (pLeft->num == pRight->num) {
+    for (; i < pRight->num && i >= 0; i += step, output += 1) {
+      if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) ||
+          isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) {
+        SET_BIGINT_NULL(output);
+        continue;
+      }
+
+      SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i));
+    }
+  } else if (pLeft->num == 1) {
+    for (; i >= 0 && i < pRight->num; i += step, output += 1) {
+      if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) {
+        SET_BIGINT_NULL(output);
+        continue;
+      }
+      SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i));
+    }
+  } else if (pRight->num == 1) {
+    for (; i >= 0 && i < pLeft->num; i += step, output += 1) {
+      if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) {
+        SET_BIGINT_NULL(output);
+        continue;
+      }
+      SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0));
+    }
+  }
+}
+
+
+void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) {
+  int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
+  __compar_fn_t fp = filterGetCompFunc(pLeft->type, optr);
+  bool res = false;
+  
+  bool *output=(bool *)out;
+  _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
+  _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type);
+
+  if (pLeft->num == pRight->num) {    
+    for (; i < pRight->num && i >= 0; i += step, output += 1) {
+      if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) ||
+          isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) {
+        res = false;
+        SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+        continue;
+      }
+      
+      res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data, i), getVectorValueAddrFnRight(pRight->data,i));
+
+      SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+    }
+  } else if (pLeft->num == 1) {
+    void *leftData = getVectorValueAddrFnLeft(pLeft->data, 0);
+    
+    for (; i >= 0 && i < pRight->num; i += step, output += 1) {
+      if (isNull(leftData, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) {
+        res = false;
+        SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+        continue;
+      }
+
+      res = filterDoCompare(fp, optr, leftData, getVectorValueAddrFnRight(pRight->data,i));
+
+      SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+    }
+  } else if (pRight->num == 1) {
+      void *rightData = getVectorValueAddrFnRight(pRight->data, 0);
+
+    for (; i >= 0 && i < pLeft->num; i += step, output += 1) {
+      if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) {
+        res = false;
+        SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+        continue;
+      }
+
+      res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data,i), rightData);
+
+      SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+    }
+  }
+}
+
+void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) {
+  SScalarParam pLeftOut = {0}; 
+  SScalarParam pRightOut = {0};
+  
+  vectorConvert(pLeft, pRight, &pLeftOut, &pRightOut);
+
+  SScalarParam *param1 = NULL; 
+  SScalarParam *param2 = NULL;
+
+  int32_t type = 0;
+  if (pLeftOut.type) {
+    param1 = &pLeftOut;
+  } else {
+    param1 = pLeft;
+  }
+
+  if (pRightOut.type) {
+    param2 = &pRightOut;
+  } else {
+    param2 = pRight;
+  }
+
+  vectorCompareImpl(param1, param2, out, _ord, TSDB_RELATION_GREATER);
+}
+
+void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER);
+}
+
+void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER_EQUAL);
+}
+
+void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS);
+}
+
+void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS_EQUAL);
+}
+
+void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_EQUAL);
+}
+
+void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_EQUAL);
+}
+
+void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_IN);
+}
+
+void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_IN);
+}
+
+void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LIKE);
+}
+
+void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_LIKE);
+}
+
+void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_MATCH);
+}
+
+void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NMATCH);
+}
+
+void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
+  bool res = false;
+  
+  bool *output=(bool *)out;
+  _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
+
+  for (; i >= 0 && i < pLeft->num; i += step, output += 1) {
+    if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) {
+      res = true;
+      SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+      continue;
+    }
+
+    res = false;
+    SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+  }
+}
+
+void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
+  int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
+  bool res = false;
+  
+  bool *output = (bool *)out;
+  _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type);
+
+  for (; i >= 0 && i < pLeft->num; i += step, output += 1) {
+    if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) {
+      res = false;
+      SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+      continue;
+    }
+
+    res = true;
+    SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res);
+  }
+}
+
+
 _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
   switch (binFunctionId) {
-    case TSDB_BINARY_OP_ADD:
+    case OP_TYPE_ADD:
       return vectorAdd;
-    case TSDB_BINARY_OP_SUBTRACT:
+    case OP_TYPE_SUB:
       return vectorSub;
-    case TSDB_BINARY_OP_MULTIPLY:
+    case OP_TYPE_MULTI:
       return vectorMultiply;
-    case TSDB_BINARY_OP_DIVIDE:
+    case OP_TYPE_DIV:
       return vectorDivide;
-    case TSDB_BINARY_OP_REMAINDER:
+    case OP_TYPE_MOD:
       return vectorRemainder;
-    case TSDB_BINARY_OP_CONCAT:
-      return vectorConcat;
+    case OP_TYPE_GREATER_THAN:
+      return vectorGreater;
+    case OP_TYPE_GREATER_EQUAL:
+      return vectorGreaterEqual;
+    case OP_TYPE_LOWER_THAN:
+      return vectorLower;
+    case OP_TYPE_LOWER_EQUAL:
+      return vectorLowerEqual;
+    case OP_TYPE_EQUAL:
+      return vectorEqual;
+    case OP_TYPE_NOT_EQUAL:
+      return vectorNotEqual;
+    case OP_TYPE_IN:
+      return vectorIn;
+    case OP_TYPE_NOT_IN:
+      return vectorNotIn;
+    case OP_TYPE_LIKE:
+      return vectorLike;
+    case OP_TYPE_NOT_LIKE:
+      return vectorNotLike;
+    case OP_TYPE_MATCH:
+      return vectorMatch;
+    case OP_TYPE_NMATCH:
+      return vectorNotMatch;
+    case OP_TYPE_IS_NULL:
+      return vectorIsNull;
+    case OP_TYPE_IS_NOT_NULL:
+      return vectorNotNull;
+    case OP_TYPE_BIT_AND:
+      return vectorBitAnd;
+    case OP_TYPE_BIT_OR:
+      return vectorBitOr;
     default:
       assert(0);
       return NULL;

source/libs/scalar/test/CMakeLists.txt

+
+MESSAGE(STATUS "build scalar unit test")
+
+# GoogleTest requires at least C++11
+SET(CMAKE_CXX_STANDARD 11)
+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
+TARGET_LINK_LIBRARIES(
+        scalarTest
+        PUBLIC os util common gtest qcom function nodes
+)
+
+TARGET_INCLUDE_DIRECTORIES(
+        scalarTest
+        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/"
+        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc"
+)

source/libs/scalar/test/scalarTests.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+#include <tglobal.h>
+#include <iostream>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+#pragma GCC diagnostic ignored "-Wformat"
+#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
+#pragma GCC diagnostic ignored "-Wpointer-arith"
+
+#include "os.h"
+
+#include "taos.h"
+#include "tdef.h"
+#include "tvariant.h"
+#include "tep.h"
+#include "stub.h"
+#include "addr_any.h"
+#include "scalar.h"
+
+namespace {
+
+}
+
+TEST(scalarTest, func) {
+
+}
+
+
+int main(int argc, char** argv) {
+  srand(time(NULL));
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+#pragma GCC diagnostic pop

source/util/src/compare.c

@@ -23,23 +23,51 @@
 #include "thash.h"
 #include "types.h"
 #include "ulog.h"
+#include "tdef.h"
+#include "taos.h"
 
-int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
+int32_t setChkInBytes1(const void *pLeft, const void *pRight) {
   return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
 }
 
-int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
+int32_t setChkInBytes2(const void *pLeft, const void *pRight) {
   return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
 }
 
-int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
+int32_t setChkInBytes4(const void *pLeft, const void *pRight) {
   return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
 }
 
-int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
+int32_t setChkInBytes8(const void *pLeft, const void *pRight) {
   return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
 }
 
+int32_t setChkNotInBytes1(const void *pLeft, const void *pRight) {
+  return NULL == taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
+}
+
+int32_t setChkNotInBytes2(const void *pLeft, const void *pRight) {
+  return NULL == taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
+}
+
+int32_t setChkNotInBytes4(const void *pLeft, const void *pRight) {
+  return NULL == taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
+}
+
+int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) {
+  return NULL == taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
+}
+
+
+int32_t compareChkInString(const void *pLeft, const void* pRight)  {
+  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+}
+
+int32_t compareChkNotInString(const void *pLeft, const void* pRight)  {
+  return NULL == taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+}
+
+
 int32_t compareInt8Val(const void *pLeft, const void *pRight) {
   int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
   if (left > right) return 1;
@@ -393,6 +421,156 @@ int32_t taosArrayCompareString(const void* a, const void* b) {
   return compareLenPrefixedStr(x, y);
 }
 
-int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
-  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
+
+int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) {
+  SPatternCompareInfo pInfo = {'%', '_'};
+
+  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
+  char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char));
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
+
+  size_t sz = varDataLen(pLeft);
+  char *buf = malloc(sz + 1);
+  memcpy(buf, varDataVal(pLeft), sz);
+  buf[sz] = 0;
+
+  int32_t ret = patternMatch(pattern, buf, sz, &pInfo);
+  free(buf);
+  free(pattern);
+  return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
+}
+
+int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight) {
+  return compareStrPatternMatch(pLeft, pRight) ? 0 : 1;
 }
+
+int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) {
+  SPatternCompareInfo pInfo = {'%', '_'};
+
+  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
+
+  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
+
+  int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
+  free(pattern);
+
+  return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
+}
+
+int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) {
+  return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1;
+}
+
+
+__compar_fn_t getComparFunc(int32_t type, int32_t optr) {
+  __compar_fn_t comparFn = NULL;
+
+  if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
+    switch (type) {
+      case TSDB_DATA_TYPE_BOOL:
+      case TSDB_DATA_TYPE_TINYINT:
+      case TSDB_DATA_TYPE_UTINYINT:
+        return setChkInBytes1;
+      case TSDB_DATA_TYPE_SMALLINT:
+      case TSDB_DATA_TYPE_USMALLINT:
+        return setChkInBytes2;
+      case TSDB_DATA_TYPE_INT:
+      case TSDB_DATA_TYPE_UINT:
+      case TSDB_DATA_TYPE_FLOAT:
+        return setChkInBytes4;
+      case TSDB_DATA_TYPE_BIGINT:
+      case TSDB_DATA_TYPE_UBIGINT:
+      case TSDB_DATA_TYPE_DOUBLE:
+      case TSDB_DATA_TYPE_TIMESTAMP:
+        return setChkInBytes8;
+      default:
+        assert(0);
+    }
+  }
+
+  if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
+    switch (type) {
+      case TSDB_DATA_TYPE_BOOL:
+      case TSDB_DATA_TYPE_TINYINT:
+      case TSDB_DATA_TYPE_UTINYINT:
+        return setChkNotInBytes1;
+      case TSDB_DATA_TYPE_SMALLINT:
+      case TSDB_DATA_TYPE_USMALLINT:
+        return setChkNotInBytes2;
+      case TSDB_DATA_TYPE_INT:
+      case TSDB_DATA_TYPE_UINT:
+      case TSDB_DATA_TYPE_FLOAT:
+        return setChkNotInBytes4;
+      case TSDB_DATA_TYPE_BIGINT:
+      case TSDB_DATA_TYPE_UBIGINT:
+      case TSDB_DATA_TYPE_DOUBLE:
+      case TSDB_DATA_TYPE_TIMESTAMP:
+        return setChkNotInBytes8;
+      default:
+        assert(0);
+    }
+  }
+
+  switch (type) {
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_TINYINT:   comparFn = compareInt8Val;   break;
+    case TSDB_DATA_TYPE_SMALLINT:  comparFn = compareInt16Val;  break;
+    case TSDB_DATA_TYPE_INT:       comparFn = compareInt32Val;  break;
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val;  break;
+    case TSDB_DATA_TYPE_FLOAT:     comparFn = compareFloatVal;  break;
+    case TSDB_DATA_TYPE_DOUBLE:    comparFn = compareDoubleVal; break;
+    case TSDB_DATA_TYPE_BINARY: {
+      if (optr == TSDB_RELATION_MATCH) {
+        comparFn = compareStrRegexCompMatch;
+      } else if (optr == TSDB_RELATION_NMATCH) {
+        comparFn = compareStrRegexCompNMatch;
+      } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
+        comparFn = compareStrPatternMatch;
+      } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */
+        comparFn = compareStrPatternNotMatch;
+      } else if (optr == TSDB_RELATION_IN) {
+        comparFn = compareChkInString;
+      } else if (optr == TSDB_RELATION_NOT_IN) {
+        comparFn = compareChkNotInString;
+      } else { /* normal relational comparFn */
+        comparFn = compareLenPrefixedStr;
+      }
+
+      break;
+    }
+
+    case TSDB_DATA_TYPE_NCHAR: {
+      if (optr == TSDB_RELATION_MATCH) {
+        comparFn = compareStrRegexCompMatch;
+      } else if (optr == TSDB_RELATION_NMATCH) {
+        comparFn = compareStrRegexCompNMatch;
+      } else if (optr == TSDB_RELATION_LIKE) {
+        comparFn = compareWStrPatternMatch;
+      } else if (optr == TSDB_RELATION_NOT_LIKE) {
+        comparFn = compareWStrPatternNotMatch;
+      } else if (optr == TSDB_RELATION_IN) {
+        comparFn = compareChkInString;
+      } else if (optr == TSDB_RELATION_NOT_IN) {
+        comparFn = compareChkNotInString;
+      } else {
+        comparFn = compareLenPrefixedWStr;
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_UTINYINT:  comparFn = compareUint8Val; break;
+    case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break;
+    case TSDB_DATA_TYPE_UINT:      comparFn = compareUint32Val;break;
+    case TSDB_DATA_TYPE_UBIGINT:   comparFn = compareUint64Val;break;
+
+    default:
+      comparFn = compareInt32Val;
+      break;
+  }
+
+  return comparFn;
+}
+
+

source/util/src/thashutil.c

@@ -164,20 +164,20 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) {
   return fn;
 }
 
-//int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) {
-//  return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b);
-//}
-//
-//int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) {
-//  return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b);
-//}
+int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) {
+  return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b);
+}
+
+int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) {
+  return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b);
+}
 
 _equal_fn_t taosGetDefaultEqualFunction(int32_t type) {
   _equal_fn_t fn = NULL;
-//  switch (type) {
-//    case TSDB_DATA_TYPE_FLOAT:  fn = taosFloatEqual;  break;
-//    case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break;
-//    default: fn = memcmp; break;
-//  }
+  switch (type) {
+    case TSDB_DATA_TYPE_FLOAT:  fn = taosFloatEqual;  break;
+    case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break;
+    default: fn = memcmp; break;
+  }
   return fn;
 }