Merge pull request #12263 from taosdata/feature/TS-1436-2.4

feat(query): added bitwise operations

src/client/src/tscSQLParser.c

@@ -145,6 +145,8 @@ static int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo);
 static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex);
 static void convertWhereStringCharset(tSqlExpr* pRight);
 
+static bool isLogicalOperator(tSqlExpr* pExpr);
+static bool isComparisonOperator(tSqlExpr* pExpr);
 int validateTableName(char *tblName, int len, SStrToken* psTblToken, bool *dbIncluded);
 
 static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) {
@@ -284,6 +286,16 @@ static uint8_t convertRelationalOperator(SStrToken *pToken) {
       return TSDB_BINARY_OP_REMAINDER;
     case TK_BITAND:
       return TSDB_BINARY_OP_BITAND;
+    case TK_BITOR:
+      return TSDB_BINARY_OP_BITOR;
+    case TK_BITXOR:
+      return TSDB_BINARY_OP_BITXOR;
+    case TK_BITNOT:
+      return TSDB_BINARY_OP_BITNOT;
+    case TK_LSHIFT:
+      return TSDB_BINARY_OP_LSHIFT;
+    case TK_RSHIFT:
+      return TSDB_BINARY_OP_RSHIFT;
     case TK_LIKE:
       return TSDB_RELATION_LIKE;
     case TK_MATCH:
@@ -4274,15 +4286,10 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
 
   STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
   SSchema*    pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
-  int32_t     ret = 0;
+  int32_t     ret = TSDB_CODE_SUCCESS;
   const char* msg1 = "non binary column not support like/match operator";
   const char* msg3 = "bool column not support this operator";
   const char* msg4 = "primary key not support this operator";
-
-  SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->id.uid, pSchema);
-
-  pColumn->info.flist.numOfFilters++;
-  
   /*
    * in case of TK_AND filter condition, we first find the corresponding column and build the query condition together
    * the already existed condition.
@@ -4296,8 +4303,8 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
   pColFilter->filterstr =
       ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0);
 
-  if (!pColFilter->filterstr) {
-    if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) {
+  if (!pColFilter->filterstr && tSqlExprIsParentOfLeaf(pExpr)) {
+     if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) {
       ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
       goto _err_ret;
     }
@@ -4310,11 +4317,10 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
       }
     }
   }
-
-  pColumn->columnIndex = pIndex->columnIndex;
-  pColumn->tableUid = pTableMeta->id.uid;
-  if (pColumn->columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pExpr->tokenId == TK_IN) {
-     return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
+  if (pIndex->columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pTableMeta->id.uid == TK_IN
+      && tSqlExprIsParentOfLeaf(pExpr)) {
+     ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
+     goto _err_ret;
   }
 
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
@@ -4326,8 +4332,32 @@ _err_ret:
   return ret;
 }
 
+static int32_t addAllColumn(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, uint32_t conTokenId,
+                            SColumnIndex* pIndex) {
+  int32_t ret = TSDB_CODE_SUCCESS;
+
+  if(!tSqlExprIsLeaf(pExpr) || pExpr->tokenId == TK_ARROW) {
+    ret = addAllColumn(pCmd, pQueryInfo, pExpr->pLeft, conTokenId, pIndex);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+    return addAllColumn(pCmd, pQueryInfo, pExpr->pRight, conTokenId, pIndex);
+  } else if (pExpr->tokenId == TK_ID) {
+    if (getColumnIndexByName(&pExpr->columnName, pQueryInfo, pIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    }
+    STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex);
+    STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
+    SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
+    SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->id.uid, pSchema);
+    pColumn->info.flist.numOfFilters++;
+  }
+  return ret;
+}
+
 static int32_t getColQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) {
   int32_t ret = TSDB_CODE_SUCCESS;
+  const char* msg6 = "illegal condition expression";
   
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
     tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i);
@@ -4339,7 +4369,18 @@ static int32_t getColQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlEx
 
     SArray* colList = taosArrayInit(10, sizeof(SColIndex));  
     ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL);
+    size_t colNum = taosArrayGetSize(colList);
+    for (int32_t k = 0; k < colNum; k++) {
+      SColIndex* pColIndex = taosArrayGet(colList, k);
+      if (TSDB_COL_IS_TAG(pColIndex->flag)) {
+        ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
+        break;
+      }
+    }
     taosArrayDestroy(&colList);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = exprTreeValidateTree(tscGetErrorMsgPayload(pCmd), p);
+    }
 
     SBufferWriter bw = tbufInitWriter(NULL, false);
 
@@ -4387,7 +4428,7 @@ static int32_t checkColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
   }
   pQueryInfo->onlyHasTagCond &= false;
 
-  if (!tSqlExprIsParentOfLeaf(pExpr)) {  // internal node
+  if (isLogicalOperator(pExpr)) {  // internal node
     int32_t ret = checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
@@ -4396,12 +4437,10 @@ static int32_t checkColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
     return checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId);
   } else {  // handle leaf node
     SColumnIndex index = COLUMN_INDEX_INITIALIZER;
-    if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_TSC_INVALID_OPERATION;
-    }
-
+    addAllColumn(pCmd, pQueryInfo, pExpr, pExpr->tokenId, &index);
     return checkColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr);
   }
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) {
@@ -4678,19 +4717,23 @@ static int32_t validateSQLExprItemOperatorExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, S
   if (ret != TSDB_CODE_SUCCESS) {
     return ret;
   }
+
   int32_t rightHeight = 0;
-  ret = validateSQLExprItem(pCmd, pExpr->pRight, pQueryInfo, pList, &rightType, &uidRight, &rightHeight);
-  if (ret != TSDB_CODE_SUCCESS) {
-    return ret;
-  }
+  if (pExpr->tokenId != TK_BITNOT) {
+    ret = validateSQLExprItem(pCmd, pExpr->pRight, pQueryInfo, pList, &rightType, &uidRight, &rightHeight);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
 
-  if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) {
-    return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) {
+      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    }
   }
+
   *uid = uidLeft;
 
-  *height = (leftHeight > rightHeight) ? leftHeight + 1 : rightHeight+1;
-  {
+  *height = (leftHeight > rightHeight) ? leftHeight + 1 : rightHeight + 1;
+  if (pExpr->tokenId != TK_BITNOT) {
     if (leftType == SQLEXPR_TYPE_UNASSIGNED || rightType == SQLEXPR_TYPE_UNASSIGNED) {
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "invalid operand expression");
     }
@@ -4727,7 +4770,10 @@ static int32_t validateSQLExprItemOperatorExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, S
                pExpr->tokenId == TK_CONTAINS || pExpr->tokenId == TK_IN) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "unsupported filtering operations");
     }
+  } else {
+    *type = SQLEXPR_TYPE_SCALAR;
   }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -5137,6 +5183,38 @@ void convertWhereStringCharset(tSqlExpr* pRight){
   free(newData);
 }
 
+static int32_t handleColumnInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SColumnIndex* index) {
+  const char* msg2 = "illegal column name";
+  int32_t ret = TSDB_CODE_SUCCESS;
+  if (pExpr == NULL) {
+    return ret;
+  } else if (!tSqlExprIsLeaf(pExpr)) {
+    if (isComparisonOperator(pExpr)) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    }
+    ret = handleColumnInQueryCond(pCmd, pQueryInfo, pExpr->pLeft, index);
+    if( ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+    ret = handleColumnInQueryCond(pCmd, pQueryInfo, pExpr->pRight, index);
+    return ret;
+  }
+
+  SStrToken* colName = NULL;
+  if (pExpr->tokenId == TK_ARROW) {
+    colName = &(pExpr->pLeft->columnName);
+  } else if (pExpr->tokenId == TK_ID) {
+    colName = &(pExpr->columnName);
+  }
+  
+  if (colName) {
+    if (getColumnIndexByName(colName, pQueryInfo, index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
+      ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+    }
+  }
+  return ret;
+}
+
 static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr,
                                      int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr,
                                      tSqlExpr** tsExpr, bool joinQuery) {
@@ -5144,7 +5222,6 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   const char* msg2 = "illegal column name";
   const char* msg4 = "too many join tables";
   const char* msg5 = "not support ordinary column join";
-  const char* msg6 = "illegal condition expression";
 
   tSqlExpr* pLeft  = (*pExpr)->pLeft;
   tSqlExpr* pRight = (*pExpr)->pRight;
@@ -5158,14 +5235,23 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   int32_t ret = TSDB_CODE_SUCCESS;
 
   SColumnIndex index = COLUMN_INDEX_INITIALIZER;
-  if (getColumnIndexByName(colName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
-    return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+  if (!tSqlExprIsParentOfLeaf(*pExpr)) {
+    ret = handleColumnInQueryCond(pCmd, pQueryInfo, pLeft, &index);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+    ret = handleColumnInQueryCond(pCmd, pQueryInfo, pRight, &index);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+  } else {
+    if (getColumnIndexByName(colName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
+      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+    }
   }
 
   *tbIdx = index.tableIndex;
 
-  assert(tSqlExprIsParentOfLeaf(*pExpr));
-
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   STableMeta*     pTableMeta = pTableMetaInfo->pTableMeta;
 
@@ -5194,7 +5280,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   }
 
   if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) {  // query on time range
-    if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
+    if (!tSqlExprIsParentOfLeaf(*pExpr) || !validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
 
@@ -5316,8 +5402,6 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
     if (pRight->tokenId == TK_ID) {
       if (joinQuery) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); // other column cannot be served as the join column
-      } else {
-        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
       }
     }
 
@@ -5368,7 +5452,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr
   int32_t leftTbIdx = 0;
   int32_t rightTbIdx = 0;
 
-  if (!tSqlExprIsParentOfLeaf(*pExpr)) {
+  if (isLogicalOperator(*pExpr)) {
     ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, type ? &leftType : NULL, &leftTbIdx, (*pExpr)->tokenId, &columnLeft, &tsLeft, joinQuery);
     if (ret != TSDB_CODE_SUCCESS) {
       goto err_ret;
@@ -5429,6 +5513,11 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr
     goto err_ret;
   }
 
+  if (!tSqlExprIsLeaf((*pExpr)->pRight) ) {
+    ret = TSDB_CODE_TSC_INVALID_OPERATION;
+    goto err_ret;
+  }
+
   ret = handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, tbIdx, parentOptr, columnExpr, tsExpr, joinQuery);
   if (ret) {
     goto err_ret;
@@ -5451,19 +5540,21 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo*
     return;
   }
   
-  if (tSqlExprIsParentOfLeaf(*pExpr)) {
+  if (!isLogicalOperator(*pExpr)) {
     tSqlExpr* pLeft = (*pExpr)->pLeft;
 
-    SColumnIndex index = COLUMN_INDEX_INITIALIZER;
-    if(pLeft->tokenId == TK_ARROW) {
-      pLeft = pLeft->pLeft;
-    }
-    if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
-      return;
-    }
+    if (pLeft->tokenId == TK_ARROW || pLeft->tokenId == TK_ID) {
+      SColumnIndex index = COLUMN_INDEX_INITIALIZER;
+      if(pLeft->tokenId == TK_ARROW) {
+        pLeft = pLeft->pLeft;
+      }
+      if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
+        return;
+      }
 
-    if (index.tableIndex != tableIndex) {
-      return;
+      if (index.tableIndex != tableIndex) {
+        return;
+      }
     }
 
     *pOut = *pExpr;
@@ -5686,7 +5777,6 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInf
 */
 
 static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
-  const char *msg1 = "invalid tag operator";
   const char* msg2 = "not supported filter condition";
   
   do {
@@ -5706,10 +5796,6 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
     if (p->_node.pRight && (retVal = validateTagCondExpr(pCmd, p->_node.pRight)) != TSDB_CODE_SUCCESS) {
       return retVal;
     }
-
-    if (IS_ARITHMETIC_OPTR(p->_node.optr)) {
-      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
-    }
     
     if (!IS_RELATION_OPTR(p->_node.optr)) {
       break;
@@ -5771,8 +5857,13 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
   return TSDB_CODE_SUCCESS;
 }
 
+bool isTablename(char* colName) {
+   return (strlen(colName) == strlen(TSQL_TBNAME_L) && strncasecmp(TSQL_TBNAME_L, colName, strlen(TSQL_TBNAME_L)) == 0);
+}
+
 static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) {
   int32_t ret = TSDB_CODE_SUCCESS;
+  const char* msg6 = "illegal condition expression";
 
   if (pCondExpr->pTagCond == NULL) {
     return ret;
@@ -5788,6 +5879,17 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
   
     SArray* colList = taosArrayInit(10, sizeof(SColIndex));
     ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL);
+    size_t colNum = taosArrayGetSize(colList);
+    for (int32_t k = 0; k < colNum; k++) {
+      SColIndex* pColIndex = taosArrayGet(colList, k);
+      if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag) && !isTablename(pColIndex->name)) {
+        ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
+        break;
+      }
+    }
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = exprTreeValidateTree(tscGetErrorMsgPayload(pCmd), p);
+    }
     //if (ret == TSDB_CODE_SUCCESS) {
     //  ret = filterInitFromTree(p, &pQueryInfo->tagFilter, (int32_t)taosArrayGetSize(colList), NULL);
     //}
@@ -10297,23 +10399,6 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
       }
     }
 
-    if (pSqlExpr->tokenId == TK_BITAND && pSqlExpr->pLeft != NULL && pSqlExpr->pRight != NULL) {
-      // for example: col type is "bool" but expr "col & 1" received
-      uint8_t colType = pLeft->pSchema->type;
-      SStrToken *exprToken = &pSqlExpr->pRight->exprToken;
-      if (pSqlExpr->pLeft->type == SQL_NODE_TABLE_COLUMN && pSqlExpr->pRight->type == SQL_NODE_VALUE) {
-        if (colType == TSDB_DATA_TYPE_BOOL) {
-          if ((exprToken->n != 4 || strncasecmp(exprToken->z, "true", 4)) && (exprToken->n != 5 || strncasecmp(exprToken->z, "false", 5))) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-        } else if (IS_SIGNED_NUMERIC_TYPE(colType) || IS_UNSIGNED_NUMERIC_TYPE(colType)) {
-          if ((exprToken->n == 4 && strncasecmp(exprToken->z, "true", 4) == 0) || (exprToken->n == 5 || strncasecmp(exprToken->z, "false", 5) == 0)) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-        }
-      }
-    }
-
     if (pSqlExpr->pRight != NULL) {
       int32_t ret = exprTreeFromSqlExpr(pCmd, &pRight, pSqlExpr->pRight, pQueryInfo, pCols, uid);
       if (ret != TSDB_CODE_SUCCESS) {
@@ -10352,8 +10437,11 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
           pLeft = pLeft->_node.pLeft;
         }
         if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && pLeft->nodeType == TSQL_NODE_COL) {
-          if (((*pExpr)->_node.optr != TSDB_BINARY_OP_BITAND && pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) ||
-              pLeft->pSchema->type == TSDB_DATA_TYPE_JSON) {
+          if ((((*pExpr)->_node.optr != TSDB_BINARY_OP_BITAND || (*pExpr)->_node.optr != TSDB_BINARY_OP_BITOR ||
+                (*pExpr)->_node.optr != TSDB_BINARY_OP_BITXOR || (*pExpr)->_node.optr != TSDB_BINARY_OP_BITNOT ||
+                (*pExpr)->_node.optr != TSDB_BINARY_OP_LSHIFT || (*pExpr)->_node.optr != TSDB_BINARY_OP_RSHIFT) &&
+              pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) || pLeft->pSchema->type == TSDB_DATA_TYPE_JSON)
+          {
             return TSDB_CODE_TSC_INVALID_OPERATION;
           }
         }
@@ -10462,3 +10550,35 @@ void normalizeSqlNode(SSqlNode* pSqlNode, const char* dbName) {
 }
 
 #endif
+
+bool isLogicalOperator(tSqlExpr* pExpr) {
+  if (pExpr->tokenId == TK_AND || pExpr->tokenId == TK_OR) {
+    return true;
+  }
+  return false;
+}
+
+bool isComparisonOperator(tSqlExpr* pExpr) {
+ switch (pExpr->tokenId) {
+    case TK_EQ:
+    case TK_NE:
+    case TK_ISNULL:
+    case TK_NOTNULL:
+    case  TK_IS:
+    case  TK_LIKE:
+    case  TK_MATCH:
+    case  TK_NMATCH:
+    case  TK_CONTAINS:
+    case  TK_GLOB:
+    case  TK_BETWEEN:
+    case  TK_IN:
+    case  TK_GT:
+    case  TK_GE:
+    case  TK_LT:
+    case  TK_LE:
+      return true;
+    default:
+      return false;
+  }
+  return false;
+}

src/client/src/tscUtil.c

@@ -1004,7 +1004,7 @@ static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, void* pFilterI
     filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar);
     int8_t* p = NULL;
     //bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p);
-    bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, 0);
+    bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, (int16_t)taosArrayGetSize(pBlock->pDataBlock));
     if (gotNchar) {
       filterFreeNcharColumns(pFilterInfo);
     }

src/common/src/texpr.c

@@ -86,9 +86,13 @@ int32_t exprTreeValidateFunctionNode(char* msgbuf, tExprNode *pExpr) {
 }
 
 int32_t exprTreeValidateExprNode(tExprNode *pExpr) {
-  int16_t leftType = pExpr->_node.pLeft->resultType;
-  int16_t rightType = pExpr->_node.pRight->resultType;
-  int16_t resultType = leftType;
+  tExprNode *pLeft = pExpr->_node.pLeft;
+  tExprNode *pRight = pExpr->_node.pRight;
+
+  int16_t leftType = pLeft->resultType;
+  int16_t rightType = pRight->resultType;
+
+  int32_t leftTreeChecked = 0, rightTreeChecked = 0;
 
   if (pExpr->_node.optr == TSDB_BINARY_OP_ADD || pExpr->_node.optr == TSDB_BINARY_OP_SUBTRACT ||
       pExpr->_node.optr == TSDB_BINARY_OP_MULTIPLY || pExpr->_node.optr == TSDB_BINARY_OP_DIVIDE ||
@@ -99,171 +103,177 @@ int32_t exprTreeValidateExprNode(tExprNode *pExpr) {
     pExpr->resultType = TSDB_DATA_TYPE_DOUBLE;
     pExpr->resultBytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
     return TSDB_CODE_SUCCESS;
-  } else if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND) {
-    if ((leftType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(leftType) && !IS_UNSIGNED_NUMERIC_TYPE(leftType)) ||
-        (rightType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(rightType) && !IS_UNSIGNED_NUMERIC_TYPE(rightType)))
-    {
-      return TSDB_CODE_TSC_INVALID_OPERATION;
-    }
+  } else if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND || pExpr->_node.optr == TSDB_BINARY_OP_BITOR ||
+             pExpr->_node.optr == TSDB_BINARY_OP_BITXOR || pExpr->_node.optr == TSDB_BINARY_OP_LSHIFT ||
+             pExpr->_node.optr == TSDB_BINARY_OP_RSHIFT)
+  {
+    if (!IS_NUMERIC_TYPE(leftType) || !IS_NUMERIC_TYPE(rightType)) {
+      if (pLeft->_node.pLeft) {
+        if (!IS_NUMERIC_TYPE(pLeft->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
 
-    uint8_t schemaType;
-    // now leftType and rightType are both numeric
-    if (pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_COL) {
-      if (leftType != rightType) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
+        if (IS_FLOAT_TYPE(pLeft->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        leftTreeChecked++;
       }
-    } else if (pExpr->_node.pLeft->nodeType == TSQL_NODE_COL) {
-      if (pExpr->_node.pRight->nodeType != TSQL_NODE_VALUE) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
-      } else {
-        schemaType = pExpr->_node.pLeft->pSchema->type;
-        int64_t sVal = pExpr->_node.pRight->pVal->i64;
-        uint64_t uVal = pExpr->_node.pRight->pVal->u64;
-
-        switch (schemaType) {
-        case TSDB_DATA_TYPE_BOOL:
-          if ((pExpr->_node.pRight->pVal->nType != TSDB_DATA_TYPE_BOOL) ||
-              (pExpr->_node.pRight->pVal->i64 != 0 &&
-               pExpr->_node.pRight->pVal->i64 != 1 &&
-               pExpr->_node.pRight->pVal->i64 != TSDB_DATA_BOOL_NULL))
-          {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_TINYINT:
-          if (sVal < -128 || sVal > 127) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_SMALLINT:
-          if (sVal < -32768 || sVal > 32767) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_INT:
-          if (sVal < INT32_MIN || sVal > INT32_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_BIGINT:
-          if (sVal < INT64_MIN || sVal > INT64_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_UTINYINT:
-          if (uVal > 255) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_USMALLINT:
-          if (uVal > 65535) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_UINT:
-          if (uVal > UINT32_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_UBIGINT:
-          if (uVal > UINT64_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
+
+      if (pLeft->_node.pRight) {
+        if (!IS_NUMERIC_TYPE(pLeft->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
         }
 
-        pExpr->_node.pRight->pSchema->type = schemaType;
-        pExpr->_node.pRight->pVal->nType = schemaType;
+        if (IS_FLOAT_TYPE(pLeft->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
 
-        pExpr->_node.pRight->resultType = schemaType;
-        pExpr->_node.pRight->resultBytes = tDataTypes[schemaType].bytes;
+        leftTreeChecked++;
       }
-    } else {
-      if (pExpr->_node.pLeft->nodeType != TSQL_NODE_VALUE) {
+
+      if (pRight->_node.pLeft) {
+        if (!IS_NUMERIC_TYPE(pRight->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        if (IS_FLOAT_TYPE(pRight->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        rightTreeChecked++;
+      }
+
+      if (pRight->_node.pRight) {
+        if (!IS_NUMERIC_TYPE(pRight->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        if (IS_FLOAT_TYPE(pRight->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        rightTreeChecked++;
+      }
+
+      if ((!IS_NUMERIC_TYPE(leftType) && leftTreeChecked == 0) || (!IS_NUMERIC_TYPE(rightType) && rightTreeChecked == 0)) {
         return TSDB_CODE_TSC_INVALID_OPERATION;
-      } else {
-        schemaType = pExpr->_node.pRight->pSchema->type;
-        int64_t sVal = pExpr->_node.pLeft->pVal->i64;
-        uint64_t uVal = pExpr->_node.pLeft->pVal->u64;
-        switch (schemaType) {
-        case TSDB_DATA_TYPE_BOOL:
-          if ((pExpr->_node.pLeft->pVal->nType != TSDB_DATA_TYPE_BOOL) ||
-              (pExpr->_node.pLeft->pVal->i64 != 0 &&
-               pExpr->_node.pLeft->pVal->i64 != 1 &&
-               pExpr->_node.pLeft->pVal->i64 != TSDB_DATA_BOOL_NULL))
-          {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 1;
-          break;
-        case TSDB_DATA_TYPE_TINYINT:
-          if (sVal < -128 || sVal > 127) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 1;
-          break;
-        case TSDB_DATA_TYPE_SMALLINT:
-          if (sVal < -32768 || sVal > 32767) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 2;
-          break;
-        case TSDB_DATA_TYPE_INT:
-          if (sVal < INT32_MIN || sVal > INT32_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 4;
-          break;
-        case TSDB_DATA_TYPE_BIGINT:
-          if (sVal < INT64_MIN || sVal > INT64_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 8;
-          break;
-        case TSDB_DATA_TYPE_UTINYINT:
-          if (uVal > 255) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 1;
-          break;
-        case TSDB_DATA_TYPE_USMALLINT:
-          if (uVal > 65535) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 2;
-          break;
-        case TSDB_DATA_TYPE_UINT:
-          if (uVal > UINT32_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 4;
-          break;
-        case TSDB_DATA_TYPE_UBIGINT:
-          if (uVal > UINT64_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 8;
-          break;
+      }
+
+      leftTreeChecked = 0;
+      rightTreeChecked = 0;
+    }
+    if (IS_FLOAT_TYPE(leftType) || IS_FLOAT_TYPE(rightType)) {
+      if (pLeft->_node.pLeft) {
+        if (!IS_NUMERIC_TYPE(pLeft->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        if (IS_FLOAT_TYPE(pLeft->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        leftTreeChecked++;
+      }
+
+      if (pLeft->_node.pRight) {
+        if (!IS_NUMERIC_TYPE(pLeft->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        if (IS_FLOAT_TYPE(pLeft->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        leftTreeChecked++;
+      }
+
+      if (pRight->_node.pLeft) {
+        if (!IS_NUMERIC_TYPE(pRight->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        if (IS_FLOAT_TYPE(pRight->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        rightTreeChecked++;
+      }
+
+      if (pRight->_node.pRight) {
+        if (!IS_NUMERIC_TYPE(pRight->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
         }
 
-        pExpr->_node.pLeft->pSchema->type = schemaType;
-        pExpr->_node.pLeft->pVal->nType = schemaType;
+        if (IS_FLOAT_TYPE(pRight->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
 
-        pExpr->_node.pLeft->resultType = schemaType;
-        pExpr->_node.pLeft->resultBytes = tDataTypes[schemaType].bytes;
+        rightTreeChecked++;
       }
 
-      resultType = schemaType;
+      if ((IS_FLOAT_TYPE(leftType) && leftTreeChecked == 0) || (IS_FLOAT_TYPE(rightType) && rightTreeChecked == 0)) {
+        return TSDB_CODE_TSC_INVALID_OPERATION;
+      }
+    }
+
+    // colx logic_op n bitwise_op coly
+    if (pLeft->resultType == TSDB_DATA_TYPE_DOUBLE) {
+      pLeft->resultType = TSDB_DATA_TYPE_BIGINT;
+      pLeft->resultBytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
     }
 
-    if (resultType == TSDB_DATA_TYPE_BOOL) {
-      pExpr->resultType = TSDB_DATA_TYPE_BOOL;
-      pExpr->resultBytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
+    // colx bitwise_op coly logic_op n
+    if (pRight->resultType == TSDB_DATA_TYPE_DOUBLE) {
+      pRight->resultType = TSDB_DATA_TYPE_BIGINT;
+      pRight->resultBytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
+    }
+
+    if (pExpr->_node.optr == TSDB_BINARY_OP_LSHIFT || pExpr->_node.optr == TSDB_BINARY_OP_RSHIFT) {
+      pExpr->resultType = pLeft->resultType;
+      pExpr->resultBytes = tDataTypes[pLeft->resultType].bytes;
     } else {
-      pExpr->resultType = resultType;
-      pExpr->resultBytes = tDataTypes[resultType].bytes;
+      pExpr->resultType = TSDB_DATA_TYPE_BIGINT;
+      pExpr->resultBytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
     }
     return TSDB_CODE_SUCCESS;
+  } else if (pExpr->_node.optr == TSDB_BINARY_OP_BITNOT) {
+    if (!IS_NUMERIC_TYPE(leftType) || IS_FLOAT_TYPE(leftType)) {
+      if (pLeft->_node.pLeft) {
+        if (!IS_NUMERIC_TYPE(pLeft->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        if (IS_FLOAT_TYPE(pLeft->_node.pLeft->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        leftTreeChecked++;
+      }
+
+      if (pLeft->_node.pRight) {
+        if (!IS_NUMERIC_TYPE(pLeft->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        if (IS_FLOAT_TYPE(pLeft->_node.pRight->resultType)) {
+          return TSDB_CODE_TSC_INVALID_OPERATION;
+        }
+
+        leftTreeChecked++;
+      }
+
+      if (leftTreeChecked == 0) {
+        return TSDB_CODE_TSC_INVALID_OPERATION;
+      }
+    }
+    if (leftType == TSDB_DATA_TYPE_DOUBLE) {
+      pExpr->resultType = TSDB_DATA_TYPE_BIGINT;
+    } else {
+      pExpr->resultType = leftType;
+    }
+    pExpr->resultBytes = tDataTypes[pExpr->resultType].bytes;
+    return TSDB_CODE_SUCCESS;
   } else {
     return TSDB_CODE_SUCCESS;
   }
@@ -591,8 +601,8 @@ void exprTreeExprNodeTraverse(tExprNode *pExpr, int32_t numOfRows, tExprOperandI
 
   tExprNode *pLeft = pExpr->_node.pLeft;
   tExprNode *pRight = pExpr->_node.pRight;
-  char *ltmp = NULL, *rtmp = NULL;
-  char *leftIn = NULL, *rightIn = NULL;
+  char *ltmp = NULL, *rtmp = NULL, *pl = NULL, *pr = NULL, *pt = NULL;
+  char *leftIn = NULL, *rightIn = NULL, *transl = NULL, *transr = NULL;
   int32_t leftNum = 0, rightNum = 0;
   int32_t leftType = 0, rightType = 0;
   int32_t fnOrder = TSDB_ORDER_ASC;
@@ -602,9 +612,78 @@ void exprTreeExprNodeTraverse(tExprNode *pExpr, int32_t numOfRows, tExprOperandI
     tExprOperandInfo left;
     left.data = ltmp;
     exprTreeInternalNodeTraverse(pLeft, numOfRows, &left, param, order, getSourceDataBlock);
-    
+
     leftIn = ltmp;
     leftType = left.type;
+
+    if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND || pExpr->_node.optr == TSDB_BINARY_OP_BITOR ||
+        pExpr->_node.optr == TSDB_BINARY_OP_BITXOR || pExpr->_node.optr == TSDB_BINARY_OP_BITNOT ||
+        pExpr->_node.optr == TSDB_BINARY_OP_LSHIFT || pExpr->_node.optr == TSDB_BINARY_OP_RSHIFT)
+    {
+      transl = (char *)malloc(sizeof(int64_t) * left.numOfRows);
+      if (transl == NULL) {
+        return;
+      }
+
+      pl = ltmp;
+      pt = transl;
+
+      switch (left.type) {
+      case TSDB_DATA_TYPE_TINYINT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((int8_t *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_SMALLINT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((int16_t *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_INT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((int32_t *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_BIGINT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((int64_t *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_UTINYINT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((uint8_t *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_USMALLINT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((uint16_t *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_UINT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((uint32_t *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_UBIGINT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((uint64_t *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_FLOAT:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((float *) pl + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_DOUBLE:
+        for (int16_t i = 0; i < left.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((double *) pl + i));
+        }
+        break;
+      }
+
+      leftIn = transl;
+      leftType = TSDB_DATA_TYPE_BIGINT;
+    }
     leftNum = left.numOfRows;
   } else if (pLeft->nodeType == TSQL_NODE_COL) {
     char *pInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId);
@@ -631,9 +710,78 @@ void exprTreeExprNodeTraverse(tExprNode *pExpr, int32_t numOfRows, tExprOperandI
     tExprOperandInfo right;
     right.data = rtmp;
     exprTreeInternalNodeTraverse(pRight, numOfRows, &right, param, order, getSourceDataBlock);
-    
+
     rightIn = rtmp;
     rightType = right.type;
+
+    if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND || pExpr->_node.optr == TSDB_BINARY_OP_BITOR ||
+        pExpr->_node.optr == TSDB_BINARY_OP_BITXOR || pExpr->_node.optr == TSDB_BINARY_OP_BITNOT ||
+        pExpr->_node.optr == TSDB_BINARY_OP_LSHIFT || pExpr->_node.optr == TSDB_BINARY_OP_RSHIFT)
+    {
+      transr = (char *)malloc(sizeof(int64_t) * right.numOfRows);
+      if (transr == NULL) {
+        return;
+      }
+
+      pr = rtmp;
+      pt = transr;
+
+      switch (right.type) {
+      case TSDB_DATA_TYPE_TINYINT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((int8_t *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_SMALLINT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((int16_t *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_INT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((int32_t *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_BIGINT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((int64_t *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_UTINYINT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((uint8_t *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_USMALLINT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((uint16_t *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_UINT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((uint32_t *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_UBIGINT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((uint64_t *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_FLOAT:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((float *) pr + i));
+        }
+        break;
+      case TSDB_DATA_TYPE_DOUBLE:
+        for (int16_t i = 0; i < right.numOfRows; i++) {
+          *((int64_t *) pt + i) = (int64_t)(*((double *) pr + i));
+        }
+        break;
+      }
+
+      rightIn = transr;
+      rightType = TSDB_DATA_TYPE_BIGINT;
+    }
     rightNum = right.numOfRows;
   } else if (pRight->nodeType == TSQL_NODE_COL) {
     char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
@@ -648,6 +796,9 @@ void exprTreeExprNodeTraverse(tExprNode *pExpr, int32_t numOfRows, tExprOperandI
 
     rightType = pRight->pSchema->type;
     rightNum = numOfRows;    
+  } else if (pRight->nodeType == TSQL_NODE_DUMMY) {
+    /* BITNOT */
+    rightNum = 0;
   } else {
     assert(pRight->nodeType == TSQL_NODE_VALUE);
     rightIn = (char *)&pRight->pVal->i64;
@@ -662,7 +813,10 @@ void exprTreeExprNodeTraverse(tExprNode *pExpr, int32_t numOfRows, tExprOperandI
   if(leftType == TSDB_DATA_TYPE_TIMESTAMP || rightType == TSDB_DATA_TYPE_TIMESTAMP) {
     output->type = TSDB_DATA_TYPE_BIGINT;
   } else {
-    if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND) {
+    if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND || pExpr->_node.optr == TSDB_BINARY_OP_BITOR ||
+        pExpr->_node.optr == TSDB_BINARY_OP_BITXOR || pExpr->_node.optr == TSDB_BINARY_OP_BITNOT ||
+        pExpr->_node.optr == TSDB_BINARY_OP_LSHIFT || pExpr->_node.optr == TSDB_BINARY_OP_RSHIFT)
+    {
       output->type = leftType; // rightType must be the same as leftType
     } else {
       output->type = TSDB_DATA_TYPE_DOUBLE;
@@ -670,6 +824,9 @@ void exprTreeExprNodeTraverse(tExprNode *pExpr, int32_t numOfRows, tExprOperandI
   }
   output->bytes = tDataTypes[output->type].bytes;
 
+  if (transl) tfree(transl);
+  if (transr) tfree(transr);
+
   tfree(ltmp);
   tfree(rtmp);
 }
@@ -1276,6 +1433,7 @@ int32_t exprValidateMathNode(tExprNode *pExpr) {
 
       return TSDB_CODE_SUCCESS;
     }
+
     case TSDB_FUNC_SCALAR_ABS: {
       if (pExpr->_func.numChildren != 1) {
         return TSDB_CODE_TSC_INVALID_OPERATION;

src/inc/taosdef.h

@@ -177,10 +177,15 @@ do { \
 #define TSDB_BINARY_OP_DIVIDE     33
 #define TSDB_BINARY_OP_REMAINDER  34
 #define TSDB_BINARY_OP_BITAND     35
+#define TSDB_BINARY_OP_BITOR      36
+#define TSDB_BINARY_OP_BITXOR     37
+#define TSDB_BINARY_OP_BITNOT     38
+#define TSDB_BINARY_OP_LSHIFT     39
+#define TSDB_BINARY_OP_RSHIFT     40
 
 
 #define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
-#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_BITAND))
+#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_RSHIFT))
 
 #define TS_PATH_DELIMITER_LEN     1
 

src/inc/ttokendef.h

@@ -50,172 +50,173 @@
 #define TK_LE                              32
 #define TK_BITAND                          33
 #define TK_BITOR                           34
-#define TK_LSHIFT                          35
-#define TK_RSHIFT                          36
-#define TK_PLUS                            37
-#define TK_MINUS                           38
-#define TK_DIVIDE                          39
-#define TK_TIMES                           40
-#define TK_STAR                            41
-#define TK_SLASH                           42
-#define TK_REM                             43
-#define TK_UMINUS                          44
-#define TK_UPLUS                           45
-#define TK_BITNOT                          46
-#define TK_ARROW                           47
-#define TK_SHOW                            48
-#define TK_DATABASES                       49
-#define TK_TOPICS                          50
-#define TK_FUNCTIONS                       51
-#define TK_MNODES                          52
-#define TK_DNODES                          53
-#define TK_ACCOUNTS                        54
-#define TK_USERS                           55
-#define TK_MODULES                         56
-#define TK_QUERIES                         57
-#define TK_CONNECTIONS                     58
-#define TK_STREAMS                         59
-#define TK_VARIABLES                       60
-#define TK_SCORES                          61
-#define TK_GRANTS                          62
-#define TK_VNODES                          63
-#define TK_DOT                             64
-#define TK_CREATE                          65
-#define TK_TABLE                           66
-#define TK_STABLE                          67
-#define TK_DATABASE                        68
-#define TK_TABLES                          69
-#define TK_STABLES                         70
-#define TK_VGROUPS                         71
-#define TK_DROP                            72
-#define TK_TOPIC                           73
-#define TK_FUNCTION                        74
-#define TK_DNODE                           75
-#define TK_USER                            76
-#define TK_ACCOUNT                         77
-#define TK_USE                             78
-#define TK_DESCRIBE                        79
-#define TK_DESC                            80
-#define TK_ALTER                           81
-#define TK_PASS                            82
-#define TK_PRIVILEGE                       83
-#define TK_LOCAL                           84
-#define TK_COMPACT                         85
-#define TK_LP                              86
-#define TK_RP                              87
-#define TK_IF                              88
-#define TK_EXISTS                          89
-#define TK_AS                              90
-#define TK_OUTPUTTYPE                      91
-#define TK_AGGREGATE                       92
-#define TK_BUFSIZE                         93
-#define TK_PPS                             94
-#define TK_TSERIES                         95
-#define TK_DBS                             96
-#define TK_STORAGE                         97
-#define TK_QTIME                           98
-#define TK_CONNS                           99
-#define TK_STATE                          100
-#define TK_COMMA                          101
-#define TK_KEEP                           102
-#define TK_CACHE                          103
-#define TK_REPLICA                        104
-#define TK_QUORUM                         105
-#define TK_DAYS                           106
-#define TK_MINROWS                        107
-#define TK_MAXROWS                        108
-#define TK_BLOCKS                         109
-#define TK_CTIME                          110
-#define TK_WAL                            111
-#define TK_FSYNC                          112
-#define TK_COMP                           113
-#define TK_PRECISION                      114
-#define TK_UPDATE                         115
-#define TK_CACHELAST                      116
-#define TK_PARTITIONS                     117
-#define TK_UNSIGNED                       118
-#define TK_TAGS                           119
-#define TK_USING                          120
-#define TK_NULL                           121
-#define TK_NOW                            122
-#define TK_VARIABLE                       123
-#define TK_SELECT                         124
-#define TK_UNION                          125
-#define TK_ALL                            126
-#define TK_DISTINCT                       127
-#define TK_FROM                           128
-#define TK_RANGE                          129
-#define TK_INTERVAL                       130
-#define TK_EVERY                          131
-#define TK_SESSION                        132
-#define TK_STATE_WINDOW                   133
-#define TK_FILL                           134
-#define TK_SLIDING                        135
-#define TK_ORDER                          136
-#define TK_BY                             137
-#define TK_ASC                            138
-#define TK_GROUP                          139
-#define TK_HAVING                         140
-#define TK_LIMIT                          141
-#define TK_OFFSET                         142
-#define TK_SLIMIT                         143
-#define TK_SOFFSET                        144
-#define TK_WHERE                          145
-#define TK_RESET                          146
-#define TK_QUERY                          147
-#define TK_SYNCDB                         148
-#define TK_ADD                            149
-#define TK_COLUMN                         150
-#define TK_MODIFY                         151
-#define TK_TAG                            152
-#define TK_CHANGE                         153
-#define TK_SET                            154
-#define TK_KILL                           155
-#define TK_CONNECTION                     156
-#define TK_STREAM                         157
-#define TK_COLON                          158
-#define TK_ABORT                          159
-#define TK_AFTER                          160
-#define TK_ATTACH                         161
-#define TK_BEFORE                         162
-#define TK_BEGIN                          163
-#define TK_CASCADE                        164
-#define TK_CLUSTER                        165
-#define TK_CONFLICT                       166
-#define TK_COPY                           167
-#define TK_DEFERRED                       168
-#define TK_DELIMITERS                     169
-#define TK_DETACH                         170
-#define TK_EACH                           171
-#define TK_END                            172
-#define TK_EXPLAIN                        173
-#define TK_FAIL                           174
-#define TK_FOR                            175
-#define TK_IGNORE                         176
-#define TK_IMMEDIATE                      177
-#define TK_INITIALLY                      178
-#define TK_INSTEAD                        179
-#define TK_KEY                            180
-#define TK_OF                             181
-#define TK_RAISE                          182
-#define TK_REPLACE                        183
-#define TK_RESTRICT                       184
-#define TK_ROW                            185
-#define TK_STATEMENT                      186
-#define TK_TRIGGER                        187
-#define TK_VIEW                           188
-#define TK_IPTOKEN                        189
-#define TK_SEMI                           190
-#define TK_NONE                           191
-#define TK_PREV                           192
-#define TK_LINEAR                         193
-#define TK_IMPORT                         194
-#define TK_TBNAME                         195
-#define TK_JOIN                           196
-#define TK_INSERT                         197
-#define TK_INTO                           198
-#define TK_VALUES                         199
-#define TK_FILE                           200
+#define TK_BITXOR                          35
+#define TK_LSHIFT                          36
+#define TK_RSHIFT                          37
+#define TK_PLUS                            38
+#define TK_MINUS                           39
+#define TK_DIVIDE                          40
+#define TK_TIMES                           41
+#define TK_STAR                            42
+#define TK_SLASH                           43
+#define TK_REM                             44
+#define TK_UMINUS                          45
+#define TK_UPLUS                           46
+#define TK_BITNOT                          47
+#define TK_ARROW                           48
+#define TK_SHOW                            49
+#define TK_DATABASES                       50
+#define TK_TOPICS                          51
+#define TK_FUNCTIONS                       52
+#define TK_MNODES                          53
+#define TK_DNODES                          54
+#define TK_ACCOUNTS                        55
+#define TK_USERS                           56
+#define TK_MODULES                         57
+#define TK_QUERIES                         58
+#define TK_CONNECTIONS                     59
+#define TK_STREAMS                         60
+#define TK_VARIABLES                       61
+#define TK_SCORES                          62
+#define TK_GRANTS                          63
+#define TK_VNODES                          64
+#define TK_DOT                             65
+#define TK_CREATE                          66
+#define TK_TABLE                           67
+#define TK_STABLE                          68
+#define TK_DATABASE                        69
+#define TK_TABLES                          70
+#define TK_STABLES                         71
+#define TK_VGROUPS                         72
+#define TK_DROP                            73
+#define TK_TOPIC                           74
+#define TK_FUNCTION                        75
+#define TK_DNODE                           76
+#define TK_USER                            77
+#define TK_ACCOUNT                         78
+#define TK_USE                             79
+#define TK_DESCRIBE                        80
+#define TK_DESC                            81
+#define TK_ALTER                           82
+#define TK_PASS                            83
+#define TK_PRIVILEGE                       84
+#define TK_LOCAL                           85
+#define TK_COMPACT                         86
+#define TK_LP                              87
+#define TK_RP                              88
+#define TK_IF                              89
+#define TK_EXISTS                          90
+#define TK_AS                              91
+#define TK_OUTPUTTYPE                      92
+#define TK_AGGREGATE                       93
+#define TK_BUFSIZE                         94
+#define TK_PPS                             95
+#define TK_TSERIES                         96
+#define TK_DBS                             97
+#define TK_STORAGE                         98
+#define TK_QTIME                           99
+#define TK_CONNS                          100
+#define TK_STATE                          101
+#define TK_COMMA                          102
+#define TK_KEEP                           103
+#define TK_CACHE                          104
+#define TK_REPLICA                        105
+#define TK_QUORUM                         106
+#define TK_DAYS                           107
+#define TK_MINROWS                        108
+#define TK_MAXROWS                        109
+#define TK_BLOCKS                         110
+#define TK_CTIME                          111
+#define TK_WAL                            112
+#define TK_FSYNC                          113
+#define TK_COMP                           114
+#define TK_PRECISION                      115
+#define TK_UPDATE                         116
+#define TK_CACHELAST                      117
+#define TK_PARTITIONS                     118
+#define TK_UNSIGNED                       119
+#define TK_TAGS                           120
+#define TK_USING                          121
+#define TK_NULL                           122
+#define TK_NOW                            123
+#define TK_VARIABLE                       124
+#define TK_SELECT                         125
+#define TK_UNION                          126
+#define TK_ALL                            127
+#define TK_DISTINCT                       128
+#define TK_FROM                           129
+#define TK_RANGE                          130
+#define TK_INTERVAL                       131
+#define TK_EVERY                          132
+#define TK_SESSION                        133
+#define TK_STATE_WINDOW                   134
+#define TK_FILL                           135
+#define TK_SLIDING                        136
+#define TK_ORDER                          137
+#define TK_BY                             138
+#define TK_ASC                            139
+#define TK_GROUP                          140
+#define TK_HAVING                         141
+#define TK_LIMIT                          142
+#define TK_OFFSET                         143
+#define TK_SLIMIT                         144
+#define TK_SOFFSET                        145
+#define TK_WHERE                          146
+#define TK_RESET                          147
+#define TK_QUERY                          148
+#define TK_SYNCDB                         149
+#define TK_ADD                            150
+#define TK_COLUMN                         151
+#define TK_MODIFY                         152
+#define TK_TAG                            153
+#define TK_CHANGE                         154
+#define TK_SET                            155
+#define TK_KILL                           156
+#define TK_CONNECTION                     157
+#define TK_STREAM                         158
+#define TK_COLON                          159
+#define TK_ABORT                          160
+#define TK_AFTER                          161
+#define TK_ATTACH                         162
+#define TK_BEFORE                         163
+#define TK_BEGIN                          164
+#define TK_CASCADE                        165
+#define TK_CLUSTER                        166
+#define TK_CONFLICT                       167
+#define TK_COPY                           168
+#define TK_DEFERRED                       169
+#define TK_DELIMITERS                     170
+#define TK_DETACH                         171
+#define TK_EACH                           172
+#define TK_END                            173
+#define TK_EXPLAIN                        174
+#define TK_FAIL                           175
+#define TK_FOR                            176
+#define TK_IGNORE                         177
+#define TK_IMMEDIATE                      178
+#define TK_INITIALLY                      179
+#define TK_INSTEAD                        180
+#define TK_KEY                            181
+#define TK_OF                             182
+#define TK_RAISE                          183
+#define TK_REPLACE                        184
+#define TK_RESTRICT                       185
+#define TK_ROW                            186
+#define TK_STATEMENT                      187
+#define TK_TRIGGER                        188
+#define TK_VIEW                           189
+#define TK_IPTOKEN                        190
+#define TK_SEMI                           191
+#define TK_NONE                           192
+#define TK_PREV                           193
+#define TK_LINEAR                         194
+#define TK_IMPORT                         195
+#define TK_TBNAME                         196
+#define TK_JOIN                           197
+#define TK_INSERT                         198
+#define TK_INTO                           199
+#define TK_VALUES                         200
+#define TK_FILE                           201
 
 
 #define TK_SPACE                          300

src/inc/ttype.h

@@ -289,6 +289,7 @@ void* getDataMax(int32_t type);
 int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned);
 
 #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL)
+#define SET_BIGINT_NULL(v) (*(int64_t *)(v) = TSDB_DATA_BIGINT_NULL)
 
 #ifdef __cplusplus
 }

src/kit/shell/src/shellEngine.c

@@ -244,22 +244,41 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
   int64_t   st, et;
   wordexp_t full_path;
   char *    sptr = NULL;
+  char *    tmp = NULL;
   char *    cptr = NULL;
   char *    fname = NULL;
   bool      printMode = false;
-
-  if ((sptr = tstrstr(command, ">>", true)) != NULL) {
-    cptr = tstrstr(command, ";", true);
-    if (cptr != NULL) {
-      *cptr = '\0';
-    }
-
-    if (wordexp(sptr + 2, &full_path, 0) != 0) {
-      fprintf(stderr, "ERROR: invalid filename: %s\n", sptr + 2);
-      return;
+  int       match;
+
+  sptr = command;
+  while ((sptr = tstrstr(sptr, ">>", true)) != NULL) {
+    // find the last ">>" if any
+    tmp = sptr;
+    sptr += 2;
+  }
+
+  sptr = tmp;
+
+  if (sptr != NULL) {
+    // select ... where col >> n op m ...;
+    match = regex_match(sptr + 2, "^\\s*.{1,}\\s*[\\>|\\<|\\<=|\\>=|=|!=]\\s*.{1,};\\s*$", REG_EXTENDED | REG_ICASE);
+    if (match == 0) {
+      // select col >> n from ...;
+      match = regex_match(sptr + 2, "^\\s*.{1,}\\s{1,}.{1,};\\s*$", REG_EXTENDED | REG_ICASE);
+      if (match == 0) {
+        cptr = tstrstr(command, ";", true);
+        if (cptr != NULL) {
+          *cptr = '\0';
+        }
+
+        if (wordexp(sptr + 2, &full_path, 0) != 0) {
+          fprintf(stderr, "ERROR: invalid filename: %s\n", sptr + 2);
+          return;
+        }
+        *sptr = '\0';
+        fname = full_path.we_wordv[0];
+      }
     }
-    *sptr = '\0';
-    fname = full_path.we_wordv[0];
   }
 
   if ((sptr = tstrstr(command, "\\G", true)) != NULL) {

src/query/inc/qFilter.h

@@ -39,10 +39,11 @@ extern "C" {
 enum {
   FLD_TYPE_COLUMN = 1,
   FLD_TYPE_VALUE = 2,  
-  FLD_TYPE_MAX = 3,
-  FLD_DESC_NO_FREE = 4,
-  FLD_DATA_NO_FREE = 8,
-  FLD_DATA_IS_HASH = 16,
+  FLD_TYPE_EXPR = 3,
+  FLD_TYPE_MAX = 4,
+  FLD_DESC_NO_FREE = 16,
+  FLD_DATA_NO_FREE = 32,
+  FLD_DATA_IS_HASH = 64,
 };
 
 enum {
@@ -182,6 +183,7 @@ typedef struct SFilterGroupCtx {
   uint32_t         colNum;
   uint32_t        *colIdx;
   SFilterColInfo  *colInfo;
+  bool             hasExpr;
 } SFilterGroupCtx;
 
 typedef struct SFilterColCtx {
@@ -206,6 +208,8 @@ typedef struct SFilterComUnit {
   void *colData;
   void *valData;
   void *valData2;
+  void *expr;
+  void *exprData;
   uint16_t colId;
   uint16_t dataSize;
   uint8_t dataType;
@@ -289,7 +293,10 @@ typedef struct SFilterInfo {
 #define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
 #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
 #define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
-#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
+#define FILTER_GET_TYPE(fl) ((fl) & 0xF)
+#define FILTER_GET_FIELD_DESC(fi) ((fi)->desc)
+#define FILTER_GET_EXPR_TYPE(i, id) (((tExprNode*)(FILTER_GET_FIELD_DESC(FILTER_GET_FIELD(i, id))))->resultType)
+#define FILTER_GET_EXPR_SIZE(i, id) (((tExprNode*)(FILTER_GET_FIELD_DESC(FILTER_GET_FIELD(i, id))))->resultBytes)
 
 #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
 #define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left)

src/query/inc/sql.y

@@ -13,7 +13,7 @@
 %right NOT.
 %left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH CONTAINS GLOB BETWEEN IN.
 %left GT GE LT LE.
-%left BITAND BITOR LSHIFT RSHIFT.
+%left BITAND BITOR BITXOR LSHIFT RSHIFT.
 %left PLUS MINUS.
 %left DIVIDE TIMES.
 %left STAR SLASH REM.
@@ -787,6 +787,11 @@ expr(A) ::= expr(X) STAR  expr(Y).   {A = tSqlExprCreate(X, Y, TK_STAR);  }
 expr(A) ::= expr(X) SLASH expr(Y).   {A = tSqlExprCreate(X, Y, TK_DIVIDE);}
 expr(A) ::= expr(X) REM   expr(Y).   {A = tSqlExprCreate(X, Y, TK_REM);   }
 expr(A) ::= expr(X) BITAND expr(Y).  {A = tSqlExprCreate(X, Y, TK_BITAND);}
+expr(A) ::= expr(X) BITOR expr(Y).   {A = tSqlExprCreate(X, Y, TK_BITOR); }
+expr(A) ::= expr(X) BITXOR expr(Y).  {A = tSqlExprCreate(X, Y, TK_BITXOR);}
+expr(A) ::= BITNOT expr(X).          {A = tSqlExprCreate(X, NULL, TK_BITNOT);}
+expr(A) ::= expr(X) LSHIFT expr(Y).  {A = tSqlExprCreate(X, Y, TK_LSHIFT);}
+expr(A) ::= expr(X) RSHIFT expr(Y).  {A = tSqlExprCreate(X, Y, TK_RSHIFT);}
 
 // like expression
 expr(A) ::= expr(X) LIKE expr(Y).    {A = tSqlExprCreate(X, Y, TK_LIKE);  }

src/query/src/qExecutor.c

@@ -3181,6 +3181,10 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi
 FORCE_INLINE int32_t getColumnDataFromId(void *param, int32_t id, void **data) {
   int32_t numOfCols = ((SColumnDataParam *)param)->numOfCols;
   SArray* pDataBlock = ((SColumnDataParam *)param)->pDataBlock;
+  if (id == INT32_MAX) {
+    *data = pDataBlock;
+    return TSDB_CODE_SUCCESS;
+  }
   
   for (int32_t j = 0; j < numOfCols; ++j) {
     SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j);

src/query/src/qFilter.c

@@ -54,11 +54,16 @@ static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const v
   return tVariantCompare(val1, val2);
 }
 
+static FORCE_INLINE int32_t filterExprCompare(const void *desc1, const void *desc2) {
+  return -1;
+}
+
 
 filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = {
   NULL,
   filterFieldColDescCompare,
-  filterFieldValDescCompare
+  filterFieldValDescCompare,
+  filterExprCompare
 };
 
 bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
@@ -833,7 +838,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type,
     info->fields[type].fields[idx].desc = desc;
     info->fields[type].fields[idx].data = data ? *data : NULL;
 
-    if (type == FLD_TYPE_COLUMN) {
+    if (type == FLD_TYPE_COLUMN || type == FLD_TYPE_EXPR) {
       FILTER_SET_FLAG(info->fields[type].fields[idx].flag, FLD_DATA_NO_FREE);
     }
 
@@ -873,7 +878,8 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte
 
 int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) {
   CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node");
-  CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR);
+  CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE
+          &&node->nodeType != TSQL_NODE_EXPR, TSDB_CODE_QRY_APP_ERROR);
 
   int32_t type;
   void *v;
@@ -882,6 +888,9 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI
     type = FLD_TYPE_COLUMN;
     v = node->pSchema;
     node->pSchema = NULL;
+  } else if (node->nodeType == TSQL_NODE_EXPR) {
+    type = FLD_TYPE_EXPR;
+    v = node;
   } else {
     type = FLD_TYPE_VALUE;
     v = node->pVal;
@@ -933,9 +942,11 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
   }
   
   SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u);
-  assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN));
-  
-  info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col);
+  if (FILTER_GET_TYPE(col->flag) == FLD_TYPE_COLUMN) {
+    info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col);
+  } else {
+    info->units[info->unitNum].compare.type = (uint8_t)FILTER_GET_EXPR_TYPE(info, u->left);
+  }
 
   *uidx = info->unitNum;
 
@@ -1188,7 +1199,7 @@ static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pL
     jsonKeyMd5((*pLeft)->_node.pRight->pVal->pz, (*pLeft)->_node.pRight->pVal->nLen, keyMd5);
     memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
     (*pLeft) = (*pLeft)->_node.pLeft;   // -> operation use left as input
-  }else if(((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) &&
+  }else if((*pLeft)->nodeType == TSQL_NODE_COL && ((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) &&
       (tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL)){
     SSchema* schema = (*pLeft)->pSchema;
     char    keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
@@ -1212,7 +1223,16 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
   if((ret = filterDealJson(info, tree, &pLeft)) != TSDB_CODE_SUCCESS) return ret;
   SFilterFieldId left = {0}, right = {0};
   filterAddFieldFromNode(info, pLeft, &left);
-  uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
+  if (pLeft->nodeType != TSQL_NODE_VALUE && pLeft->nodeType != TSQL_NODE_COL) {
+    tree->_node.pLeft = NULL;
+  }
+
+  uint8_t type;
+  if (left.type == FLD_TYPE_EXPR) {
+    type = (uint8_t)FILTER_GET_EXPR_TYPE(info, left);
+  } else {
+    type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
+  }
   int32_t len = 0;
   uint32_t uidx = 0;
 
@@ -1576,6 +1596,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
         char str[512] = {0};
         
         SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
+        if (FILTER_GET_TYPE(left->flag) == FLD_TYPE_EXPR) {
+          continue;
+        }
         SSchema *sch = left->desc;
         if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_CONTAINS){
           len = sprintf(str, "UNIT[%d] => [%d][%s]  %s  [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str);
@@ -1751,6 +1774,9 @@ void filterFreeField(SFilterField* field, int32_t type) {
   if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) {
     if (type == FLD_TYPE_VALUE) {
       tVariantDestroy(field->desc);
+    } else if (type == FLD_TYPE_EXPR) {
+      tExprTreeDestroy(field->desc, NULL);
+      field->desc = NULL;
     }
     
     tfree(field->desc);
@@ -2100,20 +2126,30 @@ _return:
 
 int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) {
   bool empty = false;
+  if (info->fields[FLD_TYPE_COLUMN].num == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
   uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t));
   uint32_t colIdxi = 0;
   uint32_t gResIdx = 0;
+  bool hasExpr = false;
   
   for (uint32_t i = 0; i < info->groupNum; ++i) {
     SFilterGroup* g = info->groups + i;
 
     gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx));
     gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo));
+    gRes[gResIdx]->hasExpr = false;
     colIdxi = 0;
     empty = false;
     
     for (uint32_t j = 0; j < g->unitNum; ++j) {
       SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j);
+      if(u->left.type == FLD_TYPE_EXPR) {
+        gRes[gResIdx]->hasExpr = true;
+        hasExpr = true;
+        continue;
+      }
       uint32_t cidx = FILTER_UNIT_COL_IDX(u);
 
       if (gRes[gResIdx]->colInfo[cidx].info == NULL) {
@@ -2160,6 +2196,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
     ++gResIdx;
   }
 
+  if (hasExpr) {
+    FILTER_CLR_FLAG(info->status, FI_STATUS_REWRITE);
+  }
   tfree(colIdx);
 
   *gResNum = gResIdx;
@@ -2174,6 +2213,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
 void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) {
   uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
   bool equal = false;
+  if (gRes1->hasExpr || gRes2->hasExpr) {
+    *conflict = true;
+    return;
+  }
   
   for (; m < gRes1->colNum; ++m) {
     idx1 = gRes1->colIdx[m];
@@ -2646,7 +2689,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
     info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2);
     info->cunits[i].optr = FILTER_UNIT_OPTR(unit);
     info->cunits[i].colData = NULL;
-    info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
+    info->cunits[i].expr = NULL;
+    if (unit->left.type == FLD_TYPE_COLUMN) {
+      info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
+    } else if (unit->left.type == FLD_TYPE_EXPR) {
+      info->cunits[i].expr = FILTER_GET_FIELD_DESC(FILTER_GET_FIELD(info, unit->left));
+    }
     
     if (unit->right.type == FLD_TYPE_VALUE) {
       if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){   // json value is tVariant
@@ -2662,8 +2710,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
     } else {
       info->cunits[i].valData2 = info->cunits[i].valData;
     }
-    
-    info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit);
+
+    if (unit->left.type == FLD_TYPE_COLUMN) {
+      info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit);
+    } else {
+      info->cunits[i].dataSize = FILTER_GET_EXPR_SIZE(info, unit->left);
+    }
     info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit);
   }
   
@@ -2673,8 +2725,14 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
 int32_t filterUpdateComUnits(SFilterInfo *info) {
   for (uint32_t i = 0; i < info->unitNum; ++i) {
     SFilterUnit *unit = &info->units[i];
-
-    info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
+    if (unit->left.type == FLD_TYPE_EXPR) {
+      SFilterField *t = FILTER_UNIT_LEFT_FIELD(info, unit);
+      info->cunits[i].colData = NULL;
+      info->cunits[i].exprData = t->data;
+    } else {
+      info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
+      info->cunits[i].exprData = NULL;
+    }
   }
 
   return TSDB_CODE_SUCCESS;
@@ -2923,6 +2981,11 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p,
 
 
 int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols, bool* all) {
+  for (uint32_t i = 0; i < info->unitNum; ++i) {
+    if(info->cunits[i].expr) {
+      return 1;
+    }
+  }
   if (statis && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) {    
     info->blkFlag = 0;
     
@@ -2953,6 +3016,20 @@ _return:
   return TSDB_CODE_SUCCESS;
 }
 
+char *getExprColumnData(void *param, const char* name, int32_t colId) {
+  void *data = NULL;
+  getColumnDataFromId(param, colId, &data);
+  return (char *) data;
+}
+
+void* filterExprTraverse (SFilterInfo *info, int32_t numOfRows, int16_t numOfCols) {
+  tExprOperandInfo output;
+  output.data = malloc(sizeof(int64_t) * numOfRows);
+  SSDataBlock* pBlock = (SSDataBlock*) info->cunits[0].exprData;
+  SColumnDataParam param = {.numOfCols = numOfCols, .pDataBlock = (SArray*)pBlock};
+  exprTreeNodeTraverse(info->cunits[0].expr, numOfRows, &output, &param, TSDB_ORDER_ASC, getExprColumnData);
+  return output.data;
+}
 
 static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
   return true;
@@ -2963,18 +3040,26 @@ static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, i
 static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
   SFilterInfo *info = (SFilterInfo *)pinfo;
   bool all = true;
+  char *exprData = NULL;
+  uint32_t uidx = info->groups[0].unitIdxs[0];
+  void *colData = NULL;
 
   if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
     return all;
   }
 
+  if (info->cunits[0].expr) {
+    exprData = filterExprTraverse(info, numOfRows, numOfCols);
+  } else {
+    exprData = info->cunits[uidx].colData;
+  }
+
   if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
   }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint32_t uidx = info->groups[0].unitIdxs[0];
-    void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
+    colData = (char *)exprData + info->cunits[uidx].dataSize * i;
     if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
       if (!colData){  // for json->'key' is null
         (*p)[i] = 1;
@@ -2992,23 +3077,35 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
     }    
   }
 
+  if (info->cunits[0].expr) {
+    tfree(exprData);
+  }
+
   return all;
 }
 static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
   SFilterInfo *info = (SFilterInfo *)pinfo;
   bool all = true;
+  char *exprData = NULL;
+  uint32_t uidx = info->groups[0].unitIdxs[0];
+  void *colData = NULL;
 
   if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
     return all;
   }
 
+  if (info->cunits[0].expr) {
+    exprData = filterExprTraverse(info, numOfRows, numOfCols);
+  } else {
+    exprData = info->cunits[uidx].colData;
+  }
+
   if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
   }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint32_t uidx = info->groups[0].unitIdxs[0];
-    void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
+    colData = (char *)exprData + info->cunits[uidx].dataSize * i;
 
     if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
       if (!colData) {   // for json->'key' is not null
@@ -3028,6 +3125,10 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
     }
   }
 
+  if (info->cunits[0].expr) {
+    tfree(exprData);
+  }
+
   return all;
 }
 
@@ -3081,11 +3182,16 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataSta
   void *valData = info->cunits[0].valData;
   void *valData2 = info->cunits[0].valData2;
   __compar_fn_t func = gDataCompare[info->cunits[0].func];
+  char *exprData = NULL;
 
   if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
     return all;
   }
 
+  if (info->cunits[0].expr) {
+    exprData = colData = filterExprTraverse(info, numOfRows, numOfCols);
+  }
+
   if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
   }
@@ -3105,6 +3211,9 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataSta
     
     colData += dataSize;
   }
+  if (info->cunits[0].expr) {
+    tfree(exprData);
+  }
 
   return all;
 }
@@ -3112,18 +3221,26 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataSta
 bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
   SFilterInfo *info = (SFilterInfo *)pinfo;
   bool all = true;
+  char *exprData = NULL;
+  uint32_t uidx = info->groups[0].unitIdxs[0];
+  void *colData = NULL;
 
   if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
     return all;
   }
   
+  if (info->cunits[0].expr) {
+    exprData = filterExprTraverse(info, numOfRows, numOfCols);
+  } else {
+    exprData = info->cunits[uidx].colData;
+  }
+
   if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
   }
-  
+
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint32_t uidx = info->groups[0].unitIdxs[0];
-    void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
+    colData = (char *)exprData + info->cunits[uidx].dataSize * i;
     if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) {
       (*p)[i] = 0;
       all = false;
@@ -3152,6 +3269,10 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStat
     }
   }
 
+  if (info->cunits[0].expr) {
+    tfree(exprData);
+  }
+
   return all;
 }
 
@@ -3167,6 +3288,27 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *
     *p = calloc(numOfRows, sizeof(int8_t));
   }
   
+  SArray* tmpData = NULL;
+  for (uint32_t g = 0; g < info->groupNum; ++g) {
+    SFilterGroup *group = &info->groups[g];
+    for (uint32_t u = 0; u < group->unitNum; ++u) {
+      uint32_t uidx = group->unitIdxs[u];
+      SFilterComUnit *cunit = &info->cunits[uidx];
+      if (cunit->expr) {
+        if (!tmpData) {
+          tmpData = taosArrayInit(10, POINTER_BYTES);
+        }
+        tExprOperandInfo output;
+        output.data = malloc(sizeof(int64_t) * numOfRows);
+        taosArrayPush(tmpData, output.data);
+        SSDataBlock* pBlock = (SSDataBlock*) cunit->exprData;
+        SColumnDataParam param = {.numOfCols = numOfCols, .pDataBlock = (SArray*)pBlock};
+        exprTreeNodeTraverse(cunit->expr, numOfRows, &output, &param, TSDB_ORDER_ASC, getExprColumnData);
+        cunit->colData = (char *) output.data;
+      }
+    }
+  }
+
   for (int32_t i = 0; i < numOfRows; ++i) {
     //FILTER_UNIT_CLR_F(info);
   
@@ -3228,6 +3370,9 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *
     }    
   }
 
+  if (tmpData) {
+    taosArrayDestroy(&tmpData);
+  }
   return all;
 }
 
@@ -3317,7 +3462,8 @@ _return:
 
 int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
   CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
-  CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
+  CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0 && info->fields[FLD_TYPE_EXPR].num <= 0,
+           TSDB_CODE_QRY_APP_ERROR, "no column fileds");
 
   if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
     return TSDB_CODE_SUCCESS;
@@ -3330,6 +3476,11 @@ int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from
     (*fp)(param, sch->colId, &fi->data);
   }
 
+   for (uint32_t i = 0; i < info->fields[FLD_TYPE_EXPR].num; ++i) {
+     SFilterField* fi = &info->fields[FLD_TYPE_EXPR].fields[i];
+     (*fp)(param, INT32_MAX, &fi->data);
+   }
+
   filterUpdateComUnits(info);
 
   return TSDB_CODE_SUCCESS;

src/query/src/qSqlParser.c

@@ -321,7 +321,9 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
   if ((pLeft != NULL && pRight != NULL) &&
       (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM ||
        optrType == TK_EQ || optrType == TK_NE || optrType == TK_LT || optrType == TK_GT || optrType == TK_LE || optrType == TK_GE ||
-       optrType == TK_AND || optrType == TK_OR)) {
+       optrType == TK_AND || optrType == TK_OR || optrType == TK_BITAND || optrType == TK_BITOR || optrType == TK_BITXOR ||
+       optrType == TK_LSHIFT || optrType == TK_RSHIFT))
+  {
     /*
      * if a exprToken is noted as the TK_TIMESTAMP, the time precision is microsecond
      * Otherwise, the time precision is adaptive, determined by the time precision from databases.
@@ -398,6 +400,61 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
           pExpr->value.i64 = (pLeft->value.i64 || pRight->value.i64) ? 1 : 0;
           break;
         }
+        case TK_BITAND: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 & pRight->value.i64;
+          }
+          break;
+        }
+        case TK_BITOR: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 | pRight->value.i64;
+          }
+          break;
+        }
+        case TK_BITXOR: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 ^ pRight->value.i64;
+          }
+          break;
+        }
+        case TK_LSHIFT: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 << pRight->value.i64;
+          }
+          break;
+        }
+        case TK_RSHIFT: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 >> pRight->value.i64;
+          }
+          break;
+        }
       }
 
       tSqlExprDestroy(pLeft);
@@ -509,6 +566,13 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
           pExpr->value.i64 = (left || right) ? 1 : 0;
           break;
         }
+        case TK_BITAND:
+        case TK_BITOR:
+        case TK_BITXOR:
+        case TK_LSHIFT:
+        case TK_RSHIFT:
+          pExpr->value.i64 = TSDB_DATA_DOUBLE_NULL;
+          break;
       }
 
       tSqlExprDestroy(pLeft);
@@ -542,6 +606,12 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
     }
 
     pExpr->pRight = pRight;
+
+    if (optrType == TK_BITNOT) {
+      pExpr->exprToken.z = pLeft->exprToken.z - 1;
+      pExpr->exprToken.n = pLeft->exprToken.n + 1;
+      pExpr->exprToken.type = pLeft->exprToken.type;
+    }
   }
 
   return pExpr;

src/tsdb/src/tsdbRead.c

@@ -173,6 +173,11 @@ typedef struct SRange {
   int32_t to;
 } SRange;
 
+typedef struct STagBlockInfo {
+  SSkipListNode *pSkipListNode;
+  SArray *pBlock;
+} STagBlockInfo;
+
 static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList);
 static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList);
 static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle);
@@ -4334,7 +4339,15 @@ static FORCE_INLINE int32_t tsdbGetTagDataFromId(void *param, int32_t id, void *
   return TSDB_CODE_SUCCESS;
 }
 
-
+static FORCE_INLINE int32_t tsdbGetTagData(void *param, int32_t id, void **data) {
+  STagBlockInfo* pInfo = (STagBlockInfo*) param;
+  if (id == INT32_MAX) {
+   *data = pInfo->pBlock;
+  } else {
+    return tsdbGetTagDataFromId(pInfo->pSkipListNode, id, data);
+  }
+  return TSDB_CODE_SUCCESS;
+}
 
 static void queryIndexedColumn(SSkipList* pSkipList, void* filterInfo, SArray* res) {
   SSkipListIterator* iter = NULL;
@@ -4389,26 +4402,77 @@ static void queryIndexedColumn(SSkipList* pSkipList, void* filterInfo, SArray* r
   tsdbDebug("filter index column end");
 }
 
+static void getAllExprColId(tExprNode* pExpr, SArray* array) {
+  if (!pExpr) {
+    return;
+  }
+  if (pExpr->nodeType == TSQL_NODE_FUNC) {
+    for (int32_t i = 0; i < pExpr->_func.numChildren; ++i) {
+      getAllExprColId(pExpr->_func.pChildren[i], array);
+    }
+  } else if (pExpr->nodeType == TSQL_NODE_EXPR) {
+    getAllExprColId(pExpr->_node.pLeft, array);
+    getAllExprColId(pExpr->_node.pRight, array);
+  } else if (pExpr->nodeType == TSQL_NODE_COL) {
+    taosArrayPush(array, &pExpr->pSchema->colId);
+  }
+}
+
+static void getAllFilterExprColId(SFilterFields* pSf, SArray* array) {
+  for (uint32_t i = 0; i < pSf->num; ++i) {
+    SFilterField* fi = &(pSf->fields[i]);
+    if (FILTER_GET_TYPE(fi->flag) == FLD_TYPE_EXPR) {
+      getAllExprColId(fi->desc, array);
+    }
+  }
+  taosArraySort(array, getComparFunc(TSDB_DATA_TYPE_SMALLINT, 0));
+  taosArrayRemoveDuplicate(array, getComparFunc(TSDB_DATA_TYPE_SMALLINT, 0), NULL);
+}
+
 static void queryIndexlessColumn(SSkipList* pSkipList, void* filterInfo, SArray* res) {
   SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
   int8_t *addToResult = NULL;
+  SFilterInfo *sfInfo = (SFilterInfo *)filterInfo;
+  SArray *array = NULL;
+  SArray *pDataBlock = NULL;
 
+  if (sfInfo->fields[FLD_TYPE_EXPR].num > 0) {
+    array = taosArrayInit(10, sizeof(int16_t));
+    getAllFilterExprColId(&(sfInfo->fields[FLD_TYPE_EXPR]), array);
+  }
   while (tSkipListIterNext(iter)) {
 
     SSkipListNode *pNode = tSkipListIterGet(iter);
 
-    filterSetColFieldData(filterInfo, pNode, tsdbGetTagDataFromId);
 
-    char *pData = SL_GET_NODE_DATA(pNode);
+    if (sfInfo->fields[FLD_TYPE_EXPR].num > 0) {
+      pDataBlock = taosArrayInit(10, sizeof(SColumnInfoData));
+      size_t num = taosArrayGetSize(array);
+      for(int32_t i = 0; i < num; ++i) {
+        int16_t *pColId = taosArrayGet(array, i);
+        void *data = NULL;
+        tsdbGetTagDataFromId(pNode, *pColId, &data);
+        SColumnInfoData colData = {{0}};
+        colData.pData = data;
+        colData.info.colId = *pColId;
+        taosArrayPush(pDataBlock, &colData);
+      }
+    }
+    STagBlockInfo stInfo = {.pSkipListNode = pNode, .pBlock = pDataBlock};
+    filterSetColFieldData(filterInfo, &stInfo, tsdbGetTagData);
 
-    bool all = filterExecute(filterInfo, 1, &addToResult, NULL, 0);
+    char *pData = SL_GET_NODE_DATA(pNode);
+    int16_t numOfCols = array ? (int16_t)taosArrayGetSize(array) : 0;
+    bool all = filterExecute(filterInfo, 1, &addToResult, NULL, numOfCols);
 
     if (all || (addToResult && *addToResult)) {
       STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL};
       taosArrayPush(res, &info);
     }
+    taosArrayDestroy(&pDataBlock);
   }
 
+  taosArrayDestroy(&array);
   tfree(addToResult);
 
   tSkipListDestroyIter(iter);

src/util/src/ttokenizer.c

@@ -64,6 +64,7 @@ static SKeyword keywordTable[] = {
     {"LE",           TK_LE},
     {"BITAND",       TK_BITAND},
     {"BITOR",        TK_BITOR},
+    {"BITXOR",       TK_BITXOR},
     {"LSHIFT",       TK_LSHIFT},
     {"RSHIFT",       TK_RSHIFT},
     {"PLUS",         TK_PLUS},
@@ -413,6 +414,10 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) {
       *tokenId = TK_BITNOT;
       return 1;
     }
+    case '^': {
+      *tokenId = TK_BITXOR;
+      return 1;
+    }
     case '?': {
       *tokenId = TK_QUESTION;
       return 1;

tests/parallel_test/cases.task

@@ -751,7 +751,8 @@
 4,,pytest,python3 test.py -f alter/alter_create_exception.py 
 4,,pytest,python3 test.py -f insert/line_insert.py
 3,,pytest,python3 test.py -f tag_lite/binary.py 
-3,,pytest,python3 test.py -f query/filterAllIntTypes.py 
+3,,pytest,python3 test.py -f query/filterAllIntTypes.py
+3,,pytest,python3 test.py -f query/queryLogicOperators.py 
 3,,pytest,python3 ./test.py -f query/queryNcharNull.py
 3,,develop-test,python3 ./test.py -f 2-query/ts_hidden_column.py 
 3,,develop-test,python3 ./test.py -f 1-insert/uppercase_in_stmt.py

tests/pytest/query/queryLogicOperators.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+
+from numpy.lib.function_base import insert
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+import numpy as np
+
+# constant define
+WAITS = 5 # wait seconds
+
+class TDTestCase:
+
+    # init
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        tdSql.prepare()        
+        self.ts = 1500000000000
+        self.rows = 100
+
+
+    # run case  
+    def run(self):
+        self.case1()
+        self.case2()
+        self.case3()
+        
+    def case1(self):
+        tdSql.execute("create table tb1(ts timestamp, c1 int, c2 int, c3 bool)")
+        sql = "insert into tb1 values "
+        for i in range(self.rows):
+            sql += "(%d, %d, 1, %d)" % (self.ts + i, i % 10, i % 2)
+        tdSql.execute(sql)
+        
+        operators = ['&', '|', '^', '<<', '>>']
+        
+        for operator in operators:
+            tdSql.query("select c1 %s c2 from tb1" % operator)
+            tdSql.checkRows(100)
+            
+            tdSql.query("select * from tb1 where c1 %s c2 = 0" % operator)
+            if operator == '|':
+                tdSql.checkRows(0)
+            elif operator == '&':
+                tdSql.checkRows(50)
+            elif operator == '^':
+                tdSql.checkRows(10)
+            elif operator == '<<':
+                tdSql.checkRows(10)
+            elif operator == '<<':
+                tdSql.checkRows(20)
+        
+        tdSql.query("select ~c1 from tb1")
+        tdSql.checkRows(100)
+
+        tdSql.query("select ~c1 from (select * from tb1 where c1 & c2 > 0) where c1 ^ c2 < 1 ")
+        tdSql.checkRows(10)
+    
+    def case2(self):
+        tdSql.execute("create table tb2(ts timestamp, c1 tinyint, c2 tinyint unsigned, c3 smallint, c4 smallint unsigned, c5 int, c6 int unsigned, c7 bigint, c8 bigint unsigned, c9 float, c10 double, c11 bool, c12 binary(10), c13 nchar(10))")
+        sql = "insert into tb2 values "
+        for i in range(self.rows):
+            sql += "(%d, 1, 1, 2, 2, 3, 3, 4, 4, 1.1, 2.5, True, 'test', 'test')" % (self.ts + i)
+        tdSql.execute(sql)
+        
+        operators = ['&', '|', '^', '<<', '>>']
+        
+        for operator in operators:
+            for i in range(13):                
+                if i < 8:
+                    tdSql.query("select 1 %s c%d from tb2" % (operator, i + 1))
+                    tdSql.checkRows(self.rows)
+                else:
+                    tdSql.error("select 1 %s c%d from tb2" % (operator, i + 1))
+                    tdSql.error("select c%d %s c%d from tb2" % (i + 1, operator, i + 1))
+                
+        for i in range(13):                    
+            if i < 8:
+                tdSql.query("select  %sc%d from tb2" % ('~', i + 1))
+                tdSql.checkRows(self.rows)
+            else:
+                tdSql.error("select %sc%d from tb2" % ('~', i + 1))
+        
+    def case3(self):
+        tdSql.execute("create table st1(ts timestamp, c1 int, c2 int, c3 binary(20)) tags(t1 int, t2 int, t3 nchar(20))")
+        for i in range(self.rows):
+            tdSql.execute("create table t%d using st1 tags(%d, 1, 'test')" % (i, i))
+            sql = "insert into t%d values " % i       
+            for i in range(self.rows):
+                sql += "(%d, %d, 1, %d)" % (self.ts + i, i % 10, i % 2)
+            tdSql.execute(sql)
+        
+        operators = ['&', '|', '^', '<<', '>>']
+        
+        for operator in operators:
+            tdSql.query("select c1 %s c2 from st1" % operator)
+            tdSql.checkRows(10000)
+            
+            tdSql.query("select * from st1 where c1 %s c2 = 0" % operator)
+            if operator == '|':
+                tdSql.checkRows(0)
+            elif operator == '&':
+                tdSql.checkRows(5000)
+            elif operator == '^':
+                tdSql.checkRows(1000)
+            elif operator == '<<':
+                tdSql.checkRows(1000)
+            elif operator == '>>':
+                tdSql.checkRows(2000)
+        
+        tdSql.query("select ~c1 from st1")
+        tdSql.checkRows(10000)
+
+        tdSql.query("select c1, c2 from (select * from st1 where c1 & c2 > 0) where c1 | c2 = 1")
+        tdSql.checkRows(1000)
+
+
+    # stop 
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file