Merge branch 'feature/TS-1210' into feature/TS-1436-2.4

src/client/src/tscSQLParser.c

@@ -145,6 +145,8 @@ static int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo);
 static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex);
 static void convertWhereStringCharset(tSqlExpr* pRight);
 
+static bool isLogicalOperator(tSqlExpr* pExpr);
+static bool isComparisonOperator(tSqlExpr* pExpr);
 int validateTableName(char *tblName, int len, SStrToken* psTblToken, bool *dbIncluded);
 
 static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) {
@@ -284,6 +286,16 @@ static uint8_t convertRelationalOperator(SStrToken *pToken) {
       return TSDB_BINARY_OP_REMAINDER;
     case TK_BITAND:
       return TSDB_BINARY_OP_BITAND;
+    case TK_BITOR:
+      return TSDB_BINARY_OP_BITOR;
+    case TK_BITXOR:
+      return TSDB_BINARY_OP_BITXOR;
+    case TK_BITNOT:
+      return TSDB_BINARY_OP_BITNOT;
+    case TK_LSHIFT:
+      return TSDB_BINARY_OP_LSHIFT;
+    case TK_RSHIFT:
+      return TSDB_BINARY_OP_RSHIFT;
     case TK_LIKE:
       return TSDB_RELATION_LIKE;
     case TK_MATCH:
@@ -4274,15 +4286,10 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
 
   STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
   SSchema*    pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
-  int32_t     ret = 0;
+  int32_t     ret = TSDB_CODE_SUCCESS;
   const char* msg1 = "non binary column not support like/match operator";
   const char* msg3 = "bool column not support this operator";
   const char* msg4 = "primary key not support this operator";
-
-  SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->id.uid, pSchema);
-
-  pColumn->info.flist.numOfFilters++;
-  
   /*
    * in case of TK_AND filter condition, we first find the corresponding column and build the query condition together
    * the already existed condition.
@@ -4296,8 +4303,8 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
   pColFilter->filterstr =
       ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0);
 
-  if (!pColFilter->filterstr) {
-    if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) {
+  if (!pColFilter->filterstr && tSqlExprIsParentOfLeaf(pExpr)) {
+     if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH) {
       ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
       goto _err_ret;
     }
@@ -4310,11 +4317,10 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
       }
     }
   }
-
-  pColumn->columnIndex = pIndex->columnIndex;
-  pColumn->tableUid = pTableMeta->id.uid;
-  if (pColumn->columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pExpr->tokenId == TK_IN) {
-     return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
+  if (pIndex->columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pTableMeta->id.uid == TK_IN
+      && tSqlExprIsParentOfLeaf(pExpr)) {
+     ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
+     goto _err_ret;
   }
 
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
@@ -4326,8 +4332,32 @@ _err_ret:
   return ret;
 }
 
+static int32_t addAllColumn(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, uint32_t conTokenId,
+                            SColumnIndex* pIndex) {
+  int32_t ret = TSDB_CODE_SUCCESS;
+
+  if(!tSqlExprIsLeaf(pExpr) || pExpr->tokenId == TK_ARROW) {
+    ret = addAllColumn(pCmd, pQueryInfo, pExpr->pLeft, conTokenId, pIndex);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+    return addAllColumn(pCmd, pQueryInfo, pExpr->pRight, conTokenId, pIndex);
+  } else if (pExpr->tokenId == TK_ID) {
+    if (getColumnIndexByName(&pExpr->columnName, pQueryInfo, pIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    }
+    STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex);
+    STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
+    SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
+    SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->id.uid, pSchema);
+    pColumn->info.flist.numOfFilters++;
+  }
+  return ret;
+}
+
 static int32_t getColQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr) {
   int32_t ret = TSDB_CODE_SUCCESS;
+  const char* msg6 = "illegal condition expression";
   
   for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
     tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i);
@@ -4339,7 +4369,18 @@ static int32_t getColQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlEx
 
     SArray* colList = taosArrayInit(10, sizeof(SColIndex));  
     ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL);
+    size_t colNum = taosArrayGetSize(colList);
+    for (int32_t k = 0; k < colNum; k++) {
+      SColIndex* pColIndex = taosArrayGet(colList, k);
+      if (TSDB_COL_IS_TAG(pColIndex->flag)) {
+        ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
+        break;
+      }
+    }
     taosArrayDestroy(&colList);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = exprTreeValidateTree(tscGetErrorMsgPayload(pCmd), p);
+    }
 
     SBufferWriter bw = tbufInitWriter(NULL, false);
 
@@ -4387,7 +4428,7 @@ static int32_t checkColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
   }
   pQueryInfo->onlyHasTagCond &= false;
 
-  if (!tSqlExprIsParentOfLeaf(pExpr)) {  // internal node
+  if (isLogicalOperator(pExpr)) {  // internal node
     int32_t ret = checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
@@ -4396,12 +4437,10 @@ static int32_t checkColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
     return checkColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId);
   } else {  // handle leaf node
     SColumnIndex index = COLUMN_INDEX_INITIALIZER;
-    if (getColumnIndexByName(&pExpr->pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
-      return TSDB_CODE_TSC_INVALID_OPERATION;
-    }
-
+    addAllColumn(pCmd, pQueryInfo, pExpr, pExpr->tokenId, &index);
     return checkColumnFilterInfo(pCmd, pQueryInfo, &index, pExpr, relOptr);
   }
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) {
@@ -4678,19 +4717,23 @@ static int32_t validateSQLExprItemOperatorExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, S
   if (ret != TSDB_CODE_SUCCESS) {
     return ret;
   }
+
   int32_t rightHeight = 0;
-  ret = validateSQLExprItem(pCmd, pExpr->pRight, pQueryInfo, pList, &rightType, &uidRight, &rightHeight);
-  if (ret != TSDB_CODE_SUCCESS) {
-    return ret;
-  }
+  if (pExpr->tokenId != TK_BITNOT) {
+    ret = validateSQLExprItem(pCmd, pExpr->pRight, pQueryInfo, pList, &rightType, &uidRight, &rightHeight);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
 
-  if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) {
-    return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) {
+      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    }
   }
+
   *uid = uidLeft;
 
-  *height = (leftHeight > rightHeight) ? leftHeight + 1 : rightHeight+1;
-  {
+  *height = (leftHeight > rightHeight) ? leftHeight + 1 : rightHeight + 1;
+  if (pExpr->tokenId != TK_BITNOT) {
     if (leftType == SQLEXPR_TYPE_UNASSIGNED || rightType == SQLEXPR_TYPE_UNASSIGNED) {
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "invalid operand expression");
     }
@@ -4726,8 +4769,15 @@ static int32_t validateSQLExprItemOperatorExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, S
                pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_NMATCH ||
                pExpr->tokenId == TK_CONTAINS || pExpr->tokenId == TK_IN) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "unsupported filtering operations");
+    } else if (pExpr->tokenId == TK_LSHIFT || pExpr->tokenId == TK_RSHIFT) {
+      if (rightType != SQLEXPR_TYPE_VALUE) {
+        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "non numeric right operand");
+      }
     }
+  } else {
+    *type = SQLEXPR_TYPE_SCALAR;
   }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -5137,6 +5187,38 @@ void convertWhereStringCharset(tSqlExpr* pRight){
   free(newData);
 }
 
+static int32_t handleColumnInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SColumnIndex* index) {
+  const char* msg2 = "illegal column name";
+  int32_t ret = TSDB_CODE_SUCCESS;
+  if (pExpr == NULL) {
+    return ret;
+  } else if (!tSqlExprIsLeaf(pExpr)) {
+    if (isComparisonOperator(pExpr)) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    }
+    ret = handleColumnInQueryCond(pCmd, pQueryInfo, pExpr->pLeft, index);
+    if( ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+    ret = handleColumnInQueryCond(pCmd, pQueryInfo, pExpr->pRight, index);
+    return ret;
+  }
+
+  SStrToken* colName = NULL;
+  if (pExpr->tokenId == TK_ARROW) {
+    colName = &(pExpr->pLeft->columnName);
+  } else if (pExpr->tokenId == TK_ID) {
+    colName = &(pExpr->columnName);
+  }
+  
+  if (colName) {
+    if (getColumnIndexByName(colName, pQueryInfo, index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
+      ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+    }
+  }
+  return ret;
+}
+
 static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr,
                                      int32_t* type, int32_t* tbIdx, int32_t parentOptr, tSqlExpr** columnExpr,
                                      tSqlExpr** tsExpr, bool joinQuery) {
@@ -5144,7 +5226,6 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   const char* msg2 = "illegal column name";
   const char* msg4 = "too many join tables";
   const char* msg5 = "not support ordinary column join";
-  const char* msg6 = "illegal condition expression";
 
   tSqlExpr* pLeft  = (*pExpr)->pLeft;
   tSqlExpr* pRight = (*pExpr)->pRight;
@@ -5158,14 +5239,23 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   int32_t ret = TSDB_CODE_SUCCESS;
 
   SColumnIndex index = COLUMN_INDEX_INITIALIZER;
-  if (getColumnIndexByName(colName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
-    return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+  if (!tSqlExprIsParentOfLeaf(*pExpr)) {
+    ret = handleColumnInQueryCond(pCmd, pQueryInfo, pLeft, &index);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+    ret = handleColumnInQueryCond(pCmd, pQueryInfo, pRight, &index);
+    if (ret != TSDB_CODE_SUCCESS) {
+      return ret;
+    }
+  } else {
+    if (getColumnIndexByName(colName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
+      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
+    }
   }
 
   *tbIdx = index.tableIndex;
 
-  assert(tSqlExprIsParentOfLeaf(*pExpr));
-
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
   STableMeta*     pTableMeta = pTableMetaInfo->pTableMeta;
 
@@ -5194,7 +5284,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
   }
 
   if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) {  // query on time range
-    if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
+    if (!tSqlExprIsParentOfLeaf(*pExpr) || !validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
 
@@ -5316,8 +5406,6 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
     if (pRight->tokenId == TK_ID) {
       if (joinQuery) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); // other column cannot be served as the join column
-      } else {
-        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
       }
     }
 
@@ -5368,7 +5456,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr
   int32_t leftTbIdx = 0;
   int32_t rightTbIdx = 0;
 
-  if (!tSqlExprIsParentOfLeaf(*pExpr)) {
+  if (isLogicalOperator(*pExpr)) {
     ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, type ? &leftType : NULL, &leftTbIdx, (*pExpr)->tokenId, &columnLeft, &tsLeft, joinQuery);
     if (ret != TSDB_CODE_SUCCESS) {
       goto err_ret;
@@ -5429,6 +5517,11 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr
     goto err_ret;
   }
 
+  if (!tSqlExprIsLeaf((*pExpr)->pRight) ) {
+    ret = TSDB_CODE_TSC_INVALID_OPERATION;
+    goto err_ret;
+  }
+
   ret = handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, tbIdx, parentOptr, columnExpr, tsExpr, joinQuery);
   if (ret) {
     goto err_ret;
@@ -5451,19 +5544,21 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo*
     return;
   }
   
-  if (tSqlExprIsParentOfLeaf(*pExpr)) {
+  if (!isLogicalOperator(*pExpr)) {
     tSqlExpr* pLeft = (*pExpr)->pLeft;
 
-    SColumnIndex index = COLUMN_INDEX_INITIALIZER;
-    if(pLeft->tokenId == TK_ARROW) {
-      pLeft = pLeft->pLeft;
-    }
-    if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
-      return;
-    }
+    if (pLeft->tokenId == TK_ARROW || pLeft->tokenId == TK_ID) {
+      SColumnIndex index = COLUMN_INDEX_INITIALIZER;
+      if(pLeft->tokenId == TK_ARROW) {
+        pLeft = pLeft->pLeft;
+      }
+      if (getColumnIndexByName(&pLeft->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
+        return;
+      }
 
-    if (index.tableIndex != tableIndex) {
-      return;
+      if (index.tableIndex != tableIndex) {
+        return;
+      }
     }
 
     *pOut = *pExpr;
@@ -5686,7 +5781,6 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInf
 */
 
 static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
-  const char *msg1 = "invalid tag operator";
   const char* msg2 = "not supported filter condition";
   
   do {
@@ -5706,10 +5800,6 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
     if (p->_node.pRight && (retVal = validateTagCondExpr(pCmd, p->_node.pRight)) != TSDB_CODE_SUCCESS) {
       return retVal;
     }
-
-    if (IS_ARITHMETIC_OPTR(p->_node.optr)) {
-      return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
-    }
     
     if (!IS_RELATION_OPTR(p->_node.optr)) {
       break;
@@ -5771,8 +5861,13 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
   return TSDB_CODE_SUCCESS;
 }
 
+bool isTablename(char* colName) {
+   return (strlen(colName) == strlen(TSQL_TBNAME_L) && strncasecmp(TSQL_TBNAME_L, colName, strlen(TSQL_TBNAME_L)) == 0);
+}
+
 static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) {
   int32_t ret = TSDB_CODE_SUCCESS;
+  const char* msg6 = "illegal condition expression";
 
   if (pCondExpr->pTagCond == NULL) {
     return ret;
@@ -5788,6 +5883,17 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
   
     SArray* colList = taosArrayInit(10, sizeof(SColIndex));
     ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL);
+    size_t colNum = taosArrayGetSize(colList);
+    for (int32_t k = 0; k < colNum; k++) {
+      SColIndex* pColIndex = taosArrayGet(colList, k);
+      if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag) && !isTablename(pColIndex->name)) {
+        ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
+        break;
+      }
+    }
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = exprTreeValidateTree(tscGetErrorMsgPayload(pCmd), p);
+    }
     //if (ret == TSDB_CODE_SUCCESS) {
     //  ret = filterInitFromTree(p, &pQueryInfo->tagFilter, (int32_t)taosArrayGetSize(colList), NULL);
     //}
@@ -10279,23 +10385,6 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
       }
     }
 
-    if (pSqlExpr->tokenId == TK_BITAND && pSqlExpr->pLeft != NULL && pSqlExpr->pRight != NULL) {
-      // for example: col type is "bool" but expr "col & 1" received
-      uint8_t colType = pLeft->pSchema->type;
-      SStrToken *exprToken = &pSqlExpr->pRight->exprToken;
-      if (pSqlExpr->pLeft->type == SQL_NODE_TABLE_COLUMN && pSqlExpr->pRight->type == SQL_NODE_VALUE) {
-        if (colType == TSDB_DATA_TYPE_BOOL) {
-          if ((exprToken->n != 4 || strncasecmp(exprToken->z, "true", 4)) && (exprToken->n != 5 || strncasecmp(exprToken->z, "false", 5))) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-        } else if (IS_SIGNED_NUMERIC_TYPE(colType) || IS_UNSIGNED_NUMERIC_TYPE(colType)) {
-          if ((exprToken->n == 4 && strncasecmp(exprToken->z, "true", 4) == 0) || (exprToken->n == 5 || strncasecmp(exprToken->z, "false", 5) == 0)) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-        }
-      }
-    }
-
     if (pSqlExpr->pRight != NULL) {
       int32_t ret = exprTreeFromSqlExpr(pCmd, &pRight, pSqlExpr->pRight, pQueryInfo, pCols, uid);
       if (ret != TSDB_CODE_SUCCESS) {
@@ -10334,8 +10423,11 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
           pLeft = pLeft->_node.pLeft;
         }
         if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && pLeft->nodeType == TSQL_NODE_COL) {
-          if (((*pExpr)->_node.optr != TSDB_BINARY_OP_BITAND && pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) ||
-              pLeft->pSchema->type == TSDB_DATA_TYPE_JSON) {
+          if ((((*pExpr)->_node.optr != TSDB_BINARY_OP_BITAND || (*pExpr)->_node.optr != TSDB_BINARY_OP_BITOR ||
+                (*pExpr)->_node.optr != TSDB_BINARY_OP_BITXOR || (*pExpr)->_node.optr != TSDB_BINARY_OP_BITNOT ||
+                (*pExpr)->_node.optr != TSDB_BINARY_OP_LSHIFT || (*pExpr)->_node.optr != TSDB_BINARY_OP_RSHIFT) &&
+              pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) || pLeft->pSchema->type == TSDB_DATA_TYPE_JSON)
+          {
             return TSDB_CODE_TSC_INVALID_OPERATION;
           }
         }
@@ -10444,3 +10536,35 @@ void normalizeSqlNode(SSqlNode* pSqlNode, const char* dbName) {
 }
 
 #endif
+
+bool isLogicalOperator(tSqlExpr* pExpr) {
+  if (pExpr->tokenId == TK_AND || pExpr->tokenId == TK_OR) {
+    return true;
+  }
+  return false;
+}
+
+bool isComparisonOperator(tSqlExpr* pExpr) {
+ switch (pExpr->tokenId) {
+    case TK_EQ:
+    case TK_NE:
+    case TK_ISNULL:
+    case TK_NOTNULL:
+    case  TK_IS:
+    case  TK_LIKE:
+    case  TK_MATCH:
+    case  TK_NMATCH:
+    case  TK_CONTAINS:
+    case  TK_GLOB:
+    case  TK_BETWEEN:
+    case  TK_IN:
+    case  TK_GT:
+    case  TK_GE:
+    case  TK_LT:
+    case  TK_LE:
+      return true;
+    default:
+      return false;
+  }
+  return false;
+}

src/client/src/tscUtil.c

@@ -1004,7 +1004,7 @@ static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, void* pFilterI
     filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar);
     int8_t* p = NULL;
     //bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p);
-    bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, 0);
+    bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, (int16_t)taosArrayGetSize(pBlock->pDataBlock));
     if (gotNchar) {
       filterFreeNcharColumns(pFilterInfo);
     }

src/common/src/texpr.c

@@ -88,7 +88,6 @@ int32_t exprTreeValidateFunctionNode(char* msgbuf, tExprNode *pExpr) {
 int32_t exprTreeValidateExprNode(tExprNode *pExpr) {
   int16_t leftType = pExpr->_node.pLeft->resultType;
   int16_t rightType = pExpr->_node.pRight->resultType;
-  int16_t resultType = leftType;
 
   if (pExpr->_node.optr == TSDB_BINARY_OP_ADD || pExpr->_node.optr == TSDB_BINARY_OP_SUBTRACT ||
       pExpr->_node.optr == TSDB_BINARY_OP_MULTIPLY || pExpr->_node.optr == TSDB_BINARY_OP_DIVIDE ||
@@ -99,171 +98,31 @@ int32_t exprTreeValidateExprNode(tExprNode *pExpr) {
     pExpr->resultType = TSDB_DATA_TYPE_DOUBLE;
     pExpr->resultBytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
     return TSDB_CODE_SUCCESS;
-  } else if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND) {
-    if ((leftType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(leftType) && !IS_UNSIGNED_NUMERIC_TYPE(leftType)) ||
-        (rightType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(rightType) && !IS_UNSIGNED_NUMERIC_TYPE(rightType)))
-    {
+  } else if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND || pExpr->_node.optr == TSDB_BINARY_OP_BITOR ||
+             pExpr->_node.optr == TSDB_BINARY_OP_BITXOR || pExpr->_node.optr == TSDB_BINARY_OP_LSHIFT ||
+             pExpr->_node.optr == TSDB_BINARY_OP_RSHIFT)
+  {
+    if (!IS_NUMERIC_TYPE(leftType) || !IS_NUMERIC_TYPE(rightType)) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
-
-    uint8_t schemaType;
-    // now leftType and rightType are both numeric
-    if (pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_COL) {
-      if (leftType != rightType) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
-      }
-    } else if (pExpr->_node.pLeft->nodeType == TSQL_NODE_COL) {
-      if (pExpr->_node.pRight->nodeType != TSQL_NODE_VALUE) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
-      } else {
-        schemaType = pExpr->_node.pLeft->pSchema->type;
-        int64_t sVal = pExpr->_node.pRight->pVal->i64;
-        uint64_t uVal = pExpr->_node.pRight->pVal->u64;
-
-        switch (schemaType) {
-        case TSDB_DATA_TYPE_BOOL:
-          if ((pExpr->_node.pRight->pVal->nType != TSDB_DATA_TYPE_BOOL) ||
-              (pExpr->_node.pRight->pVal->i64 != 0 &&
-               pExpr->_node.pRight->pVal->i64 != 1 &&
-               pExpr->_node.pRight->pVal->i64 != TSDB_DATA_BOOL_NULL))
-          {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_TINYINT:
-          if (sVal < -128 || sVal > 127) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_SMALLINT:
-          if (sVal < -32768 || sVal > 32767) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_INT:
-          if (sVal < INT32_MIN || sVal > INT32_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_BIGINT:
-          if (sVal < INT64_MIN || sVal > INT64_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_UTINYINT:
-          if (uVal > 255) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_USMALLINT:
-          if (uVal > 65535) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_UINT:
-          if (uVal > UINT32_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        case TSDB_DATA_TYPE_UBIGINT:
-          if (uVal > UINT64_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          break;
-        }
-
-        pExpr->_node.pRight->pSchema->type = schemaType;
-        pExpr->_node.pRight->pVal->nType = schemaType;
-
-        pExpr->_node.pRight->resultType = schemaType;
-        pExpr->_node.pRight->resultBytes = tDataTypes[schemaType].bytes;
-      }
-    } else {
-      if (pExpr->_node.pLeft->nodeType != TSQL_NODE_VALUE) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
-      } else {
-        schemaType = pExpr->_node.pRight->pSchema->type;
-        int64_t sVal = pExpr->_node.pLeft->pVal->i64;
-        uint64_t uVal = pExpr->_node.pLeft->pVal->u64;
-        switch (schemaType) {
-        case TSDB_DATA_TYPE_BOOL:
-          if ((pExpr->_node.pLeft->pVal->nType != TSDB_DATA_TYPE_BOOL) ||
-              (pExpr->_node.pLeft->pVal->i64 != 0 &&
-               pExpr->_node.pLeft->pVal->i64 != 1 &&
-               pExpr->_node.pLeft->pVal->i64 != TSDB_DATA_BOOL_NULL))
-          {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 1;
-          break;
-        case TSDB_DATA_TYPE_TINYINT:
-          if (sVal < -128 || sVal > 127) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 1;
-          break;
-        case TSDB_DATA_TYPE_SMALLINT:
-          if (sVal < -32768 || sVal > 32767) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 2;
-          break;
-        case TSDB_DATA_TYPE_INT:
-          if (sVal < INT32_MIN || sVal > INT32_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 4;
-          break;
-        case TSDB_DATA_TYPE_BIGINT:
-          if (sVal < INT64_MIN || sVal > INT64_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 8;
-          break;
-        case TSDB_DATA_TYPE_UTINYINT:
-          if (uVal > 255) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 1;
-          break;
-        case TSDB_DATA_TYPE_USMALLINT:
-          if (uVal > 65535) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 2;
-          break;
-        case TSDB_DATA_TYPE_UINT:
-          if (uVal > UINT32_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 4;
-          break;
-        case TSDB_DATA_TYPE_UBIGINT:
-          if (uVal > UINT64_MAX) {
-            return TSDB_CODE_TSC_INVALID_OPERATION;
-          }
-          pExpr->_node.pLeft->pVal->nLen = 8;
-          break;
-        }
-
-        pExpr->_node.pLeft->pSchema->type = schemaType;
-        pExpr->_node.pLeft->pVal->nType = schemaType;
-
-        pExpr->_node.pLeft->resultType = schemaType;
-        pExpr->_node.pLeft->resultBytes = tDataTypes[schemaType].bytes;
-      }
-
-      resultType = schemaType;
+    if (IS_FLOAT_TYPE(leftType) || IS_FLOAT_TYPE(rightType)) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
     }
-
-    if (resultType == TSDB_DATA_TYPE_BOOL) {
-      pExpr->resultType = TSDB_DATA_TYPE_BOOL;
-      pExpr->resultBytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
+    if (pExpr->_node.optr == TSDB_BINARY_OP_LSHIFT || pExpr->_node.optr == TSDB_BINARY_OP_RSHIFT) {
+      pExpr->resultType = leftType;
+      pExpr->resultBytes = tDataTypes[leftType].bytes;
     } else {
-      pExpr->resultType = resultType;
-      pExpr->resultBytes = tDataTypes[resultType].bytes;
+      pExpr->resultType = TSDB_DATA_TYPE_BIGINT;
+      pExpr->resultBytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
     }
     return TSDB_CODE_SUCCESS;
+  } else if (pExpr->_node.optr == TSDB_BINARY_OP_BITNOT) {
+    if (!IS_NUMERIC_TYPE(leftType) || IS_FLOAT_TYPE(leftType)) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    }
+    pExpr->resultType = leftType;
+    pExpr->resultBytes = tDataTypes[leftType].bytes;
+    return TSDB_CODE_SUCCESS;
   } else {
     return TSDB_CODE_SUCCESS;
   }
@@ -647,6 +506,9 @@ void exprTreeExprNodeTraverse(tExprNode *pExpr, int32_t numOfRows, tExprOperandI
 
     rightType = pRight->pSchema->type;
     rightNum = numOfRows;    
+  } else if (pRight->nodeType == TSQL_NODE_DUMMY) {
+    /* BITNOT */
+    rightNum = 0;
   } else {
     assert(pRight->nodeType == TSQL_NODE_VALUE);
     rightIn = (char *)&pRight->pVal->i64;
@@ -661,7 +523,10 @@ void exprTreeExprNodeTraverse(tExprNode *pExpr, int32_t numOfRows, tExprOperandI
   if(leftType == TSDB_DATA_TYPE_TIMESTAMP || rightType == TSDB_DATA_TYPE_TIMESTAMP) {
     output->type = TSDB_DATA_TYPE_BIGINT;
   } else {
-    if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND) {
+    if (pExpr->_node.optr == TSDB_BINARY_OP_BITAND || pExpr->_node.optr == TSDB_BINARY_OP_BITOR ||
+        pExpr->_node.optr == TSDB_BINARY_OP_BITXOR || pExpr->_node.optr == TSDB_BINARY_OP_BITNOT ||
+        pExpr->_node.optr == TSDB_BINARY_OP_LSHIFT || pExpr->_node.optr == TSDB_BINARY_OP_RSHIFT)
+    {
       output->type = leftType; // rightType must be the same as leftType
     } else {
       output->type = TSDB_DATA_TYPE_DOUBLE;

src/inc/taosdef.h

@@ -177,10 +177,15 @@ do { \
 #define TSDB_BINARY_OP_DIVIDE     33
 #define TSDB_BINARY_OP_REMAINDER  34
 #define TSDB_BINARY_OP_BITAND     35
+#define TSDB_BINARY_OP_BITOR      36
+#define TSDB_BINARY_OP_BITXOR     37
+#define TSDB_BINARY_OP_BITNOT     38
+#define TSDB_BINARY_OP_LSHIFT     39
+#define TSDB_BINARY_OP_RSHIFT     40
 
 
 #define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
-#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_BITAND))
+#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_RSHIFT))
 
 #define TS_PATH_DELIMITER_LEN     1
 

src/inc/ttokendef.h

@@ -50,172 +50,173 @@
 #define TK_LE                              32
 #define TK_BITAND                          33
 #define TK_BITOR                           34
-#define TK_LSHIFT                          35
-#define TK_RSHIFT                          36
-#define TK_PLUS                            37
-#define TK_MINUS                           38
-#define TK_DIVIDE                          39
-#define TK_TIMES                           40
-#define TK_STAR                            41
-#define TK_SLASH                           42
-#define TK_REM                             43
-#define TK_UMINUS                          44
-#define TK_UPLUS                           45
-#define TK_BITNOT                          46
-#define TK_ARROW                           47
-#define TK_SHOW                            48
-#define TK_DATABASES                       49
-#define TK_TOPICS                          50
-#define TK_FUNCTIONS                       51
-#define TK_MNODES                          52
-#define TK_DNODES                          53
-#define TK_ACCOUNTS                        54
-#define TK_USERS                           55
-#define TK_MODULES                         56
-#define TK_QUERIES                         57
-#define TK_CONNECTIONS                     58
-#define TK_STREAMS                         59
-#define TK_VARIABLES                       60
-#define TK_SCORES                          61
-#define TK_GRANTS                          62
-#define TK_VNODES                          63
-#define TK_DOT                             64
-#define TK_CREATE                          65
-#define TK_TABLE                           66
-#define TK_STABLE                          67
-#define TK_DATABASE                        68
-#define TK_TABLES                          69
-#define TK_STABLES                         70
-#define TK_VGROUPS                         71
-#define TK_DROP                            72
-#define TK_TOPIC                           73
-#define TK_FUNCTION                        74
-#define TK_DNODE                           75
-#define TK_USER                            76
-#define TK_ACCOUNT                         77
-#define TK_USE                             78
-#define TK_DESCRIBE                        79
-#define TK_DESC                            80
-#define TK_ALTER                           81
-#define TK_PASS                            82
-#define TK_PRIVILEGE                       83
-#define TK_LOCAL                           84
-#define TK_COMPACT                         85
-#define TK_LP                              86
-#define TK_RP                              87
-#define TK_IF                              88
-#define TK_EXISTS                          89
-#define TK_AS                              90
-#define TK_OUTPUTTYPE                      91
-#define TK_AGGREGATE                       92
-#define TK_BUFSIZE                         93
-#define TK_PPS                             94
-#define TK_TSERIES                         95
-#define TK_DBS                             96
-#define TK_STORAGE                         97
-#define TK_QTIME                           98
-#define TK_CONNS                           99
-#define TK_STATE                          100
-#define TK_COMMA                          101
-#define TK_KEEP                           102
-#define TK_CACHE                          103
-#define TK_REPLICA                        104
-#define TK_QUORUM                         105
-#define TK_DAYS                           106
-#define TK_MINROWS                        107
-#define TK_MAXROWS                        108
-#define TK_BLOCKS                         109
-#define TK_CTIME                          110
-#define TK_WAL                            111
-#define TK_FSYNC                          112
-#define TK_COMP                           113
-#define TK_PRECISION                      114
-#define TK_UPDATE                         115
-#define TK_CACHELAST                      116
-#define TK_PARTITIONS                     117
-#define TK_UNSIGNED                       118
-#define TK_TAGS                           119
-#define TK_USING                          120
-#define TK_NULL                           121
-#define TK_NOW                            122
-#define TK_VARIABLE                       123
-#define TK_SELECT                         124
-#define TK_UNION                          125
-#define TK_ALL                            126
-#define TK_DISTINCT                       127
-#define TK_FROM                           128
-#define TK_RANGE                          129
-#define TK_INTERVAL                       130
-#define TK_EVERY                          131
-#define TK_SESSION                        132
-#define TK_STATE_WINDOW                   133
-#define TK_FILL                           134
-#define TK_SLIDING                        135
-#define TK_ORDER                          136
-#define TK_BY                             137
-#define TK_ASC                            138
-#define TK_GROUP                          139
-#define TK_HAVING                         140
-#define TK_LIMIT                          141
-#define TK_OFFSET                         142
-#define TK_SLIMIT                         143
-#define TK_SOFFSET                        144
-#define TK_WHERE                          145
-#define TK_RESET                          146
-#define TK_QUERY                          147
-#define TK_SYNCDB                         148
-#define TK_ADD                            149
-#define TK_COLUMN                         150
-#define TK_MODIFY                         151
-#define TK_TAG                            152
-#define TK_CHANGE                         153
-#define TK_SET                            154
-#define TK_KILL                           155
-#define TK_CONNECTION                     156
-#define TK_STREAM                         157
-#define TK_COLON                          158
-#define TK_ABORT                          159
-#define TK_AFTER                          160
-#define TK_ATTACH                         161
-#define TK_BEFORE                         162
-#define TK_BEGIN                          163
-#define TK_CASCADE                        164
-#define TK_CLUSTER                        165
-#define TK_CONFLICT                       166
-#define TK_COPY                           167
-#define TK_DEFERRED                       168
-#define TK_DELIMITERS                     169
-#define TK_DETACH                         170
-#define TK_EACH                           171
-#define TK_END                            172
-#define TK_EXPLAIN                        173
-#define TK_FAIL                           174
-#define TK_FOR                            175
-#define TK_IGNORE                         176
-#define TK_IMMEDIATE                      177
-#define TK_INITIALLY                      178
-#define TK_INSTEAD                        179
-#define TK_KEY                            180
-#define TK_OF                             181
-#define TK_RAISE                          182
-#define TK_REPLACE                        183
-#define TK_RESTRICT                       184
-#define TK_ROW                            185
-#define TK_STATEMENT                      186
-#define TK_TRIGGER                        187
-#define TK_VIEW                           188
-#define TK_IPTOKEN                        189
-#define TK_SEMI                           190
-#define TK_NONE                           191
-#define TK_PREV                           192
-#define TK_LINEAR                         193
-#define TK_IMPORT                         194
-#define TK_TBNAME                         195
-#define TK_JOIN                           196
-#define TK_INSERT                         197
-#define TK_INTO                           198
-#define TK_VALUES                         199
-#define TK_FILE                           200
+#define TK_BITXOR                          35
+#define TK_LSHIFT                          36
+#define TK_RSHIFT                          37
+#define TK_PLUS                            38
+#define TK_MINUS                           39
+#define TK_DIVIDE                          40
+#define TK_TIMES                           41
+#define TK_STAR                            42
+#define TK_SLASH                           43
+#define TK_REM                             44
+#define TK_UMINUS                          45
+#define TK_UPLUS                           46
+#define TK_BITNOT                          47
+#define TK_ARROW                           48
+#define TK_SHOW                            49
+#define TK_DATABASES                       50
+#define TK_TOPICS                          51
+#define TK_FUNCTIONS                       52
+#define TK_MNODES                          53
+#define TK_DNODES                          54
+#define TK_ACCOUNTS                        55
+#define TK_USERS                           56
+#define TK_MODULES                         57
+#define TK_QUERIES                         58
+#define TK_CONNECTIONS                     59
+#define TK_STREAMS                         60
+#define TK_VARIABLES                       61
+#define TK_SCORES                          62
+#define TK_GRANTS                          63
+#define TK_VNODES                          64
+#define TK_DOT                             65
+#define TK_CREATE                          66
+#define TK_TABLE                           67
+#define TK_STABLE                          68
+#define TK_DATABASE                        69
+#define TK_TABLES                          70
+#define TK_STABLES                         71
+#define TK_VGROUPS                         72
+#define TK_DROP                            73
+#define TK_TOPIC                           74
+#define TK_FUNCTION                        75
+#define TK_DNODE                           76
+#define TK_USER                            77
+#define TK_ACCOUNT                         78
+#define TK_USE                             79
+#define TK_DESCRIBE                        80
+#define TK_DESC                            81
+#define TK_ALTER                           82
+#define TK_PASS                            83
+#define TK_PRIVILEGE                       84
+#define TK_LOCAL                           85
+#define TK_COMPACT                         86
+#define TK_LP                              87
+#define TK_RP                              88
+#define TK_IF                              89
+#define TK_EXISTS                          90
+#define TK_AS                              91
+#define TK_OUTPUTTYPE                      92
+#define TK_AGGREGATE                       93
+#define TK_BUFSIZE                         94
+#define TK_PPS                             95
+#define TK_TSERIES                         96
+#define TK_DBS                             97
+#define TK_STORAGE                         98
+#define TK_QTIME                           99
+#define TK_CONNS                          100
+#define TK_STATE                          101
+#define TK_COMMA                          102
+#define TK_KEEP                           103
+#define TK_CACHE                          104
+#define TK_REPLICA                        105
+#define TK_QUORUM                         106
+#define TK_DAYS                           107
+#define TK_MINROWS                        108
+#define TK_MAXROWS                        109
+#define TK_BLOCKS                         110
+#define TK_CTIME                          111
+#define TK_WAL                            112
+#define TK_FSYNC                          113
+#define TK_COMP                           114
+#define TK_PRECISION                      115
+#define TK_UPDATE                         116
+#define TK_CACHELAST                      117
+#define TK_PARTITIONS                     118
+#define TK_UNSIGNED                       119
+#define TK_TAGS                           120
+#define TK_USING                          121
+#define TK_NULL                           122
+#define TK_NOW                            123
+#define TK_VARIABLE                       124
+#define TK_SELECT                         125
+#define TK_UNION                          126
+#define TK_ALL                            127
+#define TK_DISTINCT                       128
+#define TK_FROM                           129
+#define TK_RANGE                          130
+#define TK_INTERVAL                       131
+#define TK_EVERY                          132
+#define TK_SESSION                        133
+#define TK_STATE_WINDOW                   134
+#define TK_FILL                           135
+#define TK_SLIDING                        136
+#define TK_ORDER                          137
+#define TK_BY                             138
+#define TK_ASC                            139
+#define TK_GROUP                          140
+#define TK_HAVING                         141
+#define TK_LIMIT                          142
+#define TK_OFFSET                         143
+#define TK_SLIMIT                         144
+#define TK_SOFFSET                        145
+#define TK_WHERE                          146
+#define TK_RESET                          147
+#define TK_QUERY                          148
+#define TK_SYNCDB                         149
+#define TK_ADD                            150
+#define TK_COLUMN                         151
+#define TK_MODIFY                         152
+#define TK_TAG                            153
+#define TK_CHANGE                         154
+#define TK_SET                            155
+#define TK_KILL                           156
+#define TK_CONNECTION                     157
+#define TK_STREAM                         158
+#define TK_COLON                          159
+#define TK_ABORT                          160
+#define TK_AFTER                          161
+#define TK_ATTACH                         162
+#define TK_BEFORE                         163
+#define TK_BEGIN                          164
+#define TK_CASCADE                        165
+#define TK_CLUSTER                        166
+#define TK_CONFLICT                       167
+#define TK_COPY                           168
+#define TK_DEFERRED                       169
+#define TK_DELIMITERS                     170
+#define TK_DETACH                         171
+#define TK_EACH                           172
+#define TK_END                            173
+#define TK_EXPLAIN                        174
+#define TK_FAIL                           175
+#define TK_FOR                            176
+#define TK_IGNORE                         177
+#define TK_IMMEDIATE                      178
+#define TK_INITIALLY                      179
+#define TK_INSTEAD                        180
+#define TK_KEY                            181
+#define TK_OF                             182
+#define TK_RAISE                          183
+#define TK_REPLACE                        184
+#define TK_RESTRICT                       185
+#define TK_ROW                            186
+#define TK_STATEMENT                      187
+#define TK_TRIGGER                        188
+#define TK_VIEW                           189
+#define TK_IPTOKEN                        190
+#define TK_SEMI                           191
+#define TK_NONE                           192
+#define TK_PREV                           193
+#define TK_LINEAR                         194
+#define TK_IMPORT                         195
+#define TK_TBNAME                         196
+#define TK_JOIN                           197
+#define TK_INSERT                         198
+#define TK_INTO                           199
+#define TK_VALUES                         200
+#define TK_FILE                           201
 
 
 #define TK_SPACE                          300

src/inc/ttype.h

@@ -289,6 +289,7 @@ void* getDataMax(int32_t type);
 int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned);
 
 #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL)
+#define SET_BIGINT_NULL(v) (*(int64_t *)(v) = TSDB_DATA_BIGINT_NULL)
 
 #ifdef __cplusplus
 }

src/kit/shell/src/shellEngine.c

@@ -244,22 +244,34 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
   int64_t   st, et;
   wordexp_t full_path;
   char *    sptr = NULL;
+  char *    tmp = NULL;
   char *    cptr = NULL;
   char *    fname = NULL;
   bool      printMode = false;
 
-  if ((sptr = tstrstr(command, ">>", true)) != NULL) {
-    cptr = tstrstr(command, ";", true);
-    if (cptr != NULL) {
-      *cptr = '\0';
-    }
+  sptr = command;
+  while ((sptr = tstrstr(sptr, ">>", true)) != NULL) {
+    // find the last ">>" if any
+    tmp = sptr;
+    sptr += 2;
+  }
 
-    if (wordexp(sptr + 2, &full_path, 0) != 0) {
-      fprintf(stderr, "ERROR: invalid filename: %s\n", sptr + 2);
-      return;
+  sptr = tmp;
+
+  if (sptr != NULL) {
+    if (regex_match(sptr + 2, "^\\s*[0-9]+\\s*[\\>|\\<|\\<=|\\>=|=|!=]\\s*.*;\\s*$", REG_EXTENDED | REG_ICASE) == 0) {
+      cptr = tstrstr(command, ";", true);
+      if (cptr != NULL) {
+        *cptr = '\0';
+      }
+
+      if (wordexp(sptr + 2, &full_path, 0) != 0) {
+        fprintf(stderr, "ERROR: invalid filename: %s\n", sptr + 2);
+        return;
+      }
+      *sptr = '\0';
+      fname = full_path.we_wordv[0];
     }
-    *sptr = '\0';
-    fname = full_path.we_wordv[0];
   }
 
   if ((sptr = tstrstr(command, "\\G", true)) != NULL) {

src/query/inc/qFilter.h

@@ -39,10 +39,11 @@ extern "C" {
 enum {
   FLD_TYPE_COLUMN = 1,
   FLD_TYPE_VALUE = 2,  
-  FLD_TYPE_MAX = 3,
-  FLD_DESC_NO_FREE = 4,
-  FLD_DATA_NO_FREE = 8,
-  FLD_DATA_IS_HASH = 16,
+  FLD_TYPE_EXPR = 3,
+  FLD_TYPE_MAX = 4,
+  FLD_DESC_NO_FREE = 16,
+  FLD_DATA_NO_FREE = 32,
+  FLD_DATA_IS_HASH = 64,
 };
 
 enum {
@@ -182,6 +183,7 @@ typedef struct SFilterGroupCtx {
   uint32_t         colNum;
   uint32_t        *colIdx;
   SFilterColInfo  *colInfo;
+  bool             hasExpr;
 } SFilterGroupCtx;
 
 typedef struct SFilterColCtx {
@@ -206,6 +208,8 @@ typedef struct SFilterComUnit {
   void *colData;
   void *valData;
   void *valData2;
+  void *expr;
+  void *exprData;
   uint16_t colId;
   uint16_t dataSize;
   uint8_t dataType;
@@ -289,7 +293,10 @@ typedef struct SFilterInfo {
 #define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
 #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
 #define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
-#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
+#define FILTER_GET_TYPE(fl) ((fl) & 0xF)
+#define FILTER_GET_FIELD_DESC(fi) ((fi)->desc)
+#define FILTER_GET_EXPR_TYPE(i, id) (((tExprNode*)(FILTER_GET_FIELD_DESC(FILTER_GET_FIELD(i, id))))->resultType)
+#define FILTER_GET_EXPR_SIZE(i, id) (((tExprNode*)(FILTER_GET_FIELD_DESC(FILTER_GET_FIELD(i, id))))->resultBytes)
 
 #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
 #define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left)

src/query/inc/sql.y

@@ -13,7 +13,7 @@
 %right NOT.
 %left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH CONTAINS GLOB BETWEEN IN.
 %left GT GE LT LE.
-%left BITAND BITOR LSHIFT RSHIFT.
+%left BITAND BITOR BITXOR LSHIFT RSHIFT.
 %left PLUS MINUS.
 %left DIVIDE TIMES.
 %left STAR SLASH REM.
@@ -787,6 +787,11 @@ expr(A) ::= expr(X) STAR  expr(Y).   {A = tSqlExprCreate(X, Y, TK_STAR);  }
 expr(A) ::= expr(X) SLASH expr(Y).   {A = tSqlExprCreate(X, Y, TK_DIVIDE);}
 expr(A) ::= expr(X) REM   expr(Y).   {A = tSqlExprCreate(X, Y, TK_REM);   }
 expr(A) ::= expr(X) BITAND expr(Y).  {A = tSqlExprCreate(X, Y, TK_BITAND);}
+expr(A) ::= expr(X) BITOR expr(Y).   {A = tSqlExprCreate(X, Y, TK_BITOR); }
+expr(A) ::= expr(X) BITXOR expr(Y).  {A = tSqlExprCreate(X, Y, TK_BITXOR);}
+expr(A) ::= BITNOT expr(X).          {A = tSqlExprCreate(X, NULL, TK_BITNOT);}
+expr(A) ::= expr(X) LSHIFT expr(Y).  {A = tSqlExprCreate(X, Y, TK_LSHIFT);}
+expr(A) ::= expr(X) RSHIFT expr(Y).  {A = tSqlExprCreate(X, Y, TK_RSHIFT);}
 
 // like expression
 expr(A) ::= expr(X) LIKE expr(Y).    {A = tSqlExprCreate(X, Y, TK_LIKE);  }

src/query/src/qExecutor.c

@@ -3155,6 +3155,10 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi
 FORCE_INLINE int32_t getColumnDataFromId(void *param, int32_t id, void **data) {
   int32_t numOfCols = ((SColumnDataParam *)param)->numOfCols;
   SArray* pDataBlock = ((SColumnDataParam *)param)->pDataBlock;
+  if (id == INT32_MAX) {
+    *data = pDataBlock;
+    return TSDB_CODE_SUCCESS;
+  }
   
   for (int32_t j = 0; j < numOfCols; ++j) {
     SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j);

src/query/src/qFilter.c

@@ -54,11 +54,16 @@ static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const v
   return tVariantCompare(val1, val2);
 }
 
+static FORCE_INLINE int32_t filterExprCompare(const void *desc1, const void *desc2) {
+  return -1;
+}
+
 
 filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = {
   NULL,
   filterFieldColDescCompare,
-  filterFieldValDescCompare
+  filterFieldValDescCompare,
+  filterExprCompare
 };
 
 bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
@@ -833,7 +838,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type,
     info->fields[type].fields[idx].desc = desc;
     info->fields[type].fields[idx].data = data ? *data : NULL;
 
-    if (type == FLD_TYPE_COLUMN) {
+    if (type == FLD_TYPE_COLUMN || type == FLD_TYPE_EXPR) {
       FILTER_SET_FLAG(info->fields[type].fields[idx].flag, FLD_DATA_NO_FREE);
     }
 
@@ -873,7 +878,8 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte
 
 int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) {
   CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node");
-  CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR);
+  CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE
+          &&node->nodeType != TSQL_NODE_EXPR, TSDB_CODE_QRY_APP_ERROR);
 
   int32_t type;
   void *v;
@@ -882,6 +888,9 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI
     type = FLD_TYPE_COLUMN;
     v = node->pSchema;
     node->pSchema = NULL;
+  } else if (node->nodeType == TSQL_NODE_EXPR) {
+    type = FLD_TYPE_EXPR;
+    v = node;
   } else {
     type = FLD_TYPE_VALUE;
     v = node->pVal;
@@ -933,9 +942,11 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
   }
   
   SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u);
-  assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN));
-  
-  info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col);
+  if (FILTER_GET_TYPE(col->flag) == FLD_TYPE_COLUMN) {
+    info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col);
+  } else {
+    info->units[info->unitNum].compare.type = (uint8_t)FILTER_GET_EXPR_TYPE(info, u->left);
+  }
 
   *uidx = info->unitNum;
 
@@ -1188,7 +1199,7 @@ static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pL
     jsonKeyMd5((*pLeft)->_node.pRight->pVal->pz, (*pLeft)->_node.pRight->pVal->nLen, keyMd5);
     memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
     (*pLeft) = (*pLeft)->_node.pLeft;   // -> operation use left as input
-  }else if(((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) &&
+  }else if((*pLeft)->nodeType == TSQL_NODE_COL && ((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) &&
       (tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL)){
     SSchema* schema = (*pLeft)->pSchema;
     char    keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
@@ -1212,7 +1223,16 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
   if((ret = filterDealJson(info, tree, &pLeft)) != TSDB_CODE_SUCCESS) return ret;
   SFilterFieldId left = {0}, right = {0};
   filterAddFieldFromNode(info, pLeft, &left);
-  uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
+  if (pLeft->nodeType != TSQL_NODE_VALUE && pLeft->nodeType != TSQL_NODE_COL) {
+    tree->_node.pLeft = NULL;
+  }
+
+  uint8_t type;
+  if (left.type == FLD_TYPE_EXPR) {
+    type = (uint8_t)FILTER_GET_EXPR_TYPE(info, left);
+  } else {
+    type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
+  }
   int32_t len = 0;
   uint32_t uidx = 0;
 
@@ -1576,6 +1596,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
         char str[512] = {0};
         
         SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
+        if (FILTER_GET_TYPE(left->flag) == FLD_TYPE_EXPR) {
+          continue;
+        }
         SSchema *sch = left->desc;
         if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_CONTAINS){
           len = sprintf(str, "UNIT[%d] => [%d][%s]  %s  [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str);
@@ -1751,6 +1774,9 @@ void filterFreeField(SFilterField* field, int32_t type) {
   if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) {
     if (type == FLD_TYPE_VALUE) {
       tVariantDestroy(field->desc);
+    } else if (type == FLD_TYPE_EXPR) {
+      tExprTreeDestroy(field->desc, NULL);
+      field->desc = NULL;
     }
     
     tfree(field->desc);
@@ -2100,20 +2126,30 @@ _return:
 
 int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) {
   bool empty = false;
+  if (info->fields[FLD_TYPE_COLUMN].num == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
   uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t));
   uint32_t colIdxi = 0;
   uint32_t gResIdx = 0;
+  bool hasExpr = false;
   
   for (uint32_t i = 0; i < info->groupNum; ++i) {
     SFilterGroup* g = info->groups + i;
 
     gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx));
     gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo));
+    gRes[gResIdx]->hasExpr = false;
     colIdxi = 0;
     empty = false;
     
     for (uint32_t j = 0; j < g->unitNum; ++j) {
       SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j);
+      if(u->left.type == FLD_TYPE_EXPR) {
+        gRes[gResIdx]->hasExpr = true;
+        hasExpr = true;
+        continue;
+      }
       uint32_t cidx = FILTER_UNIT_COL_IDX(u);
 
       if (gRes[gResIdx]->colInfo[cidx].info == NULL) {
@@ -2160,6 +2196,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
     ++gResIdx;
   }
 
+  if (hasExpr) {
+    FILTER_CLR_FLAG(info->status, FI_STATUS_REWRITE);
+  }
   tfree(colIdx);
 
   *gResNum = gResIdx;
@@ -2174,6 +2213,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
 void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) {
   uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
   bool equal = false;
+  if (gRes1->hasExpr || gRes2->hasExpr) {
+    *conflict = true;
+    return;
+  }
   
   for (; m < gRes1->colNum; ++m) {
     idx1 = gRes1->colIdx[m];
@@ -2646,7 +2689,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
     info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2);
     info->cunits[i].optr = FILTER_UNIT_OPTR(unit);
     info->cunits[i].colData = NULL;
-    info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
+    info->cunits[i].expr = NULL;
+    if (unit->left.type == FLD_TYPE_COLUMN) {
+      info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
+    } else if (unit->left.type == FLD_TYPE_EXPR) {
+      info->cunits[i].expr = FILTER_GET_FIELD_DESC(FILTER_GET_FIELD(info, unit->left));
+    }
     
     if (unit->right.type == FLD_TYPE_VALUE) {
       if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){   // json value is tVariant
@@ -2662,8 +2710,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
     } else {
       info->cunits[i].valData2 = info->cunits[i].valData;
     }
-    
-    info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit);
+
+    if (unit->left.type == FLD_TYPE_COLUMN) {
+      info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit);
+    } else {
+      info->cunits[i].dataSize = FILTER_GET_EXPR_SIZE(info, unit->left);
+    }
     info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit);
   }
   
@@ -2673,8 +2725,14 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
 int32_t filterUpdateComUnits(SFilterInfo *info) {
   for (uint32_t i = 0; i < info->unitNum; ++i) {
     SFilterUnit *unit = &info->units[i];
-
-    info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
+    if (unit->left.type == FLD_TYPE_EXPR) {
+      SFilterField *t = FILTER_UNIT_LEFT_FIELD(info, unit);
+      info->cunits[i].colData = NULL;
+      info->cunits[i].exprData = t->data;
+    } else {
+      info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
+      info->cunits[i].exprData = NULL;
+    }
   }
 
   return TSDB_CODE_SUCCESS;
@@ -2923,6 +2981,11 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p,
 
 
 int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols, bool* all) {
+  for (uint32_t i = 0; i < info->unitNum; ++i) {
+    if(info->cunits[i].expr) {
+      return 1;
+    }
+  }
   if (statis && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) {    
     info->blkFlag = 0;
     
@@ -2953,6 +3016,20 @@ _return:
   return TSDB_CODE_SUCCESS;
 }
 
+char *getExprColumnData(void *param, const char* name, int32_t colId) {
+  void *data = NULL;
+  getColumnDataFromId(param, colId, &data);
+  return (char *) data;
+}
+
+void* filterExprTraverse (SFilterInfo *info, int32_t numOfRows, int16_t numOfCols) {
+  tExprOperandInfo output;
+  output.data = malloc(sizeof(int64_t) * numOfRows);
+  SSDataBlock* pBlock = (SSDataBlock*) info->cunits[0].exprData;
+  SColumnDataParam param = {.numOfCols = numOfCols, .pDataBlock = (SArray*)pBlock};
+  exprTreeNodeTraverse(info->cunits[0].expr, numOfRows, &output, &param, TSDB_ORDER_ASC, getExprColumnData);
+  return output.data;
+}
 
 static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
   return true;
@@ -2963,18 +3040,26 @@ static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, i
 static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
   SFilterInfo *info = (SFilterInfo *)pinfo;
   bool all = true;
+  char *exprData = NULL;
+  uint32_t uidx = info->groups[0].unitIdxs[0];
+  void *colData = NULL;
 
   if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
     return all;
   }
 
+  if (info->cunits[0].expr) {
+    exprData = filterExprTraverse(info, numOfRows, numOfCols);
+  } else {
+    exprData = info->cunits[uidx].colData;
+  }
+
   if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
   }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint32_t uidx = info->groups[0].unitIdxs[0];
-    void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
+    colData = (char *)exprData + info->cunits[uidx].dataSize * i;
     if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
       if (!colData){  // for json->'key' is null
         (*p)[i] = 1;
@@ -2992,23 +3077,35 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
     }    
   }
 
+  if (info->cunits[0].expr) {
+    tfree(exprData);
+  }
+
   return all;
 }
 static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
   SFilterInfo *info = (SFilterInfo *)pinfo;
   bool all = true;
+  char *exprData = NULL;
+  uint32_t uidx = info->groups[0].unitIdxs[0];
+  void *colData = NULL;
 
   if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
     return all;
   }
 
+  if (info->cunits[0].expr) {
+    exprData = filterExprTraverse(info, numOfRows, numOfCols);
+  } else {
+    exprData = info->cunits[uidx].colData;
+  }
+
   if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
   }
   
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint32_t uidx = info->groups[0].unitIdxs[0];
-    void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
+    colData = (char *)exprData + info->cunits[uidx].dataSize * i;
 
     if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
       if (!colData) {   // for json->'key' is not null
@@ -3028,6 +3125,10 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
     }
   }
 
+  if (info->cunits[0].expr) {
+    tfree(exprData);
+  }
+
   return all;
 }
 
@@ -3081,11 +3182,16 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataSta
   void *valData = info->cunits[0].valData;
   void *valData2 = info->cunits[0].valData2;
   __compar_fn_t func = gDataCompare[info->cunits[0].func];
+  char *exprData = NULL;
 
   if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
     return all;
   }
 
+  if (info->cunits[0].expr) {
+    exprData = colData = filterExprTraverse(info, numOfRows, numOfCols);
+  }
+
   if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
   }
@@ -3105,6 +3211,9 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataSta
     
     colData += dataSize;
   }
+  if (info->cunits[0].expr) {
+    tfree(exprData);
+  }
 
   return all;
 }
@@ -3112,18 +3221,26 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataSta
 bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
   SFilterInfo *info = (SFilterInfo *)pinfo;
   bool all = true;
+  char *exprData = NULL;
+  uint32_t uidx = info->groups[0].unitIdxs[0];
+  void *colData = NULL;
 
   if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
     return all;
   }
   
+  if (info->cunits[0].expr) {
+    exprData = filterExprTraverse(info, numOfRows, numOfCols);
+  } else {
+    exprData = info->cunits[uidx].colData;
+  }
+
   if (*p == NULL) {
     *p = calloc(numOfRows, sizeof(int8_t));
   }
-  
+
   for (int32_t i = 0; i < numOfRows; ++i) {
-    uint32_t uidx = info->groups[0].unitIdxs[0];
-    void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
+    colData = (char *)exprData + info->cunits[uidx].dataSize * i;
     if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) {
       (*p)[i] = 0;
       all = false;
@@ -3152,6 +3269,10 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SDataStat
     }
   }
 
+  if (info->cunits[0].expr) {
+    tfree(exprData);
+  }
+
   return all;
 }
 
@@ -3167,6 +3288,27 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *
     *p = calloc(numOfRows, sizeof(int8_t));
   }
   
+  SArray* tmpData = NULL;
+  for (uint32_t g = 0; g < info->groupNum; ++g) {
+    SFilterGroup *group = &info->groups[g];
+    for (uint32_t u = 0; u < group->unitNum; ++u) {
+      uint32_t uidx = group->unitIdxs[u];
+      SFilterComUnit *cunit = &info->cunits[uidx];
+      if (cunit->expr) {
+        if (!tmpData) {
+          tmpData = taosArrayInit(10, POINTER_BYTES);
+        }
+        tExprOperandInfo output;
+        output.data = malloc(sizeof(int64_t) * numOfRows);
+        taosArrayPush(tmpData, output.data);
+        SSDataBlock* pBlock = (SSDataBlock*) cunit->exprData;
+        SColumnDataParam param = {.numOfCols = numOfCols, .pDataBlock = (SArray*)pBlock};
+        exprTreeNodeTraverse(cunit->expr, numOfRows, &output, &param, TSDB_ORDER_ASC, getExprColumnData);
+        cunit->colData = (char *) output.data;
+      }
+    }
+  }
+
   for (int32_t i = 0; i < numOfRows; ++i) {
     //FILTER_UNIT_CLR_F(info);
   
@@ -3228,6 +3370,9 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *
     }    
   }
 
+  if (tmpData) {
+    taosArrayDestroy(&tmpData);
+  }
   return all;
 }
 
@@ -3317,7 +3462,8 @@ _return:
 
 int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
   CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
-  CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
+  CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0 && info->fields[FLD_TYPE_EXPR].num <= 0,
+           TSDB_CODE_QRY_APP_ERROR, "no column fileds");
 
   if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
     return TSDB_CODE_SUCCESS;
@@ -3330,6 +3476,11 @@ int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from
     (*fp)(param, sch->colId, &fi->data);
   }
 
+   for (uint32_t i = 0; i < info->fields[FLD_TYPE_EXPR].num; ++i) {
+     SFilterField* fi = &info->fields[FLD_TYPE_EXPR].fields[i];
+     (*fp)(param, INT32_MAX, &fi->data);
+   }
+
   filterUpdateComUnits(info);
 
   return TSDB_CODE_SUCCESS;

src/query/src/qSqlParser.c

@@ -321,7 +321,9 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
   if ((pLeft != NULL && pRight != NULL) &&
       (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM ||
        optrType == TK_EQ || optrType == TK_NE || optrType == TK_LT || optrType == TK_GT || optrType == TK_LE || optrType == TK_GE ||
-       optrType == TK_AND || optrType == TK_OR)) {
+       optrType == TK_AND || optrType == TK_OR || optrType == TK_BITAND || optrType == TK_BITOR || optrType == TK_BITXOR ||
+       optrType == TK_LSHIFT || optrType == TK_RSHIFT))
+  {
     /*
      * if a exprToken is noted as the TK_TIMESTAMP, the time precision is microsecond
      * Otherwise, the time precision is adaptive, determined by the time precision from databases.
@@ -398,6 +400,61 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
           pExpr->value.i64 = (pLeft->value.i64 || pRight->value.i64) ? 1 : 0;
           break;
         }
+        case TK_BITAND: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 & pRight->value.i64;
+          }
+          break;
+        }
+        case TK_BITOR: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 | pRight->value.i64;
+          }
+          break;
+        }
+        case TK_BITXOR: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 ^ pRight->value.i64;
+          }
+          break;
+        }
+        case TK_LSHIFT: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 << pRight->value.i64;
+          }
+          break;
+        }
+        case TK_RSHIFT: {
+          if (pLeft->tokenId == TK_BOOL || pRight->tokenId == TK_BOOL ||
+              pLeft->tokenId == TK_FLOAT || pRight->tokenId == TK_FLOAT ||
+              pLeft->tokenId == TK_TIMESTAMP || pRight->tokenId == TK_TIMESTAMP)
+          {
+            pExpr->value.i64 = TSDB_DATA_BIGINT_NULL;
+          } else {
+            pExpr->value.i64 = pLeft->value.i64 >> pRight->value.i64;
+          }
+          break;
+        }
       }
 
       tSqlExprDestroy(pLeft);
@@ -509,6 +566,13 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
           pExpr->value.i64 = (left || right) ? 1 : 0;
           break;
         }
+        case TK_BITAND:
+        case TK_BITOR:
+        case TK_BITXOR:
+        case TK_LSHIFT:
+        case TK_RSHIFT:
+          pExpr->value.i64 = TSDB_DATA_DOUBLE_NULL;
+          break;
       }
 
       tSqlExprDestroy(pLeft);
@@ -542,6 +606,12 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
     }
 
     pExpr->pRight = pRight;
+
+    if (optrType == TK_BITNOT) {
+      pExpr->exprToken.z = pLeft->exprToken.z - 1;
+      pExpr->exprToken.n = pLeft->exprToken.n + 1;
+      pExpr->exprToken.type = pLeft->exprToken.type;
+    }
   }
 
   return pExpr;

src/tsdb/src/tsdbRead.c

@@ -173,6 +173,11 @@ typedef struct SRange {
   int32_t to;
 } SRange;
 
+typedef struct STagBlockInfo {
+  SSkipListNode *pSkipListNode;
+  SArray *pBlock;
+} STagBlockInfo;
+
 static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList);
 static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList);
 static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle);
@@ -4334,7 +4339,15 @@ static FORCE_INLINE int32_t tsdbGetTagDataFromId(void *param, int32_t id, void *
   return TSDB_CODE_SUCCESS;
 }
 
-
+static FORCE_INLINE int32_t tsdbGetTagData(void *param, int32_t id, void **data) {
+  STagBlockInfo* pInfo = (STagBlockInfo*) param;
+  if (id == INT32_MAX) {
+   *data = pInfo->pBlock;
+  } else {
+    return tsdbGetTagDataFromId(pInfo->pSkipListNode, id, data);
+  }
+  return TSDB_CODE_SUCCESS;
+}
 
 static void queryIndexedColumn(SSkipList* pSkipList, void* filterInfo, SArray* res) {
   SSkipListIterator* iter = NULL;
@@ -4389,26 +4402,77 @@ static void queryIndexedColumn(SSkipList* pSkipList, void* filterInfo, SArray* r
   tsdbDebug("filter index column end");
 }
 
+static void getAllExprColId(tExprNode* pExpr, SArray* array) {
+  if (!pExpr) {
+    return;
+  }
+  if (pExpr->nodeType == TSQL_NODE_FUNC) {
+    for (int32_t i = 0; i < pExpr->_func.numChildren; ++i) {
+      getAllExprColId(pExpr->_func.pChildren[i], array);
+    }
+  } else if (pExpr->nodeType == TSQL_NODE_EXPR) {
+    getAllExprColId(pExpr->_node.pLeft, array);
+    getAllExprColId(pExpr->_node.pRight, array);
+  } else if (pExpr->nodeType == TSQL_NODE_COL) {
+    taosArrayPush(array, &pExpr->pSchema->colId);
+  }
+}
+
+static void getAllFilterExprColId(SFilterFields* pSf, SArray* array) {
+  for (uint32_t i = 0; i < pSf->num; ++i) {
+    SFilterField* fi = &(pSf->fields[i]);
+    if (FILTER_GET_TYPE(fi->flag) == FLD_TYPE_EXPR) {
+      getAllExprColId(fi->desc, array);
+    }
+  }
+  taosArraySort(array, getComparFunc(TSDB_DATA_TYPE_SMALLINT, 0));
+  taosArrayRemoveDuplicate(array, getComparFunc(TSDB_DATA_TYPE_SMALLINT, 0), NULL);
+}
+
 static void queryIndexlessColumn(SSkipList* pSkipList, void* filterInfo, SArray* res) {
   SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
   int8_t *addToResult = NULL;
+  SFilterInfo *sfInfo = (SFilterInfo *)filterInfo;
+  SArray *array = NULL;
+  SArray *pDataBlock = NULL;
 
+  if (sfInfo->fields[FLD_TYPE_EXPR].num > 0) {
+    array = taosArrayInit(10, sizeof(int16_t));
+    getAllFilterExprColId(&(sfInfo->fields[FLD_TYPE_EXPR]), array);
+  }
   while (tSkipListIterNext(iter)) {
 
     SSkipListNode *pNode = tSkipListIterGet(iter);
 
-    filterSetColFieldData(filterInfo, pNode, tsdbGetTagDataFromId);
 
-    char *pData = SL_GET_NODE_DATA(pNode);
+    if (sfInfo->fields[FLD_TYPE_EXPR].num > 0) {
+      pDataBlock = taosArrayInit(10, sizeof(SColumnInfoData));
+      size_t num = taosArrayGetSize(array);
+      for(int32_t i = 0; i < num; ++i) {
+        int16_t *pColId = taosArrayGet(array, i);
+        void *data = NULL;
+        tsdbGetTagDataFromId(pNode, *pColId, &data);
+        SColumnInfoData colData = {{0}};
+        colData.pData = data;
+        colData.info.colId = *pColId;
+        taosArrayPush(pDataBlock, &colData);
+      }
+    }
+    STagBlockInfo stInfo = {.pSkipListNode = pNode, .pBlock = pDataBlock};
+    filterSetColFieldData(filterInfo, &stInfo, tsdbGetTagData);
 
-    bool all = filterExecute(filterInfo, 1, &addToResult, NULL, 0);
+    char *pData = SL_GET_NODE_DATA(pNode);
+    int16_t numOfCols = array ? (int16_t)taosArrayGetSize(array) : 0;
+    bool all = filterExecute(filterInfo, 1, &addToResult, NULL, numOfCols);
 
     if (all || (addToResult && *addToResult)) {
       STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL};
       taosArrayPush(res, &info);
     }
+    taosArrayDestroy(&pDataBlock);
   }
 
+  taosArrayDestroy(&array);
   tfree(addToResult);
 
   tSkipListDestroyIter(iter);

src/util/src/ttokenizer.c

@@ -413,6 +413,10 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) {
       *tokenId = TK_BITNOT;
       return 1;
     }
+    case '^': {
+      *tokenId = TK_BITXOR;
+      return 1;
+    }
     case '?': {
       *tokenId = TK_QUESTION;
       return 1;