Merge branch '3.0' into feature/3_liaohj

include/util/tcompare.h

@@ -105,8 +105,6 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
 
-int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
-
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char *a, const char *b, int32_t type, size_t size);

source/libs/index/inc/indexInt.h

@@ -35,15 +35,15 @@ extern "C" {
 #endif
 
 // clang-format off
-#define indexFatal(...) do { if (idxDebugFlag & DEBUG_FATAL) {  taosPrintLog("INDEX FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0)
-#define indexError(...) do { if (idxDebugFlag & DEBUG_ERROR) {  taosPrintLog("INDEX ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0)
-#define indexWarn(...)  do { if (idxDebugFlag & DEBUG_WARN)  {  taosPrintLog("INDEX WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0)
-#define indexInfo(...)  do { if (idxDebugFlag & DEBUG_INFO)  { taosPrintLog("INDEX ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0)
-#define indexDebug(...) do { if (idxDebugFlag & DEBUG_DEBUG) { taosPrintLog("INDEX ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__);} } while (0)
-#define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("INDEX ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__);} } while (0)
+#define indexFatal(...) do { if (idxDebugFlag & DEBUG_FATAL) {  taosPrintLog("IDX FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0)
+#define indexError(...) do { if (idxDebugFlag & DEBUG_ERROR) {  taosPrintLog("IDX ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0)
+#define indexWarn(...)  do { if (idxDebugFlag & DEBUG_WARN)  {  taosPrintLog("IDX WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0)
+#define indexInfo(...)  do { if (idxDebugFlag & DEBUG_INFO)  { taosPrintLog("IDX ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0)
+#define indexDebug(...) do { if (idxDebugFlag & DEBUG_DEBUG) { taosPrintLog("IDX ", DEBUG_DEBUG, idxDebugFlag, __VA_ARGS__);} } while (0)
+#define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0)
 // clang-format on
 
-typedef enum { LT, LE, GT, GE } RangeType;
+typedef enum { LT, LE, GT, GE, CONTAINS } RangeType;
 typedef enum { kTypeValue, kTypeDeletion } STermValueType;
 
 typedef struct SIndexStat {

source/libs/index/src/indexCache.c

@@ -90,7 +90,7 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe
       break;
     }
     CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
-    if (0 == strcmp(c->colVal, pCt->colVal)) {
+    if (0 == strcmp(c->colVal, pCt->colVal) && strlen(pCt->colVal) == strlen(c->colVal)) {
       if (c->operaType == ADD_VALUE) {
         INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid)
         // taosArrayPush(result, &c->uid);
@@ -222,7 +222,7 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr
   return TSDB_CODE_SUCCESS;
 }
 static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
-  return TSDB_CODE_SUCCESS;
+  return cacheSearchCompareFunc_JSON(cache, term, tr, s, CONTAINS);
 }
 static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return TSDB_CODE_SUCCESS;
@@ -242,6 +242,9 @@ static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTR
 static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return cacheSearchCompareFunc_JSON(cache, term, tr, s, GE);
 }
+static int32_t cacheSearchContain_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
+  return cacheSearchCompareFunc_JSON(cache, term, tr, s, CONTAINS);
+}
 static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) {
   return TSDB_CODE_SUCCESS;
 }

source/libs/index/src/indexComm.c

@@ -97,6 +97,11 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
   __compar_fn_t func = indexGetCompar(type);
   return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
 }
+
+static TExeCond tCompareContains(void* a, void* b, int8_t type) {
+  __compar_fn_t func = indexGetCompar(type);
+  return tCompare(func, QUERY_TERM, a, b, type);
+}
 TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
   if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) {
     return tDoCompare(func, cmptype, a, b);
@@ -185,12 +190,14 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
     case QUERY_TERM: {
       if (ret == 0) return MATCH;
     }
+    default:
+      return BREAK;
   }
   return CONTINUE;
 }
 
-static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual,
-                                                                    tCompareGreaterThan, tCompareGreaterEqual};
+static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {
+    tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual, tCompareContains};
 
 _cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; }
 

source/libs/index/src/indexFilter.c

@@ -64,6 +64,8 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
     *dst = QUERY_TERM;
   } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) {
     *dst = QUERY_REGEX;
+  } else if (src == OP_TYPE_JSON_CONTAINS) {
+    *dst = QUERY_PREFIX;
   } else {
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
@@ -186,6 +188,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
       SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
       param->colId = -1;
       param->colValType = (uint8_t)(vn->node.resType.type);
+      memcpy(param->colName, vn->literal, strlen(vn->literal));
       break;
     }
     case QUERY_NODE_COLUMN: {
@@ -237,7 +240,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
     indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight);
     SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
-  if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) {
+  if (node->opType == OP_TYPE_JSON_GET_VALUE) {
     return code;
   }
   SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam));
@@ -420,8 +423,8 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output
   return sifDoIndex(left, right, id, output);
 }
 static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) {
-  // return 0
-  return 0;
+  int id = OP_TYPE_JSON_CONTAINS;
+  return sifDoIndex(left, right, id, output);
 }
 static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) {
   // return 0
@@ -501,9 +504,11 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
   int32_t code = 0;
   int32_t nParam = sifGetOperParamNum(node->opType);
   if (nParam <= 1) {
-    SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
+    output->status = SFLT_NOT_INDEX;
+    return code;
+    // SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
-  if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) {
+  if (node->opType == OP_TYPE_JSON_GET_VALUE) {
     return code;
   }
   SIFParam *params = NULL;
@@ -617,11 +622,11 @@ EDealRes sifCalcWalker(SNode *node, void *context) {
   }
 
   if (QUERY_NODE_OPERATOR == nodeType(node)) {
-    indexInfo("node type for index filter, type: %d", nodeType(node));
+    // indexInfo("node type for index filter, type: %d", nodeType(node));
     return sifWalkOper(node, ctx);
   }
 
-  indexError("invalid node type for index filter calculating, type:%d", nodeType(node));
+  // indexError("invalid node type for index filter calculating, type:%d", nodeType(node));
   ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
   return DEAL_RES_ERROR;
 }

source/libs/index/src/indexTfile.c

@@ -425,8 +425,7 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   return TSDB_CODE_SUCCESS;
 }
 static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+  return tfSearchCompareFunc_JSON(reader, tem, tr, CONTAINS);
 }
 static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
   // impl later
@@ -466,10 +465,6 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
   AutomationCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
   FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
 
-  // FstSlice h = fstSliceCreate((uint8_t*)p, skip);
-  // fstStreamBuilderSetRange(sb, &h, ctype);
-  // fstSliceDestroy(&h);
-
   StreamWithState*       st = streamBuilderIntoStream(sb);
   StreamWithStateResult* rt = NULL;
   while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {

source/libs/parser/src/parTokenizer.c

@@ -64,6 +64,7 @@ static SKeyword keywordTable[] = {
     {"CONSUMER",      TK_CONSUMER},
     {"COUNT",         TK_COUNT},
     {"CREATE",        TK_CREATE},
+    {"CONTAINS",      TK_CONTAINS}, 
     {"DATABASE",      TK_DATABASE},
     {"DATABASES",     TK_DATABASES},
     {"DAYS",          TK_DAYS},

source/libs/parser/src/parTranslater.c

@@ -847,8 +847,12 @@ static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pO
   if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
   }
-  pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
-  pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
+  if(pOp->opType == OP_TYPE_JSON_GET_VALUE){
+    pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
+  }else if(pOp->opType == OP_TYPE_JSON_CONTAINS){
+    pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
+  }
+  pOp->node.resType.bytes = tDataTypes[pOp->node.resType.type].bytes;
   return DEAL_RES_CONTINUE;
 }
 

source/libs/scalar/src/filter.c

@@ -168,7 +168,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val
   compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
   setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, 
   compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
-  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey
+  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch
 };
 
 int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {

source/libs/scalar/src/sclvector.c

@@ -88,7 +88,7 @@ void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int
   }
 }
 
-void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
+void convertNcharToDouble(const void *inData, void *outData){
   char *tmp = taosMemoryMalloc(varDataTLen(inData));
   int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
   if (len < 0) {
@@ -97,13 +97,24 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int
 
   tmp[len] = 0;
 
-  ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
   double value = taosStr2Double(tmp, NULL);
 
   *((double *)outData) = value;
   taosMemoryFreeClear(tmp);
 }
 
+void convertBinaryToDouble(const void *inData, void *outData){
+  char *tmp = taosMemoryCalloc(1, varDataTLen(inData));
+  if(tmp == NULL){
+    *((double *)outData) = 0.;
+    return;
+  }
+  memcpy(tmp, varDataVal(inData), varDataLen(inData));
+  double ret = taosStr2Double(tmp, NULL);
+  taosMemoryFree(tmp);
+  *((double *)outData) = ret;
+}
+
 typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index);
 
 int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) {
@@ -147,7 +158,7 @@ int64_t getVectorBigintValue_JSON(void *src, int32_t index){
   if (*data == TSDB_DATA_TYPE_NULL){
     return 0;
   } else if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
-    convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+    convertNcharToDouble(data+CHAR_BYTES, &out);
   } else {
     convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
   }
@@ -445,14 +456,30 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){
   if (*data == TSDB_DATA_TYPE_NULL){
     return out;
   } else if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
-    convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+    convertNcharToDouble(data+CHAR_BYTES, &out);
   } else {
     convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
   }
   return out;
 }
 
-bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
+void* ncharTobinary(void *buf){            // todo need to remove , if tobinary is nchar
+  int32_t inputLen = varDataLen(buf);
+
+  void* t = taosMemoryCalloc(1, inputLen);
+  int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t));
+  if (len < 0) {
+    sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+             (char*)varDataVal(buf));
+    taosMemoryFree(t);
+    return NULL;
+  }
+  varDataSetLen(t, len);
+  return t;
+}
+
+bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData,
+                      void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, bool *freeRight){
   if(optr == OP_TYPE_JSON_CONTAINS) {
     return true;
   }
@@ -489,21 +516,41 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
 
   *fp = filterGetCompFunc(type, optr);
 
-  if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
-    convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type);
-    *pLeftData = pLeftOut;
-  } else if(typeLeft != type) {
-    convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
-    *pLeftData = pLeftOut;
+  if(IS_NUMERIC_TYPE(type) || IS_FLOAT_TYPE(type)){
+    if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
+      convertNcharToDouble(*pLeftData, pLeftOut);
+      *pLeftData = pLeftOut;
+    } else if(typeLeft == TSDB_DATA_TYPE_BINARY) {
+      convertBinaryToDouble(*pLeftData, pLeftOut);
+      *pLeftData = pLeftOut;
+    } else if(typeLeft != type) {
+      convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
+      *pLeftData = pLeftOut;
+    }
+
+    if(typeRight == TSDB_DATA_TYPE_NCHAR) {
+      convertNcharToDouble(*pRightData, pRightOut);
+      *pRightData = pRightOut;
+    } else if(typeRight == TSDB_DATA_TYPE_BINARY) {
+      convertBinaryToDouble(*pRightData, pRightOut);
+      *pRightData = pRightOut;
+    } else if(typeRight != type) {
+      convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
+      *pRightData = pRightOut;
+    }
+  }else if(type == TSDB_DATA_TYPE_BINARY){
+    if(typeLeft == TSDB_DATA_TYPE_NCHAR){
+      *pLeftData = ncharTobinary(*pLeftData);
+      *freeLeft = true;
+    }
+    if(typeRight == TSDB_DATA_TYPE_NCHAR){
+      *pRightData = ncharTobinary(*pRightData);
+      *freeRight = true;
+    }
+  }else{
+    ASSERT(0);
   }
 
-  if(typeRight == TSDB_DATA_TYPE_NCHAR) {
-    convertStringToDouble(*pRightData, pRightOut, typeRight, type);
-    *pRightData = pRightOut;
-  } else if(typeRight != type) {
-    convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
-    *pRightData = pRightOut;
-  }
   return true;
 }
 
@@ -935,44 +982,6 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
   }
 }
 
-STagVal getJsonValue(char *json, char *key, bool *isExist) {
-  STagVal val = {.pKey = key};
-  bool find = tTagGet(((const STag *)json), &val);  // json value is null and not exist is different
-  if(isExist){
-    *isExist = find;
-  }
-  return val;
-}
-
-void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
-  SColumnInfoData *pOutputCol = pOut->columnData;
-
-  int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
-  int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
-
-  pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
-
-  char *pRightData = colDataGetVarData(pRight->columnData, 0);
-  char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1);
-  memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData));
-  for (; i >= 0 && i < pLeft->numOfRows; i += step) {
-    if (colDataIsNull_var(pLeft->columnData, i)) {
-      colDataSetNull_var(pOutputCol, i);
-      pOutputCol->hasNull = true;
-      continue;
-    }
-    char *pLeftData = colDataGetVarData(pLeft->columnData, i);
-    bool isExist = false;
-    STagVal value = getJsonValue(pLeftData, jsonKey, &isExist);
-    char *data = isExist ? tTagValToData(&value, true) : NULL;
-    colDataAppend(pOutputCol, i, data, data == NULL);
-    if(isExist && IS_VAR_DATA_TYPE(value.type) && data){
-      taosMemoryFree(data);
-    }
-  }
-  taosMemoryFree(jsonKey);
-}
-
 void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   SColumnInfoData *pOutputCol = pOut->columnData;
 
@@ -1510,6 +1519,38 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
   doReleaseVec(pRightCol, rightConvert);
 }
 
+#define VEC_COM_INNER(pCol, index1, index2)    \
+  for (; i < pCol->numOfRows && i >= 0; i += step) {\
+    if (IS_HELPER_NULL(pLeft->columnData, index1) || IS_HELPER_NULL(pRight->columnData, index2)) {\
+      bool  res = false;\
+      colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\
+      continue;\
+    }\
+    char *pLeftData = colDataGetData(pLeft->columnData, index1);\
+    char *pRightData = colDataGetData(pRight->columnData, index2);\
+    int64_t leftOut = 0;\
+    int64_t rightOut = 0;\
+    bool freeLeft = false;\
+    bool freeRight = false;\
+    bool isJsonnull = false;\
+    bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight),\
+                                      &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight);\
+    if(isJsonnull){\
+      ASSERT(0);\
+    }\
+    if(!pLeftData || !pRightData){\
+      result = false;\
+    }\
+    if(!result){\
+      colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);\
+    }else{\
+      bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);\
+      colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\
+    }\
+    if(freeLeft) taosMemoryFreeClear(pLeftData);\
+    if(freeRight) taosMemoryFreeClear(pRightData);\
+  }
+
 void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) {
   int32_t       i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
   int32_t       step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
@@ -1533,79 +1574,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
   }
 
   if (pLeft->numOfRows == pRight->numOfRows) {
-    for (; i < pRight->numOfRows && i >= 0; i += step) {
-      if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, i)) {
-        bool  res = false;
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-        continue;  // TODO set null or ignore
-      }
-
-      char *pLeftData = colDataGetData(pLeft->columnData, i);
-      char *pRightData = colDataGetData(pRight->columnData, i);
-
-      int64_t leftOut = 0;
-      int64_t rightOut = 0;
-      bool isJsonnull = false;
-      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
-      if(isJsonnull){
-        ASSERT(0);
-      }
-      if(!result){
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
-      }else{
-        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-      }
-    }
+    VEC_COM_INNER(pLeft, i, i)
   } else if (pRight->numOfRows == 1) {
-    ASSERT(pLeft->pHashFilter == NULL);
-    for (; i >= 0 && i < pLeft->numOfRows; i += step) {
-      if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, 0)) {
-        bool  res = false;
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-        continue;
-      }
-
-      char *pLeftData = colDataGetData(pLeft->columnData, i);
-      char *pRightData = colDataGetData(pRight->columnData, 0);
-      int64_t leftOut = 0;
-      int64_t rightOut = 0;
-      bool isJsonnull = false;
-      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
-      if(isJsonnull){
-        ASSERT(0);
-      }
-      if(!result){
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
-      }else{
-        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-      }
-    }
+    VEC_COM_INNER(pLeft, i, 0)
   } else if (pLeft->numOfRows == 1) {
-    for (; i >= 0 && i < pRight->numOfRows; i += step) {
-      if (IS_HELPER_NULL(pRight->columnData, i) || IS_HELPER_NULL(pLeft->columnData, 0)) {
-        bool  res = false;
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-        continue;
-      }
-
-      char *pLeftData = colDataGetData(pLeft->columnData, 0);
-      char *pRightData = colDataGetData(pLeft->columnData, i);
-      int64_t leftOut = 0;
-      int64_t rightOut = 0;
-      bool isJsonnull = false;
-      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
-      if(isJsonnull){
-        ASSERT(0);
-      }
-      if(!result){
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
-      }else{
-        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-      }
-    }
+    VEC_COM_INNER(pRight, 0, i)
   }
 }
 
@@ -1683,10 +1656,6 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu
   vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH);
 }
 
-void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
-  vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS);
-}
-
 void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
     int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0;
@@ -1707,6 +1676,69 @@ void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
   vectorConvertImpl(pLeft, pOut);
 }
 
+STagVal getJsonValue(char *json, char *key, bool *isExist) {
+  STagVal val = {.pKey = key};
+  bool find = tTagGet(((const STag *)json), &val);  // json value is null and not exist is different
+  if(isExist){
+    *isExist = find;
+  }
+  return val;
+}
+
+void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  SColumnInfoData *pOutputCol = pOut->columnData;
+
+  int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
+
+  pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
+
+  char *pRightData = colDataGetVarData(pRight->columnData, 0);
+  char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1);
+  memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData));
+  for (; i >= 0 && i < pLeft->numOfRows; i += step) {
+    bool isExist = false;
+
+    if (!colDataIsNull_var(pLeft->columnData, i)) {
+      char *pLeftData = colDataGetVarData(pLeft->columnData, i);
+      getJsonValue(pLeftData, jsonKey, &isExist);
+    }
+
+    colDataAppend(pOutputCol, i, (const char*)(&isExist), false);
+
+  }
+  taosMemoryFree(jsonKey);
+}
+
+void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  SColumnInfoData *pOutputCol = pOut->columnData;
+
+  int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
+
+  pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
+
+  char *pRightData = colDataGetVarData(pRight->columnData, 0);
+  char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1);
+  memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData));
+  for (; i >= 0 && i < pLeft->numOfRows; i += step) {
+    if (colDataIsNull_var(pLeft->columnData, i)) {
+      colDataSetNull_var(pOutputCol, i);
+      pOutputCol->hasNull = true;
+      continue;
+    }
+    char *pLeftData = colDataGetVarData(pLeft->columnData, i);
+    bool isExist = false;
+    STagVal value = getJsonValue(pLeftData, jsonKey, &isExist);
+    char *data = isExist ? tTagValToData(&value, true) : NULL;
+    colDataAppend(pOutputCol, i, data, data == NULL);
+    if(isExist && IS_VAR_DATA_TYPE(value.type) && data){
+      taosMemoryFree(data);
+    }
+  }
+  taosMemoryFree(jsonKey);
+}
+
 _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
   switch (binFunctionId) {
     case OP_TYPE_ADD:

source/util/src/tcompare.c

@@ -222,11 +222,6 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) {
   return compareLenPrefixedWStr(pRight, pLeft);
 }
 
-int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) {
-  if(pLeft) return 0;
-  return 1;
-}
-
 // string > number > bool > null
 // ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison
 int32_t compareJsonVal(const void *pLeft, const void *pRight) {

tests/system-test/2-query/json_tag.py

@@ -118,7 +118,7 @@ class TDTestCase:
         tdSql.error("select jtag->location from jsons1")
         tdSql.error("select jtag contains location from jsons1")
         tdSql.error("select * from jsons1 where jtag contains location")
-        tdSql.error("select * from jsons1 where jtag contains''")
+        #tdSql.error("select * from jsons1 where jtag contains''")
         tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'")
         #
         # # test function error
@@ -129,6 +129,41 @@ class TDTestCase:
         tdSql.error("select ceil(jtag->'tag1') from jsons1")
         tdSql.error("select ceil(jtag) from jsons1")
         #
+
+        #test scalar operation
+        tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'")
+        tdSql.checkRows(13)
+        tdSql.checkData(0, 0, False)
+        tdSql.checkData(5, 0, True)
+        tdSql.checkData(12, 0, True)
+        tdSql.query("select jtag->'tag1' like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'")
+        tdSql.checkRows(13)
+        tdSql.checkData(10, 0, False)
+        tdSql.checkData(11, 0, False)
+        tdSql.checkData(12, 0, True)
+        tdSql.query("select jtag->'tag1' not like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'")
+        tdSql.checkRows(13)
+        tdSql.checkData(10, 0, False)
+        tdSql.checkData(11, 0, True)
+        tdSql.checkData(12, 0, False)
+        tdSql.query("select jtag->'tag1' match 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'")
+        tdSql.checkRows(13)
+        tdSql.checkData(10, 0, False)
+        tdSql.checkData(11, 0, False)
+        tdSql.checkData(12, 0, True)
+        tdSql.query("select jtag->'tag1' nmatch 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'")
+        tdSql.checkRows(13)
+        tdSql.checkData(10, 0, False)
+        tdSql.checkData(11, 0, True)
+        tdSql.checkData(12, 0, False)
+        tdSql.query("select jtag->'tag1',jtag->'tag1'>='a' from jsons1 order by jtag->'tag1'")
+        tdSql.checkRows(13)
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 1, False)
+        tdSql.checkData(7, 0, "false")
+        tdSql.checkData(7, 1, True)
+        tdSql.checkData(12, 1, True)
+
         # # test select normal column
         tdSql.query("select dataint from jsons1 order by dataint")
         tdSql.checkRows(9)

tests/system-test/fulltest.bat

 
-@REM python3 .\test.py -f 0-others\taosShell.py
+python3 .\test.py -f 0-others\taosShell.py
 python3 .\test.py -f 0-others\taosShellError.py
 python3 .\test.py -f 0-others\taosShellNetChk.py
 python3 .\test.py -f 0-others\telemetry.py
-@REM python3 .\test.py -f 0-others\taosdMonitor.py
+python3 .\test.py -f 0-others\taosdMonitor.py
 python3 .\test.py -f 0-others\udfTest.py
 python3 .\test.py -f 0-others\udf_create.py
 python3 .\test.py -f 0-others\udf_restart_taosd.py

tests/system-test/test.py

@@ -114,6 +114,7 @@ if __name__ == "__main__":
 
     if not execCmd == "":
         tdDnodes.init(deployPath)
+        print(execCmd)
         exec(execCmd)
         quit()