refactor: add compare logic for json value

include/common/ttypes.h

@@ -49,7 +49,7 @@ typedef struct {
 #define varDataCopy(dst, v)    memcpy((dst), (void *)(v), varDataTLen(v))
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
+#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))

include/util/tcompare.h

@@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
 
+int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
+
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char *a, const char *b, int32_t type, size_t size);

include/util/tdef.h

@@ -274,7 +274,6 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_JSON_TAG_LEN   16384
 #define TSDB_JSON_PLACEHOLDER      0x7F
 #define TSDB_JSON_null             0x00
-#define TSDB_JSON_KEY_NULL         0x00
 #define TSDB_JSON_NOT_NULL         0x01
 #define TSDB_JSON_NULL             0x00
 

source/libs/nodes/src/nodesUtilFuncs.c

@@ -1006,6 +1006,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
     case OP_TYPE_NOT_LIKE:
     case OP_TYPE_MATCH:
     case OP_TYPE_NMATCH:
+    case OP_TYPE_JSON_CONTAINS:
     case OP_TYPE_IS_NULL:
     case OP_TYPE_IS_NOT_NULL:
     case OP_TYPE_IS_TRUE:
@@ -1024,7 +1025,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
 bool nodesIsJsonOp(const SOperatorNode* pOp) {
   switch (pOp->opType) {
     case OP_TYPE_JSON_GET_VALUE:
-    case OP_TYPE_JSON_CONTAINS:
       return true;
     default:
       break;

source/libs/parser/src/parTranslater.c

@@ -469,14 +469,18 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
     pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE;
     pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
   } else if (nodesIsComparisonOp(pOp)) {
-    if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
+    if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
         TSDB_DATA_TYPE_BLOB == rdt.type) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
     }
     pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
     pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
-  } else {
-    // todo json operator
+  } else if (nodesIsJsonOp(pOp)){
+    if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
+      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
+    }
+    pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
+    pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
   }
   return DEAL_RES_CONTINUE;
 }
@@ -2814,25 +2818,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
 
 static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
                              SKVRowBuilder* pBuilder) {
-  if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
-    return pCxt->errCode;
-  }
-  SVariant var;
-  valueNodeToVariant(pVal, &var);
   if(pSchema->type == TSDB_DATA_TYPE_JSON){
-    if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
-      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz);
+    if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
+      return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
+    }
+
+    return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId);
   }
-    return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId);
+
+  if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
+    return pCxt->errCode;
   }
 
-  char    tagVal[TSDB_MAX_TAGS_LEN] = {0};
-  int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true);
-  if (TSDB_CODE_SUCCESS == code) {
-    tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]);
+  if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){
+    // todo
+  }else{
+    tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
   }
 
-  return code;
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,

source/libs/parser/src/parUtil.c

@@ -252,12 +252,10 @@ static bool isValidateTag(char *input) {
 
 int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
   // set json NULL data
-  uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL;
   uint8_t jsonNULL = TSDB_JSON_NULL;
   int jsonIndex = startColId + 1;
-  tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES);   // add json null type
   if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
-    tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);   // add json null value
+    tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -273,6 +271,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
   }
 
   int retCode = 0;
+  char *tagKV = NULL;
   SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
   for(int i = 0; i < size; i++) {
     cJSON* item = cJSON_GetArrayItem(root, i);
@@ -296,66 +295,64 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
     if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
       continue;
     }
-
+    // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES
+    tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1);
+    if(!tagKV) {
+      retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      goto end;
+    }
+    strncpy(varDataVal(tagKV), jsonKey, keyLen);
+    varDataSetLen(tagKV, keyLen);
     if(taosHashGetSize(keyHash) == 0){
       uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
       tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES);   // add json type
     }
-    // json key encode by binary
-    void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1);
-    if(!tagKey) {
-      retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto end;
-    }
-    strncpy(varDataVal(tagKey), jsonKey, keyLen);
     taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless
 
-    varDataSetLen(tagKey, keyLen);
-    tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey));   // add json key
-    taosMemoryFree(tagKey);
-
     if(item->type == cJSON_String){     // add json value  format: type|data
       char *jsonValue = item->valuestring;
       int32_t valLen = (int32_t)strlen(jsonValue);
-      char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1);
-      if(!tagVal) {
+      int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES;
+      char *tmp = taosMemoryRealloc(tagKV, totalLen);
+      if(!tmp) {
         retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
         goto end;
-      } ;
-      tagVal[0] = TSDB_DATA_TYPE_NCHAR;
-      char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES);
-      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData),
+      }
+      tagKV = tmp;
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = TSDB_DATA_TYPE_NCHAR;
+      if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData),
                                                   (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
         qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
         retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
-        taosMemoryFree(tagVal);
         goto end;
       }
 
-      varDataSetLen(tagData, valLen);
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData));
-      taosMemoryFree(tagVal);
+      varDataSetLen(valueData, valLen);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen);
     }else if(item->type == cJSON_Number){
       if(!isfinite(item->valuedouble)){
         qError("json value is invalidate");
         retCode =  buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
         goto end;
       }
-      char tagVal[LONG_BYTES + CHAR_BYTES] = {0};
-      tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT
-                                                                        : TSDB_DATA_TYPE_DOUBLE;
-      char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
-      if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble;
-      else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint;
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES);
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE;
+      if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble;
+      else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES);
     }else if(item->type == cJSON_True || item->type == cJSON_False){
-      char tagVal[CHAR_BYTES + CHAR_BYTES] = {0};
-      tagVal[0] = TSDB_DATA_TYPE_BOOL;
-      tagVal[1] = (char)(item->valueint);
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES);
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
+      *valueType = TSDB_DATA_TYPE_BOOL;
+      *valueData = (char)(item->valueint);
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES);
     }else if(item->type == cJSON_NULL){
-      char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL};
-      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES);
+      char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
+      *valueType = TSDB_DATA_TYPE_NULL;
+      tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
     }
     else{
       retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
@@ -364,10 +361,11 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
   }
 
   if(taosHashGetSize(keyHash) == 0){  // set json NULL true
-    tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);
+    tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
   }
 
 end:
+  taosMemoryFree(tagKV);
   taosHashCleanup(keyHash);
   cJSON_Delete(root);
   return retCode;

source/libs/scalar/inc/sclvector.h

@@ -53,6 +53,8 @@ static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index)
   return (double)*((double *)src + index);
 }
 
+double getVectorDoubleValue_JSON(void *src, int32_t index);
+
 static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
   _getDoubleValue_fn_t p = NULL;
   if (srcType == TSDB_DATA_TYPE_TINYINT) {
@@ -77,6 +79,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType)
     p = getVectorDoubleValue_DOUBLE;
   } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) {
     p = getVectorDoubleValue_BIGINT;
+  } else if (srcType == TSDB_DATA_TYPE_JSON) {
+    p = getVectorDoubleValue_JSON;
   } else {
     assert(0);
   }

source/libs/scalar/src/filter.c

@@ -170,7 +170,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val
   compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
   setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, 
   compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
-  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch
+  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey
 };
 
 int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@@ -222,6 +222,11 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
     }
   }
 
+  if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) {
+    return 28;
+  }
+
+
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:   comparFn = 1;   break;

source/libs/scalar/src/scalar.c

@@ -600,44 +600,39 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
   return DEAL_RES_CONTINUE;
 }
 
-EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
-  STargetNode *target = (STargetNode *)pNode;
-  
-  if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
-    sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
-    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
-    return DEAL_RES_ERROR;
-  }
-
-  SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
-  if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
-    sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
-    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
-    return DEAL_RES_ERROR;
-  }
-
-  SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
-  
-  SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
-  if (NULL == res) {
-    sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
-    ctx->code = TSDB_CODE_QRY_APP_ERROR;
-    return DEAL_RES_ERROR;
-  }
-  
-  for (int32_t i = 0; i < res->numOfRows; ++i) {
-    if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) {
-      colDataAppend(col, i, NULL, true);
-    } else {
-      char *p = colDataGetData(res->columnData, i);
-      colDataAppend(col, i, p, false);
-    }
-  }
-
-  sclFreeParam(res);
-  taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
-  return DEAL_RES_CONTINUE;
-}
+//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
+//  STargetNode *target = (STargetNode *)pNode;
+//
+//  if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
+//    sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
+//    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+//    return DEAL_RES_ERROR;
+//  }
+//
+//  SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
+//  if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
+//    sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
+//    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+//    return DEAL_RES_ERROR;
+//  }
+//
+//  // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
+//  SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
+//
+//  SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
+//  if (NULL == res) {
+//    sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
+//    ctx->code = TSDB_CODE_QRY_APP_ERROR;
+//    return DEAL_RES_ERROR;
+//  }
+//
+//  colDataAssign(col, res->columnData, res->numOfRows);
+//  block->info.rows = res->numOfRows;
+//
+//  sclFreeParam(res);
+//  taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
+//  return DEAL_RES_CONTINUE;
+//}
 
 EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
   if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
@@ -657,9 +652,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
     return sclWalkOperator(pNode, ctx);
   }
 
-  if (QUERY_NODE_TARGET == nodeType(pNode)) {
-    return sclWalkTarget(pNode, ctx);
-  }
+//  if (QUERY_NODE_TARGET == nodeType(pNode)) {
+//    return sclWalkTarget(pNode, ctx);
+//  }
 
   sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
   ctx->code = TSDB_CODE_QRY_INVALID_INPUT;

source/libs/scalar/src/sclvector.c

@@ -177,24 +177,24 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam* pOut, int32_t rowInd
   colDataAppendInt8(pOut->columnData, rowIndex, (int8_t*) &v);
 }
 
-static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
-  int32_t len = 0;
-  int32_t inputLen = varDataLen(buf);
-
-  char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
-  /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len);
-  varDataSetLen(t, len);
-
-  colDataAppend(pOut->columnData, rowIndex, t, false);
-  taosMemoryFree(t);
-}
+//static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
+//  int32_t len = 0;
+//  int32_t inputLen = varDataLen(buf);
+//
+//  char* t = taosMemoryCalloc(1,(inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
+//  /*int32_t resLen = */taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4*) varDataVal(t), pOut->columnData->info.bytes, &len);
+//  varDataSetLen(t, len);
+//
+//  colDataAppend(pOut->columnData, rowIndex, t, false);
+//  taosMemoryFree(t);
+//}
 
 //TODO opt performance, tmp is not needed.
 int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
   int32_t bufSize = pIn->columnData->info.bytes;
   char *tmp = taosMemoryMalloc(bufSize + VARSTR_HEADER_SIZE);
 
-  bool vton = false;
+//  bool vton = false;
 
   _bufConverteFunc func = NULL;
   if (TSDB_DATA_TYPE_BOOL == outType) {
@@ -205,9 +205,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
     func = varToUnsigned;
   } else if (IS_FLOAT_TYPE(outType)) {
     func = varToFloat;
-  } else if (outType == TSDB_DATA_TYPE_NCHAR) {
-    func = varToNchar;
-    vton = true;
+//  } else if (outType == TSDB_DATA_TYPE_NCHAR) {   // can not be nchar or binary
+//    func = varToNchar;
+//    vton = true;
   } else {
     sclError("invalid convert outType:%d", outType);
     return TSDB_CODE_QRY_APP_ERROR;
@@ -221,9 +221,9 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
     }
 
     char* data = colDataGetData(pIn->columnData, i);
-    if (vton) {
-      memcpy(tmp, data, varDataTLen(data));
-    } else {
+//    if (vton) {
+//      memcpy(tmp, data, varDataTLen(data));
+//    } else {
       if (TSDB_DATA_TYPE_VARCHAR == inType) {
         memcpy(tmp, varDataVal(data), varDataLen(data));
         tmp[varDataLen(data)] = 0;
@@ -239,7 +239,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
 
         tmp[len] = 0;
       }
-    }
+//    }
     
     (*func)(tmp, pOut, i);
   }
@@ -248,6 +248,135 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
   return TSDB_CODE_SUCCESS;
 }
 
+void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){
+  switch (outType) {
+    case TSDB_DATA_TYPE_BOOL: {
+      GET_TYPED_DATA(*((bool *)outData), bool, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_TINYINT: {
+      GET_TYPED_DATA(*((int8_t *)outData), int8_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      GET_TYPED_DATA(*((int16_t *)outData), int16_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      GET_TYPED_DATA(*((int32_t *)outData), int32_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP: {
+      GET_TYPED_DATA(*((int64_t *)outData), int64_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UTINYINT: {
+      GET_TYPED_DATA(*((uint8_t *)outData), uint8_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT: {
+      GET_TYPED_DATA(*((uint16_t *)outData), uint16_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UINT: {
+      GET_TYPED_DATA(*((uint32_t *)outData), uint32_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_UBIGINT: {
+      GET_TYPED_DATA(*((uint64_t *)outData), uint64_t, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      GET_TYPED_DATA(*((float *)outData), float, inType, inData);
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      GET_TYPED_DATA(*((double *)outData), double, inType, inData);
+      break;
+    }
+    default:{
+      ASSERT(0);
+    }
+  }
+}
+
+void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
+  char *tmp = taosMemoryMalloc(varDataTLen(inData));
+  if (inType == TSDB_DATA_TYPE_NCHAR) {
+    int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
+    if (len < 0) {
+      sclError("castConvert taosUcs4ToMbs error 1");
+    }
+
+    tmp[len] = 0;
+  } else {
+    memcpy(tmp, varDataVal(inData), varDataLen(inData));
+    tmp[varDataLen(inData)] = 0;
+  }
+
+  ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
+  double value = strtod(tmp, NULL);
+
+  *((double *)outData) = value;
+  taosMemoryFreeClear(tmp);
+}
+
+double getVectorDoubleValue_JSON(void *src, int32_t index){
+  ASSERT(!colDataIsNull_s(((SColumnInfoData*)src), index));
+  char *data = colDataGetData((SColumnInfoData*)src, index);
+  double out = 0;
+  if(*data == TSDB_DATA_TYPE_NCHAR) {
+    convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+  } {
+    convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+  }
+  return out;
+}
+
+void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut){
+  if(optr == OP_TYPE_JSON_CONTAINS) {
+    return;
+  }
+
+  if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){
+    return;
+  }
+
+  if(typeLeft == TSDB_DATA_TYPE_JSON){
+    typeLeft = **pLeftData;
+    (*pLeftData) ++;
+  }
+  if(typeRight == TSDB_DATA_TYPE_JSON){
+    typeRight = **pRightData;
+    (*pRightData) ++;
+  }
+  int8_t type = vectorGetConvertType(typeLeft, typeRight);
+
+  if(type == 0) {
+    *fp = filterGetCompFunc(typeLeft, optr);
+    return;
+  }
+
+  *fp = filterGetCompFunc(type, optr);
+
+  if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
+    convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type);
+    *pLeftData = pLeftOut;
+  } else if(typeLeft != type) {
+    convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
+    *pLeftData = pLeftOut;
+  }
+
+  if(IS_VAR_DATA_TYPE(typeRight)) {
+    convertStringToDouble(*pRightData, pRightOut, typeRight, type);
+    *pRightData = pRightOut;
+  } else if(typeRight != type) {
+    convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
+    *pRightData = pRightOut;
+  }
+}
+
 // TODO opt performance
 int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut) {
   SColumnInfoData* pInputCol  = pIn->columnData;
@@ -522,7 +651,7 @@ enum {
 static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SScalarParam* pParam, int32_t type) {
   SColumnInfoData* pCol = pParam->columnData;
 
-  if (IS_VAR_DATA_TYPE(pCol->info.type)) {
+  if (IS_VAR_DATA_TYPE(pCol->info.type) || pCol->info.type != TSDB_DATA_TYPE_JSON) {
     pDest->numOfRows = pParam->numOfRows;
 
     SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
@@ -556,7 +685,8 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, 0);
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -587,6 +717,36 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
   }
 }
 
+char *getJsonValue(char *json, char *key){    //todo
+  return NULL;
+}
+
+void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  SColumnInfoData *pOutputCol = pOut->columnData;
+
+  int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
+
+  pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
+
+  char *pRightData = colDataGetData(pRight->columnData, 0);
+  for (; i >= 0 && i < pLeft->numOfRows; i += step) {
+    if (pLeft->columnData->varmeta.offset[i] == -1) {
+      pOutputCol->varmeta.offset[i] = -1;
+      pOutputCol->hasNull = true;
+      continue;
+    }
+    char *pLeftData = colDataGetData(pLeft->columnData, i);
+    char *value = getJsonValue(pLeftData, pRightData);
+    if (!value || *value == TSDB_DATA_TYPE_NULL) {
+      pOutputCol->varmeta.offset[i] = -1;
+      pOutputCol->hasNull = true;
+      continue;
+    }
+    colDataAppend(pOutputCol, i, pLeftData, false);
+  }
+}
+
 void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   SColumnInfoData *pOutputCol = pOut->columnData;
 
@@ -605,7 +765,12 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                + getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -637,7 +802,8 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = (getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, 0)) * factor;
+      *output = (getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                 - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0)) * factor;
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -664,7 +830,12 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                - getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -696,7 +867,8 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData*
     colDataAppendNNULL(pOutputCol, 0, numOfRows);
   } else {
     for (; i >= 0 && i < numOfRows; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, 0);
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
     }
     pOutputCol->hasNull = pLeftCol->hasNull;
     if (pOutputCol->hasNull) {
@@ -722,7 +894,12 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) * getVectorDoubleValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                * getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -760,7 +937,12 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
   double *output = (double *)pOutputCol->pData;
   if (pLeft->numOfRows == pRight->numOfRows) {  // check for the 0 value
     for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-      *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, i);
+      if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+        colDataAppendNULL(pOutputCol, i);
+        continue;  // TODO set null or ignore
+      }
+      *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                 /getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
     }
 
     pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -776,7 +958,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
     } else {
       for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft(pLeftCol->pData, 0) / getVectorDoubleValueFnRight(pRightCol->pData, i);
+        *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0)
+                  / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
       }
       pOutputCol->hasNull = pRightCol->hasNull;
       if (pOutputCol->hasNull) {
@@ -788,7 +971,8 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p
       colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
     } else {
       for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) / getVectorDoubleValueFnRight(pRightCol->pData, 0);
+        *output = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i)
+                  / getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
       }
       pOutputCol->hasNull = pLeftCol->hasNull;
       if (pOutputCol->hasNull) {
@@ -825,8 +1009,8 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
         continue;
       }
 
-      double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i);
-      double rx = getVectorDoubleValueFnRight(pRightCol->pData, i);
+      double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
+      double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
       if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) {
         colDataAppendNULL(pOutputCol, i);
         continue;
@@ -835,7 +1019,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
       *output = lx - ((int64_t)(lx / rx)) * rx;
     }
   } else if (pLeft->numOfRows == 1) {
-    double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, 0);
+    double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), 0);
     if (colDataIsNull_f(pLeftCol->nullbitmap, 0) || isnan(lx) || isinf(lx)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows);
     } else {
@@ -845,7 +1029,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
           continue;
         }
 
-        double rx = getVectorDoubleValueFnRight(pRightCol->pData, i);
+        double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), i);
         if (isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
@@ -855,7 +1039,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
       }
     }
   } else if (pRight->numOfRows == 1) {
-    double rx = getVectorDoubleValueFnRight(pRightCol->pData, 0);
+    double rx = getVectorDoubleValueFnRight((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData), 0);
     if (colDataIsNull_f(pRightCol->nullbitmap, 0) || FLT_EQUAL(rx, 0)) {  // Set pLeft->numOfRows NULL value
       colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows);
     } else {
@@ -865,7 +1049,7 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam
           continue;
         }
 
-        double lx = getVectorDoubleValueFnLeft(pLeftCol->pData, i);
+        double lx = getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
         if (isnan(lx) || isinf(lx)) {
           colDataAppendNULL(pOutputCol, i);
           continue;
@@ -895,7 +1079,11 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
 
   double *output = (double *)pOutputCol->pData;
   for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) {
-    *output = - getVectorDoubleValueFnLeft(pLeftCol->pData, i);
+    if (colDataIsNull_s(pLeft->columnData, i)) {
+      colDataAppendNULL(pOutputCol, i);
+      continue;  // TODO set null or ignore
+    }
+    *output = - getVectorDoubleValueFnLeft((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData), i);
   }
 
   pOutputCol->hasNull = pLeftCol->hasNull;
@@ -1099,30 +1287,39 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
 
       char *pLeftData = colDataGetData(pLeft->columnData, i);
       char *pRightData = colDataGetData(pRight->columnData, i);
+
+      int64_t leftOut = 0;
+      int64_t rightOut = 0;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
   } else if (pRight->numOfRows == 1) {
-    char *pRightData = colDataGetData(pRight->columnData, 0);
     ASSERT(pLeft->pHashFilter == NULL);
-
     for (; i >= 0 && i < pLeft->numOfRows; i += step) {
       if (colDataIsNull_s(pLeft->columnData, i)) {
         continue;
       }
 
       char *pLeftData = colDataGetData(pLeft->columnData, i);
+      char *pRightData = colDataGetData(pRight->columnData, 0);
+      int64_t leftOut = 0;
+      int64_t rightOut = 0;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
   } else if (pLeft->numOfRows == 1) {
-    char *pLeftData = colDataGetData(pLeft->columnData, 0);
     for (; i >= 0 && i < pRight->numOfRows; i += step) {
       if (colDataIsNull_s(pRight->columnData, i)) {
         continue;
       }
 
+      char *pLeftData = colDataGetData(pLeft->columnData, 0);
       char *pRightData = colDataGetData(pLeft->columnData, i);
+      int64_t leftOut = 0;
+      int64_t rightOut = 0;
+      convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut);
       bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
       colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
     }
@@ -1203,6 +1400,10 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu
   vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH);
 }
 
+void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS);
+}
+
 void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
     int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0;
@@ -1271,6 +1472,10 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
       return vectorBitOr;
     case OP_TYPE_IS_TRUE:
       return vectorIsTrue;
+    case OP_TYPE_JSON_GET_VALUE:
+      return vectorJsonArrow;
+    case OP_TYPE_JSON_CONTAINS:
+      return vectorJsonContains;
     default:
       assert(0);
       return NULL;

source/util/src/tcompare.c

@@ -222,6 +222,11 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) {
   return compareLenPrefixedWStr(pRight, pLeft);
 }
 
+int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) {
+  if(pLeft) return 0;
+  return 1;
+}
+
 /*
  * Compare two strings
  *    TSDB_MATCH:            Match