refactor: add test casese for json compare logic

include/util/tdef.h

@@ -272,11 +272,6 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
 #define TSDB_MAX_JSON_TAG_LEN   16384
-#define TSDB_JSON_PLACEHOLDER      0x7F
-#define TSDB_JSON_null             0x00
-#define TSDB_JSON_NOT_NULL         0x01
-#define TSDB_JSON_NULL             0x00
-
 
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32

source/libs/parser/src/parUtil.c

@@ -276,7 +276,7 @@ static bool isValidateTag(char *input) {
 
 int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
   // set json NULL data
-  uint8_t jsonNULL = TSDB_JSON_NULL;
+  uint8_t jsonNULL = TSDB_DATA_TYPE_NULL;
   int jsonIndex = startColId + 1;
   if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
     tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
@@ -328,7 +328,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
     strncpy(varDataVal(tagKV), jsonKey, keyLen);
     varDataSetLen(tagKV, keyLen);
     if(taosHashGetSize(keyHash) == 0){
-      uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
+      uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON;
       tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES);   // add json type
     }
     taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES);  // add key to hash to remove dumplicate, value is useless

source/libs/scalar/src/scalar.c

@@ -604,39 +604,55 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
   return DEAL_RES_CONTINUE;
 }
 
-//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
-//  STargetNode *target = (STargetNode *)pNode;
-//
-//  if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
-//    sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
-//    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
-//    return DEAL_RES_ERROR;
-//  }
-//
-//  SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
-//  if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
-//    sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
-//    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
-//    return DEAL_RES_ERROR;
-//  }
-//
-//  // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
-//  SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
-//
-//  SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
-//  if (NULL == res) {
-//    sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
-//    ctx->code = TSDB_CODE_QRY_APP_ERROR;
-//    return DEAL_RES_ERROR;
-//  }
-//
-//  colDataAssign(col, res->columnData, res->numOfRows);
-//  block->info.rows = res->numOfRows;
-//
-//  sclFreeParam(res);
-//  taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
-//  return DEAL_RES_CONTINUE;
-//}
+EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
+  STargetNode *target = (STargetNode *)pNode;
+
+  if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
+    sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
+    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+    return DEAL_RES_ERROR;
+  }
+
+  int32_t index = -1;
+  for(int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
+    SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, i);
+    if (pb->info.blockId == target->dataBlockId) {
+      index = i;
+      break;
+    }
+  }
+
+  if (index == -1) {
+    sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
+    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+    return DEAL_RES_ERROR;
+  }
+
+  SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
+
+  if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
+    sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
+    ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
+    return DEAL_RES_ERROR;
+  }
+
+  // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
+  SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
+
+  SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
+  if (NULL == res) {
+    sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
+    ctx->code = TSDB_CODE_QRY_APP_ERROR;
+    return DEAL_RES_ERROR;
+  }
+
+  colDataAssign(col, res->columnData, res->numOfRows);
+  block->info.rows = res->numOfRows;
+
+  sclFreeParam(res);
+  taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
+  return DEAL_RES_CONTINUE;
+}
 
 EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
   if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
@@ -656,9 +672,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
     return sclWalkOperator(pNode, ctx);
   }
 
-//  if (QUERY_NODE_TARGET == nodeType(pNode)) {
-//    return sclWalkTarget(pNode, ctx);
-//  }
+  if (QUERY_NODE_TARGET == nodeType(pNode)) {
+    return sclWalkTarget(pNode, ctx);
+  }
 
   sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
   ctx->code = TSDB_CODE_QRY_INVALID_INPUT;

source/libs/scalar/src/sclvector.c

@@ -767,6 +767,17 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
 }
 
 char *getJsonValue(char *json, char *key){    //todo
+  int16_t cols = kvRowNCols(json);
+  for (int i = 0; i < cols; ++i) {
+    SColIdx *pColIdx = kvRowColIdxAt(json, i);
+    char *data = kvRowColVal(json, pColIdx);
+    if(i == 0 && *data == TSDB_DATA_TYPE_NULL){
+      return NULL;
+    }
+    if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){
+      return data + varDataTLen(data);
+    }
+  }
   return NULL;
 }
 
@@ -844,7 +855,11 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
     if (pLeft->numOfRows == pRight->numOfRows) {
       for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i);
+        if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+          colDataAppendNULL(pOutputCol, i);
+          continue;  // TODO set null or ignore
+        }
+        *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i);
       }
 
       pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);
@@ -957,7 +972,11 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut
 
     if (pLeft->numOfRows == pRight->numOfRows) {
       for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) {
-        *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i);
+        if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) {
+          colDataAppendNULL(pOutputCol, i);
+          continue;  // TODO set null or ignore
+        }
+        *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i);
       }
 
       pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull);

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -92,6 +92,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s
     blockDataEnsureCapacity(res, rows);
 
     *dataBlockId = taosArrayGetSize(pBlockList) - 1;
+    res->info.blockId = *dataBlockId;
     *slotId = 0;
   } else {
     SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);
@@ -124,6 +125,56 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
   
   *pNode = (SNode *)vnode;
 }
+void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
+  SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
+  SColumnNode *rnode = (SColumnNode *)node;
+  rnode->node.resType.type = dataType;
+  rnode->node.resType.bytes = dataBytes;
+  rnode->dataBlockId = 0;
+
+  if (NULL == *block) {
+    SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
+    res->info.numOfCols = 3;
+    res->info.rows = rowNum;
+    res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
+    for (int32_t i = 0; i < 2; ++i) {
+      SColumnInfoData idata = {{0}};
+      idata.info.type  = TSDB_DATA_TYPE_NULL;
+      idata.info.bytes = 10;
+      idata.info.colId = i + 1;
+
+      int32_t size = idata.info.bytes * rowNum;
+      idata.pData = (char *)taosMemoryCalloc(1, size);
+      taosArrayPush(res->pDataBlock, &idata);
+    }
+
+    SColumnInfoData idata = {{0}};
+    idata.info.type  = dataType;
+    idata.info.bytes = dataBytes;
+    idata.info.colId = 3;
+    int32_t size = idata.info.bytes * rowNum;
+    idata.pData = (char *)taosMemoryCalloc(1, size);
+    taosArrayPush(res->pDataBlock, &idata);
+
+    blockDataEnsureCapacity(res, rowNum);
+
+    SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
+    for (int32_t i = 0; i < rowNum; ++i) {
+      uint32_t len = pColumnInfoData->varmeta.length;
+      pColumnInfoData->varmeta.offset[i] = len;
+
+      memcpy(pColumnInfoData->pData + len, value, varDataLen(value));
+      pColumnInfoData->varmeta.length += varDataLen(value);
+    }
+
+    rnode->slotId = 2;
+    rnode->colId = 3;
+
+    *block = res;
+  }
+
+  *pNode = (SNode *)rnode;
+}
 
 void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
   SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
@@ -909,6 +960,56 @@ TEST(constantTest, greater_and_lower) {
   nodesDestroyNode(res);
 }
 
+TEST(columnTest, smallint_value_add_json_column) {
+  scltInitLogFile();
+
+  SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
+  int32_t leftv = 1;
+  char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}";
+  char key[10] = {0};
+  memcpy(varDataVal(key), "k1", 2);
+  varDataLen(key) = 2;
+
+  SKVRowBuilder kvRowBuilder;
+  tdInitKVRowBuilder(&kvRowBuilder);
+  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
+  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+
+  double eRes[5] = {1.0, -4, -3, 24, 101};
+
+  SSDataBlock *src = NULL;
+  int32_t rowNum = 1;
+
+  scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key);
+  scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row);
+  scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
+
+  SNode *right = NULL;
+  scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv);
+  scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right);
+
+  SArray *blockList = taosArrayInit(2, POINTER_BYTES);
+  taosArrayPush(blockList, &src);
+  SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
+  int16_t dataBlockId = 0, slotId = 0;
+  scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
+  scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
+
+  int32_t code = scalarCalculate(opNode, blockList, NULL);
+  ASSERT_EQ(code, 0);
+
+  SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
+  ASSERT_EQ(res->info.rows, rowNum);
+  SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
+  ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE);
+  for (int32_t i = 0; i < rowNum; ++i) {
+    ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]);
+  }
+
+  taosArrayDestroyEx(blockList, scltFreeDataBlock);
+  nodesDestroyNode(opNode);
+}
+
 TEST(columnTest, smallint_value_add_int_column) {
   scltInitLogFile();
   
@@ -928,7 +1029,7 @@ TEST(columnTest, smallint_value_add_int_column) {
   int16_t dataBlockId = 0, slotId = 0;
   scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
   scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
-  
+
   int32_t code = scalarCalculate(opNode, blockList, NULL);
   ASSERT_EQ(code, 0);