refactor: add test casese for json compare logic

include/common/tdataformat.h

@@ -502,7 +502,7 @@ typedef struct {
 
 #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
 
-#define kvRowLen(r)            (*(TDRowLenT *)(r))
+#define kvRowLen(r)            (*(uint16_t *)(r))
 #define kvRowNCols(r)          (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
 #define kvRowSetLen(r, len)    kvRowLen(r) = (len)
 #define kvRowSetNCols(r, n)    kvRowNCols(r) = (n)

source/common/src/tdatablock.c

@@ -113,14 +113,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
 
   int32_t type = pColumnInfoData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
+    int32_t dataLen = varDataTLen(pData);
+    if(type == TSDB_DATA_TYPE_JSON) {
+      if(*pData == TSDB_DATA_TYPE_NULL) {
+        dataLen = 0;
+      }else if(*pData == TSDB_DATA_TYPE_NCHAR) {
+        dataLen = varDataTLen(pData+CHAR_BYTES);
+      }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+        dataLen = LONG_BYTES;
+      }else if(*pData == TSDB_DATA_TYPE_BOOL) {
+        dataLen = CHAR_BYTES;
+      }
+      dataLen += CHAR_BYTES;
+    }
     SVarColAttr* pAttr = &pColumnInfoData->varmeta;
-    if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) {
+    if (pAttr->allocLen < pAttr->length + dataLen) {
       uint32_t newSize = pAttr->allocLen;
       if (newSize == 0) {
         newSize = 8;
       }
 
-      while (newSize < pAttr->length + varDataTLen(pData)) {
+      while (newSize < pAttr->length + dataLen) {
         newSize = newSize * 1.5;
       }
 
@@ -136,8 +149,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
     uint32_t len = pColumnInfoData->varmeta.length;
     pColumnInfoData->varmeta.offset[currentRow] = len;
 
-    memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData));
-    pColumnInfoData->varmeta.length += varDataTLen(pData);
+    memcpy(pColumnInfoData->pData + len, pData, dataLen);
+    pColumnInfoData->varmeta.length += dataLen;
   } else {
     memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
   }

source/libs/scalar/src/sclvector.c

@@ -774,7 +774,7 @@ char *getJsonValue(char *json, char *key){    //todo
     if(i == 0 && *data == TSDB_DATA_TYPE_NULL){
       return NULL;
     }
-    if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){
+    if(memcmp(key, data, varDataTLen(data)) == 0){
       return data + varDataTLen(data);
     }
   }
@@ -803,7 +803,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
       pOutputCol->hasNull = true;
       continue;
     }
-    colDataAppend(pOutputCol, i, pLeftData, false);
+    colDataAppend(pOutputCol, i, value, false);
   }
 }
 

source/libs/scalar/test/scalar/CMakeLists.txt

@@ -8,11 +8,12 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(scalarTest ${SOURCE_LIST})
 TARGET_LINK_LIBRARIES(
         scalarTest
-        PUBLIC os util common gtest qcom function nodes scalar
+        PUBLIC os util common gtest qcom function nodes scalar parser
 )
 
 TARGET_INCLUDE_DIRECTORIES(
         scalarTest
         PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar/"
+        PUBLIC "${TD_SOURCE_DIR}/source/libs/parser/inc"
         PRIVATE "${TD_SOURCE_DIR}/source/libs/scalar/inc"
 )

source/libs/scalar/test/scalar/scalarTests.cpp

@@ -125,56 +125,6 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
   
   *pNode = (SNode *)vnode;
 }
-void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
-  SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
-  SColumnNode *rnode = (SColumnNode *)node;
-  rnode->node.resType.type = dataType;
-  rnode->node.resType.bytes = dataBytes;
-  rnode->dataBlockId = 0;
-
-  if (NULL == *block) {
-    SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
-    res->info.numOfCols = 3;
-    res->info.rows = rowNum;
-    res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
-    for (int32_t i = 0; i < 2; ++i) {
-      SColumnInfoData idata = {{0}};
-      idata.info.type  = TSDB_DATA_TYPE_NULL;
-      idata.info.bytes = 10;
-      idata.info.colId = i + 1;
-
-      int32_t size = idata.info.bytes * rowNum;
-      idata.pData = (char *)taosMemoryCalloc(1, size);
-      taosArrayPush(res->pDataBlock, &idata);
-    }
-
-    SColumnInfoData idata = {{0}};
-    idata.info.type  = dataType;
-    idata.info.bytes = dataBytes;
-    idata.info.colId = 3;
-    int32_t size = idata.info.bytes * rowNum;
-    idata.pData = (char *)taosMemoryCalloc(1, size);
-    taosArrayPush(res->pDataBlock, &idata);
-
-    blockDataEnsureCapacity(res, rowNum);
-
-    SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
-    for (int32_t i = 0; i < rowNum; ++i) {
-      uint32_t len = pColumnInfoData->varmeta.length;
-      pColumnInfoData->varmeta.offset[i] = len;
-
-      memcpy(pColumnInfoData->pData + len, value, varDataLen(value));
-      pColumnInfoData->varmeta.length += varDataLen(value);
-    }
-
-    rnode->slotId = 2;
-    rnode->colId = 3;
-
-    *block = res;
-  }
-
-  *pNode = (SNode *)rnode;
-}
 
 void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
   SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN);
@@ -960,56 +910,79 @@ TEST(constantTest, greater_and_lower) {
   nodesDestroyNode(res);
 }
 
-TEST(columnTest, smallint_value_add_json_column) {
-  scltInitLogFile();
+void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
+  char key[32] = {0};
+  memcpy(varDataVal(key), key, strlen(key));
+  varDataLen(key) = strlen(key);
 
   SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
-  int32_t leftv = 1;
-  char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}";
-  char key[10] = {0};
-  memcpy(varDataVal(key), "k1", 2);
-  varDataLen(key) = 2;
-
-  SKVRowBuilder kvRowBuilder;
-  tdInitKVRowBuilder(&kvRowBuilder);
-  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
-  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
-
-  double eRes[5] = {1.0, -4, -3, 24, 101};
-
-  SSDataBlock *src = NULL;
-  int32_t rowNum = 1;
-
   scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key);
-  scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row);
-  scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
+  scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, varDataLen(json), 1, json);
+  scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight);
+}
 
+void makeOperator(SNode **opNode, SArray *blockList, int32_t resType, EOperatorType opType, int32_t rightType, void *rightData){
   SNode *right = NULL;
-  scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv);
-  scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right);
+  scltMakeValueNode(&right, rightType, rightData);
+  scltMakeOpNode(opNode, opType, resType, *opNode, right);
+
+  SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes);
+  int16_t dataBlockId = 0, slotId = 0;
+  scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, 1, &colInfo);
+  scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode);
+}
 
+void makeCalculate(void *json, void *key, void *value, double exceptValue, EOperatorType opType){
   SArray *blockList = taosArrayInit(2, POINTER_BYTES);
+  SSDataBlock *src = NULL;
+  SNode *opNode = NULL;
+
+  makeJsonArrow(&src, &opNode, row, key);
   taosArrayPush(blockList, &src);
-  SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
-  int16_t dataBlockId = 0, slotId = 0;
-  scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo);
-  scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode);
+
+  makeOperator(&opNode, blockList, TSDB_DATA_TYPE_DOUBLE, opType, TSDB_DATA_TYPE_INT, value);
 
   int32_t code = scalarCalculate(opNode, blockList, NULL);
   ASSERT_EQ(code, 0);
 
   SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList);
-  ASSERT_EQ(res->info.rows, rowNum);
+  ASSERT_EQ(res->info.rows, 1);
   SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
   ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE);
-  for (int32_t i = 0; i < rowNum; ++i) {
-    ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]);
-  }
+  ASSERT_EQ(*((double *)colDataGetData(column, 0)), exceptValue);
 
   taosArrayDestroyEx(blockList, scltFreeDataBlock);
   nodesDestroyNode(opNode);
 }
 
+TEST(columnTest, int_value_add_json_column) {
+  scltInitLogFile();
+  char *rightv= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}";
+  char *key = "k1";
+
+  SKVRowBuilder kvRowBuilder;
+  tdInitKVRowBuilder(&kvRowBuilder);
+  parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0);
+  SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
+
+  int32_t leftv = 1;
+  makeCalculate(row, key, &leftv, 5.0, OP_TYPE_ADD);
+
+  leftv = 8;
+  makeCalculate(row, key, &leftv, -4, OP_TYPE_SUB);
+
+  leftv = 2;
+  makeCalculate(row, key, &leftv, 8.0, OP_TYPE_MULTI);
+
+  leftv = 2;
+  makeCalculate(row, key, &leftv, 2.0, OP_TYPE_DIV);
+
+  leftv = 3;
+  makeCalculate(row, key, &leftv, 1.0, OP_TYPE_MODE);
+
+  makeCalculate(row, key, &leftv, -4, OP_TYPE_MINUS);
+}
+
 TEST(columnTest, smallint_value_add_int_column) {
   scltInitLogFile();