From 412e2cbb0d3631017c81783fdfb8d18ca67f304f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Apr 2022 17:52:04 +0800 Subject: [PATCH] refactor: add test casese for json compare logic --- include/util/tdef.h | 5 - source/libs/parser/src/parUtil.c | 4 +- source/libs/scalar/src/scalar.c | 88 +++++++++------ source/libs/scalar/src/sclvector.c | 23 +++- .../libs/scalar/test/scalar/scalarTests.cpp | 103 +++++++++++++++++- 5 files changed, 177 insertions(+), 46 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index eb7a51cd7e..4c3126caa9 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -272,11 +272,6 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_MAX_JSON_TAG_LEN 16384 -#define TSDB_JSON_PLACEHOLDER 0x7F -#define TSDB_JSON_null 0x00 -#define TSDB_JSON_NOT_NULL 0x01 -#define TSDB_JSON_NULL 0x00 - #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index f3eb0c5ebc..0ff5843736 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -276,7 +276,7 @@ static bool isValidateTag(char *input) { int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){ // set json NULL data - uint8_t jsonNULL = TSDB_JSON_NULL; + uint8_t jsonNULL = TSDB_DATA_TYPE_NULL; int jsonIndex = startColId + 1; if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){ tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); @@ -328,7 +328,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p strncpy(varDataVal(tagKV), jsonKey, keyLen); varDataSetLen(tagKV, keyLen); if(taosHashGetSize(keyHash) == 0){ - uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL; + uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON; tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type } taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 9c4d2d94dd..2fd688a233 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -604,39 +604,55 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } -//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { -// STargetNode *target = (STargetNode *)pNode; -// -// if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { -// sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); -// ctx->code = TSDB_CODE_QRY_INVALID_INPUT; -// return DEAL_RES_ERROR; -// } -// -// SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); -// if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { -// sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); -// ctx->code = TSDB_CODE_QRY_INVALID_INPUT; -// return DEAL_RES_ERROR; -// } -// -// // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock, -// SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); -// -// SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); -// if (NULL == res) { -// sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); -// ctx->code = TSDB_CODE_QRY_APP_ERROR; -// return DEAL_RES_ERROR; -// } -// -// colDataAssign(col, res->columnData, res->numOfRows); -// block->info.rows = res->numOfRows; -// -// sclFreeParam(res); -// taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); -// return DEAL_RES_CONTINUE; -//} +EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { + STargetNode *target = (STargetNode *)pNode; + + if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + int32_t index = -1; + for(int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) { + SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, i); + if (pb->info.blockId == target->dataBlockId) { + index = i; + break; + } + } + + if (index == -1) { + sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); + + if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { + sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock, + SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + if (NULL == res) { + sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); + ctx->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + colDataAssign(col, res->columnData, res->numOfRows); + block->info.rows = res->numOfRows; + + sclFreeParam(res); + taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + return DEAL_RES_CONTINUE; +} EDealRes sclCalcWalker(SNode* pNode, void* pContext) { if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { @@ -656,9 +672,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) { return sclWalkOperator(pNode, ctx); } -// if (QUERY_NODE_TARGET == nodeType(pNode)) { -// return sclWalkTarget(pNode, ctx); -// } + if (QUERY_NODE_TARGET == nodeType(pNode)) { + return sclWalkTarget(pNode, ctx); + } sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e97831b1f6..7ad200befc 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -767,6 +767,17 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } char *getJsonValue(char *json, char *key){ //todo + int16_t cols = kvRowNCols(json); + for (int i = 0; i < cols; ++i) { + SColIdx *pColIdx = kvRowColIdxAt(json, i); + char *data = kvRowColVal(json, pColIdx); + if(i == 0 && *data == TSDB_DATA_TYPE_NULL){ + return NULL; + } + if(strncmp(key, varDataVal(data), varDataLen(data)) == 0){ + return data + varDataTLen(data); + } + } return NULL; } @@ -844,7 +855,11 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) + getVectorDoubleValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) + getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); @@ -957,7 +972,11 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - *output = getVectorDoubleValueFnLeft(pLeftCol->pData, i) - getVectorDoubleValueFnRight(pRightCol->pData, i); + if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + colDataAppendNULL(pOutputCol, i); + continue; // TODO set null or ignore + } + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - getVectorDoubleValueFnRight(RIGHT_COL, i); } pOutputCol->hasNull = (pLeftCol->hasNull || pRightCol->hasNull); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 61ef2fdce2..975effc63a 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -92,6 +92,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s blockDataEnsureCapacity(res, rows); *dataBlockId = taosArrayGetSize(pBlockList) - 1; + res->info.blockId = *dataBlockId; *slotId = 0; } else { SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); @@ -124,6 +125,56 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } +void scltMakeJsonColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; + rnode->dataBlockId = 0; + + if (NULL == *block) { + SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 3; + res->info.rows = rowNum; + res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < 2; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = TSDB_DATA_TYPE_NULL; + idata.info.bytes = 10; + idata.info.colId = i + 1; + + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)taosMemoryCalloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + } + + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 3; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)taosMemoryCalloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + + blockDataEnsureCapacity(res, rowNum); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + for (int32_t i = 0; i < rowNum; ++i) { + uint32_t len = pColumnInfoData->varmeta.length; + pColumnInfoData->varmeta.offset[i] = len; + + memcpy(pColumnInfoData->pData + len, value, varDataLen(value)); + pColumnInfoData->varmeta.length += varDataLen(value); + } + + rnode->slotId = 2; + rnode->colId = 3; + + *block = res; + } + + *pNode = (SNode *)rnode; +} void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); @@ -909,6 +960,56 @@ TEST(constantTest, greater_and_lower) { nodesDestroyNode(res); } +TEST(columnTest, smallint_value_add_json_column) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv = 1; + char *rightv= "{"k1":4,"k2":"hello","k3":null,"k4":true,"k5":5.44}"; + char key[10] = {0}; + memcpy(varDataVal(key), "k1", 2); + varDataLen(key) = 2; + + SKVRowBuilder kvRowBuilder; + tdInitKVRowBuilder(&kvRowBuilder); + parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + + double eRes[5] = {1.0, -4, -3, 24, 101}; + + SSDataBlock *src = NULL; + int32_t rowNum = 1; + + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, key); + scltMakeJsonColumnNode(&pLeft, &src, TSDB_DATA_TYPE_JSON, varDataLen(row), rowNum, row); + scltMakeOpNode(&opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); + + SNode *right = NULL; + scltMakeValueNode(&right, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, opNode, right); + + SArray *blockList = taosArrayInit(2, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); + ASSERT_EQ(code, 0); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]); + } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); +} + TEST(columnTest, smallint_value_add_int_column) { scltInitLogFile(); @@ -928,7 +1029,7 @@ TEST(columnTest, smallint_value_add_int_column) { int16_t dataBlockId = 0, slotId = 0; scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); -- GitLab