diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 6b80257e255f852662bac280b8f12b3cb03c1a91..c2f293bbca527208b55e8a5288511b5d61309dfc 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4249,7 +4249,7 @@ static void exchangeExpr(tSqlExpr* pExpr) { tSqlExpr* pLeft = pExpr->pLeft; tSqlExpr* pRight = pExpr->pRight; - if (pRight->tokenId == TK_ID && (pLeft->tokenId == TK_INTEGER || pLeft->tokenId == TK_FLOAT || + if ((pRight->tokenId == TK_ID || pRight->tokenId == TK_ARROW) && (pLeft->tokenId == TK_INTEGER || pLeft->tokenId == TK_FLOAT || pLeft->tokenId == TK_STRING || pLeft->tokenId == TK_BOOL)) { /* * exchange value of the left handside and the value of the right-handside @@ -4487,6 +4487,8 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql const char* msg2 = "illegal column name"; const char* msg4 = "too many join tables"; const char* msg5 = "not support ordinary column join"; + const char* msg6 = "not support json tag column filter"; + const char* msg7 = "tag json key is too long, exceed to 64"; tSqlExpr* pLeft = (*pExpr)->pLeft; tSqlExpr* pRight = (*pExpr)->pRight; @@ -4603,6 +4605,15 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } + if (pSchema->type == TSDB_DATA_TYPE_JSON && pLeft != NULL && (pLeft->tokenId == TK_ID)){ + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + + if (pSchema->type == TSDB_DATA_TYPE_JSON && pLeft != NULL && (pLeft->tokenId == TK_ARROW)){ + if(pLeft->pRight && pLeft->pRight->value.nLen >= TSDB_COL_NAME_LEN) + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } + if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -4621,8 +4632,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql tSqlExpr *rexpr = NULL; if ((*pExpr)->tokenId == TK_NE && (pSchema->type != TSDB_DATA_TYPE_BINARY && pSchema->type != TSDB_DATA_TYPE_NCHAR - && pSchema->type != TSDB_DATA_TYPE_BOOL - && pSchema->type != TSDB_DATA_TYPE_JSON)) { + && pSchema->type != TSDB_DATA_TYPE_BOOL)) { handleNeOptr(&rexpr, *pExpr); *pExpr = rexpr; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 05d7d5f3534e23b05d6f9e0dce234eb4b950adf0..e2e5f9c5eab60ec7f59836306d74c19912b4009b 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -5330,6 +5330,12 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in retCode = tscSQLSyntaxErrMsg(errMsg, "json inner error", NULL); goto end; } + if(strlen(item->string) >= TSDB_COL_NAME_LEN){ + tscError("json key too long error"); + retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, exceed 64", NULL); + goto end; + } + char tagVal[TSDB_MAX_TAGS_LEN] = {0}; int32_t outLen = 0; if (JSON_TYPE_BINARY){ diff --git a/src/common/src/texpr.c b/src/common/src/texpr.c index cc2bb8803badc2aae2e80200691be0439bac3afe..29fa194b24ea1ce419b7e464bfa16975ea2f9443 100644 --- a/src/common/src/texpr.c +++ b/src/common/src/texpr.c @@ -117,6 +117,7 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { doExprTreeDestroy(&pNode, fp); } else if (pNode->nodeType == TSQL_NODE_VALUE) { tVariantDestroy(pNode->pVal); + tfree(pNode->pVal); } else if (pNode->nodeType == TSQL_NODE_COL) { tfree(pNode->pSchema); } diff --git a/src/query/inc/qFilter.h b/src/query/inc/qFilter.h index c34a56cc1cd6e135947eee897f87d060880f15c7..2ad4b43c349a6428aed88a77a11a2dcab1033ec0 100644 --- a/src/query/inc/qFilter.h +++ b/src/query/inc/qFilter.h @@ -250,7 +250,7 @@ typedef struct SFilterInfo { #define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) -#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR) +#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) #define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index 4b43288d8e16d0fcec7d49ae54abbcee7c262f61..231e7b18873b45ff2ca3d61e82f4cffe6dea6585 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -42,7 +42,12 @@ static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const v const SSchema *sch1 = desc1; const SSchema *sch2 = desc2; - return sch1->colId != sch2->colId; + if(sch1->type == TSDB_DATA_TYPE_JSON && sch2->type == TSDB_DATA_TYPE_JSON){ + return strcmp(sch1->name, sch2->name); + } + else{ + return sch1->colId != sch2->colId; + } } static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { @@ -849,21 +854,27 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); - CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR); - + int32_t type; void *v; - - if (node->nodeType == TSQL_NODE_COL) { + if(node->nodeType == TSQL_NODE_EXPR && node->_node.optr == TSDB_RELATION_ARROW){ // json tag -> operation type = FLD_TYPE_COLUMN; + assert(node->_node.pRight->pVal->nLen < TSDB_COL_NAME_LEN); + strncpy(node->_node.pLeft->pSchema->name, node->_node.pRight->pVal->pz, node->_node.pRight->pVal->nLen); v = node->pSchema; node->pSchema = NULL; - } else { - type = FLD_TYPE_VALUE; - v = node->pVal; - node->pVal = NULL; + }else{ + CHK_RET(node->nodeType != TSQL_NODE_COL && node->nodeType != TSQL_NODE_VALUE, TSDB_CODE_QRY_APP_ERROR); + if (node->nodeType == TSQL_NODE_COL) { + type = FLD_TYPE_COLUMN; + v = node->pSchema; + node->pSchema = NULL; + } else { + type = FLD_TYPE_VALUE; + v = node->pVal; + node->pVal = NULL; + } } - filterAddField(info, v, NULL, type, fid, 0, true); return TSDB_CODE_SUCCESS; @@ -1531,7 +1542,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); char *data = right->data; - if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_JSON) { + if (IS_VAR_DATA_TYPE(type) { tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } @@ -1548,7 +1559,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); char *data = right->data; - if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_JSON) { + if (IS_VAR_DATA_TYPE(type)) { tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index b8b714a5a807ddf4a6dc9878d329f599bcf3ad0d..2bc4495ca01b22d230eb08b02140b45c8466d1aa 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -473,7 +473,9 @@ bool tSqlExprIsLeaf(tSqlExpr* pExpr) { (pExpr->tokenId == TK_ID) || (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) || (pExpr->tokenId == TK_NULL) || - (pExpr->tokenId == TK_SET)); + (pExpr->tokenId == TK_SET) || + (pExpr->tokenId == TK_ARROW)|| + (pExpr->tokenId == TK_QUESTION)); } bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 11a8048864bc3e04acb3283b7ac6a3f0dc3fe21e..2da457bd3489a1f7b17305a524d81c91107eecfe 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3648,6 +3648,70 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } +//bool checkJsonKey(STable* pTable, tVariant* key){ +// void* data = taosHashGet(pTable->jsonKeyMap, key->pz, key->nLen); +// if(data == NULL) return false; +// else return true; +//} +// +//int32_t dealWithTree(STable* pTable, tExprNode* expr){ +// STSchema* pTagSchema = tsdbGetTableTagSchema(pTable); +// if(pTagSchema->columns->type != TSDB_DATA_TYPE_JSON){ +// return TSDB_CODE_SUCCESS; +// } +// +// if (expr->nodeType != TSQL_NODE_EXPR) { +// tsdbError("invalid nodeType:%d", expr->nodeType); +// return TSDB_CODE_QRY_APP_ERROR; +// } +// +// if (tree->_node.optr == TSDB_RELATION_AND) { +// leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); +// rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); +// ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); +// ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); +// +// ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); +// +// taosArrayDestroyEx(leftGroup, filterFreeGroup); +// taosArrayDestroyEx(rightGroup, filterFreeGroup); +// +// return TSDB_CODE_SUCCESS; +// } +// +// if (tree->_node.optr == TSDB_RELATION_OR) { +// ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); +// ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); +// +// return TSDB_CODE_SUCCESS; +// } +// +// tExprNode* tLeft = expr->_node.pLeft; +// +// tVariant* var = tree->_node.pRight->pVal; +// int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); +// int32_t len = 0; +// uint16_t uidx = 0; +// +// +// assert(tLeft->nodeType == TSQL_NODE_EXPR); +// +// if (tLeft->_node.optr == TSDB_RELATION_ARROW) { +// if(!checkJsonKey(pTable, tLeft->_node.pRight->pVal)){ +// tsdbError("no key in json:%d", expr->nodeType); +// return TSDB_CODE_QRY_APP_ERROR; +// } +// +// tExprTreeDestroy(tLeft, NULL); +// +// expr->_node.pLeft = calloc(1, sizeof(tExprNode)); +// +// expr->_node.pLeft->nodeType = TSQL_NODE_COL; +// expr->_node.pLeft->pSchema = calloc(1, sizeof(SSchema)); +// } +// return 0; +//} + int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { SArray* res = NULL; @@ -3711,7 +3775,6 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons } END_TRY void *filterInfo = NULL; - ret = filterInitFromTree(expr, &filterInfo, 0); if (ret != TSDB_CODE_SUCCESS) { terrno = ret; @@ -4001,18 +4064,51 @@ static void queryIndexlessColumn(SSkipList* pSkipList, void* filterInfo, SArray* tSkipListDestroyIter(iter); } +static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, void **data) { + JsonMapValue* jsonMapV = (JsonMapValue*)(param); + STable* pTable = (STable*)(jsonMapV->table); + + if (id == TSDB_TBNAME_COLUMN_INDEX) { + *data = TABLE_NAME(pTable); + } else { + *data = tdGetKVRowValOfCol(pTable->tagVal, jsonMapV->colId + 1); + } + + return TSDB_CODE_SUCCESS; +} + +static void queryByJsonTag(SArray* pTableList, void* filterInfo, SArray* res){ + int32_t size = taosArrayGetSize(pTableList); + int8_t *addToResult = NULL; + for(int i = 0; i < size; i++){ + JsonMapValue* data = taosArrayGet(pTableList, i); + filterSetColFieldData(filterInfo, data, tsdbGetJsonTagDataFromId); + bool all = filterExecute(filterInfo, 1, &addToResult, NULL, 0); + + if (all || (addToResult && *addToResult)) { + STableKeyInfo info = {.pTable = (void*)(data->table), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(res, &info); + } + } + tfree(addToResult); +} static int32_t tsdbQueryTableList(STable* pTable, SArray* pRes, void* filterInfo) { STSchema* pTSSchema = pTable->tagSchema; - bool indexQuery = false; - SSkipList *pSkipList = pTable->pIndex; - - filterIsIndexedColumnQuery(filterInfo, pTSSchema->columns->colId, &indexQuery); - - if (indexQuery) { - queryIndexedColumn(pSkipList, filterInfo, pRes); - } else { - queryIndexlessColumn(pSkipList, filterInfo, pRes); + + if(pTSSchema->columns->type == TSDB_DATA_TYPE_JSON){ + queryByJsonTag(pTable, filterInfo, pRes); + }else{ + bool indexQuery = false; + SSkipList *pSkipList = pTable->pIndex; + + filterIsIndexedColumnQuery(filterInfo, pTSSchema->columns->colId, &indexQuery); + + if (indexQuery) { + queryIndexedColumn(pSkipList, filterInfo, pRes); + } else { + queryIndexlessColumn(pSkipList, filterInfo, pRes); + } } return TSDB_CODE_SUCCESS;