From 235fbfae6aacc929acb469dea0d920861014768a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 3 Nov 2021 14:40:28 +0800 Subject: [PATCH] json tag support for group by logic --- src/client/src/tscSQLParser.c | 43 ++++++++++++++++++++++++++++++++--- src/query/src/qExecutor.c | 2 +- src/util/inc/tutil.h | 1 + src/util/src/tutil.c | 15 ++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a57fa270cc..60ee31cc5f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3711,6 +3711,8 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd const char* msg6 = "tags not allowed for table query"; const char* msg7 = "not support group by expression"; const char* msg8 = "normal column can only locate at the end of group by clause"; + const char* msg9 = "json tag format error, like json->'key'"; + const char* msg10 = "-> operation only apply for json tag"; // todo : handle two tables situation STableMetaInfo* pTableMetaInfo = NULL; @@ -3750,15 +3752,29 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd tVariant* pVar = &pItem->pVar; SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; - + char* jsonName = NULL; + char* jsonKey = NULL; + int ret = getArrowKV(pVar->pz, pVar->nLen, &jsonName, &jsonKey); + if(jsonName && jsonKey) tscDebug("group by json k:%s, v:%s", jsonName, jsonKey); + if (ret == 2){ + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); + } + if(ret == 0){ + token.z = jsonName; + token.n = strlen(jsonName); + } SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(&token, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { + tfree(jsonName); + tfree(jsonKey); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { tableIndex = index.tableIndex; } else if (tableIndex != index.tableIndex) { + tfree(jsonName); + tfree(jsonKey); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -3770,12 +3786,18 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } else { pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); } - + if(ret == 0 && pSchema->type != TSDB_DATA_TYPE_JSON){ + tfree(jsonName); + tfree(jsonKey); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg10); + } int32_t numOfCols = tscGetNumOfColumns(pTableMeta); bool groupTag = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols); if (groupTag) { if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + tfree(jsonName); + tfree(jsonKey); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -3785,7 +3807,11 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } SColIndex colIndex = { .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, }; - strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); + if(ret == 0){ + tstrncpy(colIndex.name, jsonKey, tListLen(colIndex.name)); + }else{ + tstrncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); + } taosArrayPush(pGroupExpr->columnInfo, &colIndex); index.columnIndex = relIndex; @@ -3793,6 +3819,8 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { + tfree(jsonName); + tfree(jsonKey); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } @@ -3805,6 +3833,8 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; numOfGroupCols++; } + tfree(jsonName); + tfree(jsonKey); } // 1. only one normal column allowed in the group by clause @@ -7414,6 +7444,13 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; SExprInfo* pExpr = tscExprInsert(pQueryInfo, pos, f, &index, s->type, s->bytes, getNewResColId(pCmd), s->bytes, true); + // if json->key is more than TSDB_COL_NAME_LEN + TSDB_DB_NAME_LEN, truncature it， maybe case error, can encode name by md5. + if(s->type == TSDB_DATA_TYPE_JSON){ + tstrncpy(s->name, pColIndex->name, TSDB_COL_NAME_LEN); + tVariantCreateFromBinary(&(pExpr->base.param[pExpr->base.numOfParams]), pColIndex->name, + strlen(pColIndex->name), TSDB_DATA_TYPE_BINARY); + pExpr->base.numOfParams++; + } memset(pExpr->base.aliasName, 0, sizeof(pExpr->base.aliasName)); tstrncpy(pExpr->base.aliasName, s->name, sizeof(pExpr->base.aliasName)); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e652c757c4..32f6bd33be 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3449,7 +3449,7 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCt // todo use tag column index to optimize performance GET_JSON_KEY(pLocalExprInfo) - doSetTagValueInParam(pTable, param, paramLen,pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resType, + doSetTagValueInParam(pTable, param, paramLen, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resType, pLocalExprInfo->base.resBytes); if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resType) diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index dd943e8cc4..a47fcea1fd 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -38,6 +38,7 @@ char * strntolower_s(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); char * strbetween(char *string, char *begin, char *end); char * paGetToken(char *src, char **token, int32_t *tokenLen); +int getArrowKV(char* src, int32_t len, char** name, char** key); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index c8621e2b34..6edfaa6615 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -185,6 +185,21 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) { return NULL; } +// src = json->'id' => name=json key=id +int getArrowKV(char* src, int32_t len, char** name, char** key){ + char* loc = strstr(src, "->"); + if(loc == NULL){return 1;} + if(loc-src + 4 >= len) {return 2;} + if(*(loc+2) != *(src + len - 1)) {return 2;} + if(*(loc+2) != '\'' && *(loc+2) != '"') {return 2;} + + *name = calloc(1, loc-src+1); + *key = calloc(1, len - (loc-src+4) + 1); + tstrncpy(*name, src, loc-src+1); + tstrncpy(*key, loc+2, len - (loc-src+4) + 1); + return 0; +} + char *tstrstr(char *src, char *dst, bool ignoreInEsc) { if (!ignoreInEsc) { return strstr(src, dst); -- GitLab