From 6839ed22f242f5f818a5ab0d685900c2c6cd285e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Feb 2023 18:28:35 +0800 Subject: [PATCH] fix(query): fix bug in tag filter. --- source/dnode/vnode/src/meta/metaQuery.c | 37 ++-- source/libs/executor/src/executil.c | 236 +++--------------------- 2 files changed, 51 insertions(+), 222 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 21c9c77d4d..20f789b348 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1415,26 +1415,41 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) { if (numOfElems > 0) { pSepecifiedUidMap = taosHashInit(numOfElems / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < numOfElems; i++) { - int64_t *uid = taosArrayGet(pUidTagInfo, i); - taosHashPut(pSepecifiedUidMap, uid, sizeof(int64_t), 0, 0); + STUidTagInfo *pTagInfo = taosArrayGet(pUidTagInfo, i); + taosHashPut(pSepecifiedUidMap, &pTagInfo->uid, sizeof(uint64_t), &i, sizeof(int32_t)); } } - while (1) { - tb_uid_t uid = metaCtbCursorNext(pCur); - if (uid == 0) { - break; - } + if (numOfElems == 0) { // all data needs to be added into the pUidTagInfo list + while (1) { + tb_uid_t uid = metaCtbCursorNext(pCur); + if (uid == 0) { + break; + } - if (numOfElems > 0 && taosHashGet(pSepecifiedUidMap, &uid, sizeof(int64_t)) == NULL) { - continue; - } else if (numOfElems == 0) { STUidTagInfo info = {.uid = uid, .pTagVal = pCur->pVal}; info.pTagVal = taosMemoryMalloc(pCur->vLen); memcpy(info.pTagVal, pCur->pVal, pCur->vLen); - taosArrayPush(pUidTagInfo, &info); } + } else { // only the specified tables need to be added + while (1) { + tb_uid_t uid = metaCtbCursorNext(pCur); + if (uid == 0) { + break; + } + + int32_t *index = taosHashGet(pSepecifiedUidMap, &uid, sizeof(uint64_t)); + if (index == NULL) { + continue; + } + + STUidTagInfo *pTagInfo = taosArrayGet(pUidTagInfo, *index); + if (pTagInfo->pTagVal == NULL) { + pTagInfo->pTagVal = taosMemoryMalloc(pCur->vLen); + memcpy(pTagInfo->pTagVal, pCur->pVal, pCur->vLen); + } + } } taosHashCleanup(pSepecifiedUidMap); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ce81408a91..61aac2761f 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -48,7 +48,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond); static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); -static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList); +static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle); static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } @@ -394,188 +394,6 @@ static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -static void getColInfoResult(void* metaHandle, int64_t suid, SArray* pUidList, SNode* pTagCond) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* pBlockList = NULL; - SSDataBlock* pResBlock = NULL; - SHashObj* tags = NULL; - SScalarParam output = {0}; - - tagFilterAssist ctx = {0}; - ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); - if (ctx.colHash == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - - ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo)); - if (ctx.cInfoList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - - nodesRewriteExprPostOrder(&pTagCond, getColumn, (void*)&ctx); - - SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; - - // int64_t stt = taosGetTimestampUs(); - SArray* pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); - int32_t filter = optimizeTbnameInCond(metaHandle, suid, pUidTagList, pTagCond); - if (filter == 0) { // tbname in filter is activated, do nothing and return - int32_t numOfRows = taosArrayGetSize(pUidTagList); - code = createResultData(&type, numOfRows, &output); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - qError("failed to create result, reason:%s", tstrerror(code)); - goto end; - } - - taosArrayEnsureCap(pUidList, numOfRows); - for(int32_t i = 0; i < numOfRows; ++i) { - STUidTagInfo* pInfo = taosArrayGet(pUidTagList, i); - taosArrayPush(pUidList, &pInfo->uid); - } - - terrno = 0; - goto end; - } else { - // here we retrieve all tags from the vnode table-meta store - int32_t numOfExisted = taosArrayGetSize(pUidList); - if (numOfExisted) { - for(int32_t i = 0; i < numOfExisted; ++i) { - uint64_t* uid = taosArrayGet(pUidList, i); - STUidTagInfo info = {.uid = *uid}; - taosArrayPush(pUidTagList, &info); - } - } - - code = metaGetTableTags(metaHandle, suid, pUidTagList); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid); - terrno = code; - goto end; - } - } - - pResBlock = createDataBlock(); - if (pResBlock == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - - for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i); - blockDataAppendColInfo(pResBlock, &colInfo); - } - - int32_t numOfTables = taosArrayGetSize(pUidTagList); - if (numOfTables == 0) { - goto end; - } - - code = blockDataEnsureCapacity(pResBlock, numOfTables); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - goto end; - } - - int32_t numOfCols = taosArrayGetSize(pResBlock->pDataBlock); - - for (int32_t i = 0; i < numOfTables; i++) { - STUidTagInfo* p1 = taosArrayGet(pUidTagList, i); - - for (int32_t j = 0; j < numOfCols; j++) { - SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); - - if (pColInfo->info.colId == -1) { // tbname - char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(str, p1->name); - colDataAppend(pColInfo, i, str, false); -#if TAG_FILTER_DEBUG - qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2); -#endif - } else { - STagVal tagVal = {0}; - tagVal.cid = pColInfo->info.colId; - if (p1->pTagVal == NULL) { - colDataAppendNULL(pColInfo, i); - } - - const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); - - if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { - colDataAppendNULL(pColInfo, i); - } else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) { - colDataAppend(pColInfo, i, p, false); - } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { - char* tmp = alloca(tagVal.nData + VARSTR_HEADER_SIZE + 1); - varDataSetLen(tmp, tagVal.nData); - memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); - colDataAppend(pColInfo, i, tmp, false); -#if TAG_FILTER_DEBUG - qDebug("tagfilter varch:%s", tmp + 2); -#endif - } else { - colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false); -#if TAG_FILTER_DEBUG - if (pColInfo->info.type == TSDB_DATA_TYPE_INT) { - qDebug("tagfilter int:%d", *(int*)(&tagVal.i64)); - } else if (pColInfo->info.type == TSDB_DATA_TYPE_DOUBLE) { - qDebug("tagfilter double:%f", *(double*)(&tagVal.i64)); - } -#endif - } - } - } - } - - pResBlock->info.rows = numOfTables; - - // int64_t st1 = taosGetTimestampUs(); - // qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st); - pBlockList = taosArrayInit(2, POINTER_BYTES); - taosArrayPush(pBlockList, &pResBlock); - - code = createResultData(&type, numOfTables, &output); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - goto end; - } - - code = scalarCalculate(pTagCond, pBlockList, &output); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to calculate scalar, reason:%s", tstrerror(code)); - terrno = code; - goto end; - } - - taosArrayClear(pUidList); - - bool* pResult = (bool*)output.columnData->pData; - for(int32_t i = 0; i < numOfTables; ++i) { - uint64_t uid = ((STUidTagInfo*)taosArrayGet(pUidTagList, i))->uid; - qDebug("tagfilter get uid:%" PRId64 ", res:%d", uid, pResult[i]); - - if (pResult[i]) { - taosArrayPush(pUidList, &uid); - } - - i += 1; - } - -end: - taosHashCleanup(tags); - taosHashCleanup(ctx.colHash); - taosArrayDestroy(ctx.cInfoList); - blockDataDestroy(pResBlock); - taosArrayDestroy(pBlockList); - - colDataDestroy(output.columnData); - taosMemoryFreeClear(output.columnData); -// return output.columnData; -} - static void releaseColInfoData(void* pCol) { if (pCol) { SColumnInfoData* col = (SColumnInfoData*)pCol; @@ -584,12 +402,17 @@ static void releaseColInfoData(void* pCol) { } } +void freeItem(void* p) { + STUidTagInfo *pInfo = p; + if (pInfo->pTagVal != NULL) { + taosMemoryFree(pInfo->pTagVal); + } +} + int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; - SHashObj* tags = NULL; - SArray* uidList = NULL; void* keyBuf = NULL; SArray* groupData = NULL; @@ -618,21 +441,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis REPLACE_NODE(pNode); } - pResBlock = createDataBlock(); - if (pResBlock == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - - for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i); - blockDataAppendColInfo(pResBlock, &colInfo); - } - SArray* pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); - - uidList = taosArrayInit(rows, sizeof(uint64_t)); for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); STUidTagInfo info = {.uid = pkeyInfo->uid}; @@ -640,15 +449,15 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis } // int64_t stt = taosGetTimestampUs(); - tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); code = metaGetTableTags(metaHandle, pTableListInfo->suid, pUidTagList); if (code != TSDB_CODE_SUCCESS) { goto end; } int32_t numOfTables = taosArrayGetSize(pUidTagList); - pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList); - if (code != TSDB_CODE_SUCCESS) { + pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); + if (pResBlock == NULL) { + code = terrno; goto end; } @@ -759,12 +568,11 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis end: taosMemoryFreeClear(keyBuf); - taosHashCleanup(tags); taosHashCleanup(ctx.colHash); taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); taosArrayDestroy(pBlockList); - taosArrayDestroy(uidList); + taosArrayDestroyEx(pUidTagList, freeItem); taosArrayDestroyP(groupData, releaseColInfoData); return code; } @@ -992,7 +800,7 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { taosMemoryFree(payload); } -static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList) { +static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle) { SSDataBlock* pResBlock = createDataBlock(); if (pResBlock == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1023,7 +831,12 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa if (pColInfo->info.colId == -1) { // tbname char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(str, p1->name); + if (p1->name != NULL) { + STR_TO_VARSTR(str, p1->name); + } else { // name is not retrieved during filter + metaGetTableNameByUid(metaHandle, p1->uid, str); + } + colDataAppend(pColInfo, i, str, false); #if TAG_FILTER_DEBUG qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2); @@ -1151,8 +964,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN goto end; } - pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList); - if (code != TSDB_CODE_SUCCESS) { + pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList, metaHandle); + if (pResBlock == NULL) { + code = terrno; goto end; } @@ -1163,7 +977,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN code = createResultData(&type, numOfTables, &output); if (code != TSDB_CODE_SUCCESS) { - terrno = code; + terrno = code k; goto end; } @@ -1181,11 +995,11 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); taosArrayDestroy(pBlockList); + taosArrayDestroyEx(pUidTagList, freeItem); colDataDestroy(output.columnData); taosMemoryFreeClear(output.columnData); - - return TSDB_CODE_SUCCESS; + return code; } int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, -- GitLab