From cd2eaf015264621385c06db6744bdf3d2bb34b20 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Jul 2022 16:01:28 +0800 Subject: [PATCH] fix(query): set the output column number for subscribe. --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqExec.c | 10 +++++----- source/dnode/vnode/src/tq/tqMeta.c | 3 ++- source/libs/executor/src/dataDispatcher.c | 12 ++++++------ source/libs/executor/src/executor.c | 15 ++++++++++++++- source/libs/executor/src/executorMain.c | 1 + 8 files changed, 31 insertions(+), 16 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 8b0a836ad2..d42c4b51aa 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -63,7 +63,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); * @param SReadHandle * @return */ -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers); +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols); /** * Set the input data block for the stream scan. diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 757749a9b6..07bee22a1f 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -88,7 +88,7 @@ typedef struct { STqExecTb execTb; STqExecDb execDb; }; - + int32_t numOfCols; // number of out pout column, temporarily used } STqExecHandle; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fb05aeecd9..46964b7667 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -506,7 +506,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { .initTqReader = true, .version = ver, }; - pHandle->execHandle.execCol.task[i] = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); + pHandle->execHandle.execCol.task[i] = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, &pHandle->execHandle.numOfCols); ASSERT(pHandle->execHandle.execCol.task[i]); void* scanner = NULL; qExtractStreamScanner(pHandle->execHandle.execCol.task[i], &scanner); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index f18b25bef4..3a6afb22ac 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -15,7 +15,7 @@ #include "tq.h" -static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp) { +static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) return -1; @@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp) // TODO enable compress int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); + blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); @@ -87,7 +87,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa tqDebug("task execute end, get %p", pDataBlock); if (pDataBlock != NULL) { - tqAddBlockDataToRsp(pDataBlock, pRsp); + tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols); pRsp->blockNum++; if (pRsp->withTbName) { if (pOffset->type == TMQ_OFFSET__LOG) { @@ -195,7 +195,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } - tqAddBlockDataToRsp(&block, pRsp); + tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock)); if (pRsp->withTbName) { int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; tqAddTbNameToRsp(pTq, uid, pRsp); @@ -213,7 +213,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } - tqAddBlockDataToRsp(&block, pRsp); + tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock)); if (pRsp->withTbName) { int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; tqAddTbNameToRsp(pTq, uid, pRsp); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index e6df58696d..468490350a 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -92,7 +92,8 @@ int32_t tqMetaOpen(STQ* pTq) { .initTqReader = true, .version = handle.snapshotVer, }; - handle.execHandle.execCol.task[i] = qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader); + + handle.execHandle.execCol.task[i] = qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols); ASSERT(handle.execHandle.execCol.task[i]); void* scanner = NULL; qExtractStreamScanner(handle.execHandle.execCol.task[i], &scanner); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index b8495faffd..f0fb5852a0 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -67,15 +67,15 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { return false; } +// clang-format off // data format: -// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 -// length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) -// |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | -// actual size | | -// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+--------------+-----------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// | |sizeof(int32) |sizeof(uint64_t) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// +----------------+--------------+-----------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header +// clang-format on static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { int32_t numOfCols = 0; SNode* pNode; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 90952d5786..02edf2a7b8 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -104,7 +104,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO return code; } -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers) { +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols) { if (msg == NULL) { // TODO create raw scan return NULL; @@ -125,6 +125,19 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers) { return NULL; } + // extract the number of output columns + SDataBlockDescNode* pDescNode = plan->pNode->pOutputDataBlockDesc; + *numOfCols = 0; + + SNode* pNode; + FOREACH(pNode, pDescNode->pSlots) { + SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; + if (pSlotDesc->output) { + ++(*numOfCols); + } + } + + return pTaskInfo; } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 9d3d62cabf..c3efbf9336 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -39,6 +39,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, taosThreadOnce(&initPoolOnce, initRefPool); atexit(cleanupRefPool); + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); if (code != TSDB_CODE_SUCCESS) { goto _error; -- GitLab