diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 838b851d6d29c2e30f08635ccfd2ee937ac571fa..0bc33739c6b46b6426421ca728df7d29791b1e37 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -210,6 +210,8 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; STqOffset* pOffset; + /*char topicName[TSDB_TOPIC_FNAME_LEN];*/ + /*int32_t vgId;*/ } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { @@ -407,6 +409,14 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { return 0; } +static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet) { + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); + ASSERT(waitingRspNum >= 0); + if (waitingRspNum == 0) { + tmqCommitDone(pParamSet); + } +} + int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; @@ -420,18 +430,13 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { #endif taosMemoryFree(pParam->pOffset); - if (pBuf->pData) taosMemoryFree(pBuf->pData); + taosMemoryFree(pBuf->pData); /*tscDebug("receive offset commit cb of %s on vgId:%d, offset is %" PRId64, pParam->pOffset->subKey, pParam->->vgId, * pOffset->version);*/ - // count down waiting rsp - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - ASSERT(waitingRspNum >= 0); + tmqCommitRspCountDown(pParamSet); - if (waitingRspNum == 0) { - tmqCommitDone(pParamSet); - } return 0; } @@ -591,14 +596,10 @@ FAIL: return 0; } -int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { +static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { int32_t code = -1; - if (msg != NULL) { - return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); - } - SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -646,33 +647,37 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t } } + // no request is sent if (pParamSet->totalRspNum == 0) { tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); return 0; } - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - ASSERT(waitingRspNum >= 0); - if (waitingRspNum == 0) { - tmqCommitDone(pParamSet); - } + // count down since waiting rsp num init as 1 + tmqCommitRspCountDown(pParamSet); if (!async) { tsem_wait(&pParamSet->rspSem); code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); - } - #if 0 - if (!async) { taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); - } #endif + } - return 0; + return code; +} + +int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { + if (msg) { + return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); + } else { + return tmqCommitConsumerImpl(tmq, automatic, async, userCb, userParam); + } } void tmqAssignAskEpTask(void* param, void* tmrId) { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 69e6cdce9f25159247ffc4b331d7a7ef701e58fa..7f418439a89f8a638a3dc3a86213518fdf0b297a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -149,7 +149,7 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); // tqRead -int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset); int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); @@ -181,8 +181,8 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink -void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t ver); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5e1cc150637f9acd2d6ff9e60717911cd4f252f6..6d71496008c80f30bdc666d9b9911e36f1412772 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -595,7 +595,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqInitTaosxRsp(&taosxRsp, pReq); if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { - tqScan(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); + tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); if (metaRsp.metaRspLen > 0) { if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { @@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqTableSink1; + pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 58d051bec1aaa1d6d88309eb4ef17f2a9c736d4e..305ee829826f626c4dd85e2128269a34865216b8 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -123,7 +123,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return 0; } -int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index ab195b1eb33ef301c20c1e0a2d80b8e125495776..79d7e27642bb8d56882f4525d614d244ba1df0d2 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -284,7 +284,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } -void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { +void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pBlocks = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; int64_t suid = pTask->tbSink.stbUid; @@ -528,7 +528,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { taosArrayDestroy(tagArray); } -void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { +void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; SBatchDeleteReq deleteReq = {0};